From c871a5511604288fa1ec3480890f53b5de73f4a6 Mon Sep 17 00:00:00 2001 From: lb <542828+lukebuehler@users.noreply.github.com> Date: Tue, 27 Oct 2026 13:51:03 +0100 Subject: [PATCH 1/5] envd review and hardening --- Cargo.lock | 22 +- clients/typescript/schema/api.schema.json | 578 +- clients/typescript/src/generated/methods.ts | 30 +- clients/typescript/src/generated/types.ts | 270 +- crates/api/contract/api-reference.md | 10 +- crates/api/contract/api.schema.json | 578 +- crates/api/contract/methods.json | 10 +- crates/api/contract/openrpc.json | 588 +- crates/api/src/deployment.rs | 68 +- crates/api/src/environments.rs | 187 +- crates/api/src/handshake.rs | 5 +- crates/api/src/rpc.rs | 4 +- crates/api/src/tests.rs | 38 +- crates/cli/src/env_cli.rs | 14 +- crates/cli/tests/resource_commands.rs | 8 +- crates/environment-client/Cargo.toml | 15 +- .../src/conformance/controller.rs | 631 ++ .../environment-client/src/conformance/mod.rs | 8 + crates/environment-client/src/connection.rs | 774 +++ crates/environment-client/src/control.rs | 132 - crates/environment-client/src/controller.rs | 169 + crates/environment-client/src/data.rs | 472 +- crates/environment-client/src/dial.rs | 244 + crates/environment-client/src/error.rs | 227 +- crates/environment-client/src/lib.rs | 76 +- crates/environment-client/src/options.rs | 112 + crates/environment-client/src/rpc.rs | 169 - crates/environment-client/src/transport.rs | 114 - crates/environment-client/tests/client.rs | 897 ++- crates/environment-daemon/Cargo.toml | 7 +- crates/environment-daemon/src/config.rs | 473 +- crates/environment-daemon/src/dispatch.rs | 278 + crates/environment-daemon/src/exec.rs | 323 + .../environment-daemon/src/exec/execution.rs | 551 ++ crates/environment-daemon/src/exec/input.rs | 310 + .../environment-daemon/src/exec/job_store.rs | 613 ++ crates/environment-daemon/src/exec/log.rs | 422 ++ crates/environment-daemon/src/exec/outcome.rs | 179 + .../src/exec/spawn_record.rs | 101 + crates/environment-daemon/src/filesystem.rs | 2180 ++++--- .../src/filesystem/backend/mod.rs | 85 - .../src/filesystem/backend/unix.rs | 504 -- .../src/filesystem/confinement.rs | 721 +++ .../environment-daemon/src/filesystem/os.rs | 487 ++ .../environment-daemon/src/filesystem/scan.rs | 812 ++- .../src/filesystem/search.rs | 546 ++ .../src/filesystem/transfer/capture.rs | 416 ++ .../src/filesystem/transfer/journal.rs | 262 +- .../src/filesystem/transfer/mod.rs | 1383 +++- .../src/filesystem/transfer/publish.rs | 970 +++ .../src/filesystem/transfer/session.rs | 1531 ----- .../src/filesystem/transfer/tests.rs | 2015 +++--- .../environment-daemon/src/filesystem/walk.rs | 754 +++ crates/environment-daemon/src/identity.rs | 189 +- crates/environment-daemon/src/jobs.rs | 3557 +++++----- crates/environment-daemon/src/lib.rs | 707 +- crates/environment-daemon/src/main.rs | 109 +- crates/environment-daemon/src/process.rs | 2792 +++----- .../environment-daemon/src/process_group.rs | 367 +- crates/environment-daemon/src/redaction.rs | 2 +- crates/environment-daemon/src/registration.rs | 1637 ++++- crates/environment-daemon/src/rpc.rs | 82 - crates/environment-daemon/src/server.rs | 1403 ++-- crates/environment-daemon/src/upgrade.rs | 416 +- .../tests/client_round_trip.rs | 497 ++ crates/environment-protocol/Cargo.toml | 12 + .../contract/controller.openrpc.json | 1701 +++++ .../contract/controller.schema.json | 1324 ++++ .../contract/data.openrpc.json | 5768 +++++++++++++++++ .../contract/data.schema.json | 4285 ++++++++++++ .../contract/registration.openrpc.json | 1420 ++++ .../contract/registration.schema.json | 1117 ++++ .../fixtures/bootstrap/build-facts.json | 7 + .../bootstrap/discovery-document.json | 14 + .../fixtures/controller/ingress.ensure.json | 49 + .../fixtures/controller/ingress.remove.json | 45 + .../fixtures/controller/initialize.json | 33 + .../fixtures/controller/target.adopt.json | 44 + .../fixtures/controller/target.close.json | 33 + .../fixtures/controller/target.create.json | 34 + .../fixtures/controller/target.get.json | 48 + .../fixtures/controller/target.setPower.json | 46 + .../fixtures/controller/template.list.json | 34 + .../controller_create_target_params.json | 10 - .../controller_create_target_response.json | 36 - .../controller_initialize_params.json | 4 - .../controller_list_targets_response.json | 38 - .../fixtures/data/env.getIdle.json | 15 + .../fixtures/data/fs.createDirectory.json | 15 + .../fixtures/data/fs.getMetadata.json | 24 + .../fixtures/data/fs.globFiles.json | 33 + .../fixtures/data/fs.readDirectory.json | 40 + .../fixtures/data/fs.readFile.json | 21 + .../fixtures/data/fs.remove.json | 16 + .../fixtures/data/fs.rename.json | 16 + .../fixtures/data/fs.scan.json | 89 + .../fixtures/data/fs.searchText.json | 47 + .../fixtures/data/fs.writeFile.json | 20 + .../fixtures/data/initialize.json | 97 + .../fixtures/data/job.cancel.json | 43 + .../fixtures/data/job.read.json | 115 + .../fixtures/data/job.start.json | 75 + .../fixtures/data/process.read.json | 63 + .../fixtures/data/process.start.json | 38 + .../fixtures/data/process.terminate.json | 17 + .../fixtures/data/process.write.json | 18 + .../fixtures/data/rpc.cancelRequest.json | 5 + .../fixtures/data/transfer.abort.json | 22 + .../fixtures/data/transfer.advance.json | 38 + .../fixtures/data/transfer.append.json | 40 + .../fixtures/data/transfer.begin.json | 43 + .../fixtures/data/transfer.commit.json | 22 + .../fixtures/data/transfer.getStatus.json | 23 + .../fixtures/data/transfer.listMissing.json | 20 + .../fixtures/data/transfer.read.json | 18 + .../fixtures/data/transfer.readInventory.json | 35 + .../fixtures/data/transfer.write.json | 19 + .../fixtures/data_initialize_params.json | 9 - .../fixtures/fs_read_file_response.json | 3 - .../fixtures/job_read_response.json | 39 - .../fixtures/job_start_params.json | 27 - .../fixtures/process_read_response.json | 25 - .../fixtures/process_start_params.json | 15 - .../registration/registration.challenge.json | 6 + .../registration/registration.closed.json | 6 + .../registration/registration.goodbye.json | 5 + .../registration/registration.openStream.json | 16 + .../registration/registration.register.json | 37 + .../src/bin/export-contract.rs | 25 + crates/environment-protocol/src/bootstrap.rs | 70 + crates/environment-protocol/src/bytes.rs | 303 + crates/environment-protocol/src/contract.rs | 345 + .../src/control/handshake.rs | 70 +- .../src/control/ingress.rs | 47 +- .../src/control/methods.rs | 13 - .../environment-protocol/src/control/mod.rs | 22 +- .../src/control/target.rs | 230 + .../src/control/targets.rs | 250 - .../src/control/template.rs | 63 + .../environment-protocol/src/data/events.rs | 43 - crates/environment-protocol/src/data/exec.rs | 373 ++ crates/environment-protocol/src/data/fs.rs | 545 +- .../src/data/handshake.rs | 231 +- crates/environment-protocol/src/data/idle.rs | 61 +- .../src/data/inventory.rs | 122 - crates/environment-protocol/src/data/job.rs | 302 + crates/environment-protocol/src/data/jobs.rs | 240 - .../environment-protocol/src/data/methods.rs | 36 - crates/environment-protocol/src/data/mod.rs | 12 +- .../environment-protocol/src/data/process.rs | 258 +- crates/environment-protocol/src/data/scan.rs | 231 + .../environment-protocol/src/data/transfer.rs | 442 +- .../src/data/transfer_session.rs | 149 - crates/environment-protocol/src/error.rs | 870 ++- crates/environment-protocol/src/gateway.rs | 8 - crates/environment-protocol/src/handshake.rs | 257 + crates/environment-protocol/src/ids.rs | 334 + crates/environment-protocol/src/jsonrpc.rs | 601 ++ crates/environment-protocol/src/lib.rs | 100 +- crates/environment-protocol/src/limits.rs | 80 + crates/environment-protocol/src/manifest.rs | 651 ++ crates/environment-protocol/src/metadata.rs | 400 ++ crates/environment-protocol/src/paths.rs | 475 ++ .../environment-protocol/src/registration.rs | 647 +- crates/environment-protocol/src/routes.rs | 464 ++ crates/environment-protocol/src/serde_util.rs | 13 + crates/environment-protocol/src/shared.rs | 461 -- crates/environment-protocol/src/version.rs | 17 + crates/environment-protocol/tests/contract.rs | 47 + crates/environment-protocol/tests/fixtures.rs | 203 + crates/environment-protocol/tests/serde.rs | 516 +- crates/environment-provider-incus/CLUSTER.md | 5 +- crates/environment-provider-incus/Cargo.toml | 7 +- crates/environment-provider-incus/README.md | 88 +- .../config.cluster.example.json | 2 + .../config.example.json | 2 + .../environment-provider-incus/src/config.rs | 354 +- crates/environment-provider-incus/src/edge.rs | 28 +- .../environment-provider-incus/src/incus.rs | 943 ++- .../environment-provider-incus/src/ingress.rs | 36 +- .../environment-provider-incus/src/policy.rs | 115 +- .../environment-provider-incus/src/relay.rs | 140 +- .../environment-provider-incus/src/server.rs | 2127 ++++-- .../tests/incus_live.rs | 224 +- crates/environments/src/endpoint.rs | 270 + crates/environments/src/lib.rs | 680 +- crates/environments/src/memory.rs | 388 +- crates/environments/src/tests.rs | 467 +- crates/harness/src/storage/blobs.rs | 13 +- crates/harness/src/storage/mod.rs | 3 +- .../tests/fixtures/builtin_catalogs.json | 6 +- crates/profiles/src/lib.rs | 9 +- crates/store-fs/src/blob.rs | 2 +- .../migrations/012_environment_endpoints.sql | 138 + crates/store-pg/src/auth.rs | 8 +- crates/store-pg/src/blob.rs | 2 +- crates/store-pg/src/environment.rs | 660 +- .../store-pg/src/environment_registration.rs | 10 +- crates/store-pg/src/lib.rs | 29 +- crates/store-pg/src/migrations.rs | 7 +- crates/store-pg/tests/store_pg_live.rs | 257 +- crates/temporal-runtime/Cargo.toml | 1 + crates/temporal-runtime/src/bots/fires.rs | 27 +- crates/temporal-runtime/src/config.rs | 18 - .../src/credential_injection.rs | 138 +- .../src/environments/fake_controller.rs | 754 +++ .../src/environments/gateway.rs | 129 +- .../src/environments/lifecycle.rs | 1489 ++++- .../temporal-runtime/src/environments/mod.rs | 9 +- .../src/environments/power.rs | 133 +- .../src/environments/prompts.rs | 96 +- .../src/environments/provider_controllers.rs | 1340 ++-- .../src/environments/providers.rs | 103 +- .../src/environments/registration.rs | 188 +- .../src/environments/resolver.rs | 699 +- .../src/environments/runtime.rs | 13 +- .../src/environments/skills.rs | 79 +- .../src/environments/sources.rs | 160 +- .../src/gateway/deployment.rs | 141 +- crates/temporal-runtime/src/gateway/http.rs | 769 ++- .../src/gateway/registration.rs | 1913 ++++-- .../src/gateway/request_context.rs | 2 +- .../src/gateway/service/mod.rs | 6 +- .../src/gateway/service/session_jobs.rs | 1028 +-- .../src/gateway/service/tests.rs | 1 + crates/temporal-runtime/src/main.rs | 17 +- crates/temporal-runtime/src/universe.rs | 38 +- .../src/worker/activities/environment_jobs.rs | 935 ++- .../src/worker/activities/mod.rs | 2 +- .../src/worker/session_tools.rs | 328 +- .../tests/code_capabilities_live.rs | 15 +- .../tests/environment_provider_live.rs | 147 +- .../tests/environment_registration_live.rs | 631 +- .../temporal-runtime/tests/profiles_live.rs | 21 +- .../temporal-runtime/tests/subagents_live.rs | 14 +- .../tests/vfs_transfer_live.rs | 307 +- crates/temporal-workflow/Cargo.toml | 1 - crates/temporal-workflow/src/activities.rs | 11 +- .../temporal-workflow/src/environment_jobs.rs | 495 ++ crates/temporal-workflow/src/lib.rs | 50 +- crates/temporal-workflow/src/types.rs | 129 - .../src/workflows/environment_job.rs | 265 +- .../src/workflows/session/preparation.rs | 74 +- crates/tools/Cargo.toml | 5 +- crates/tools/src/builtin/canonical.rs | 15 +- .../tools/src/builtin/output_schema_tests.rs | 61 +- crates/tools/src/builtin/shared.rs | 3 + crates/tools/src/environment.rs | 1 + crates/tools/src/environment/jobs.rs | 922 ++- crates/tools/src/environment/output.rs | 411 ++ crates/tools/src/environment/process/mod.rs | 2 +- crates/tools/src/environment/sources.rs | 80 +- crates/tools/src/environment/tools/jobs.rs | 273 +- .../src/environment_protocol/conformance.rs | 1517 ++++- crates/tools/src/environment_protocol/mod.rs | 8 +- .../tools/src/environment_protocol/remote.rs | 2084 +++--- crates/tools/src/fs/apply_patch/engine.rs | 41 +- crates/tools/src/fs/local.rs | 77 +- crates/tools/src/fs/memory.rs | 103 +- crates/tools/src/fs/mod.rs | 120 +- crates/tools/src/fs/pattern.rs | 141 + crates/tools/src/fs/read_only.rs | 52 +- crates/tools/src/fs/scoped.rs | 46 +- crates/tools/src/fs/scoped_local.rs | 96 +- crates/tools/src/fs/tools/edit_file.rs | 18 +- crates/tools/src/fs/tools/glob.rs | 165 +- crates/tools/src/fs/tools/grep.rs | 142 +- crates/tools/src/fs/tools/mod.rs | 1 + crates/tools/src/fs/tools/read_file.rs | 29 +- crates/tools/src/fs/tools/shared.rs | 137 +- crates/tools/src/fs/tools/write_file.rs | 2 +- crates/tools/src/fs/vfs.rs | 182 +- crates/tools/src/prompts/environment.rs | 135 +- crates/tools/src/skills/environment.rs | 293 +- crates/tools/src/transfer.rs | 846 ++- .../documentation/deployment/configuration.md | 3 +- docs/documentation/deployment/self-hosting.md | 9 +- .../deployment/upgrades-and-recovery.md | 20 +- .../development/changing-contracts.md | 51 +- .../development/testing-and-evaluation.md | 6 +- .../environments/bring-your-own-compute.md | 34 +- docs/documentation/environments/incus-vms.md | 83 +- .../environments/networking-and-ingress.md | 27 +- docs/documentation/environments/overview.md | 9 +- .../environments/power-and-cleanup.md | 33 +- .../environments/processes-and-jobs.md | 108 +- .../environments/using-environments.md | 13 + .../environments/vfs-transfer.md | 112 +- docs/documentation/index.md | 4 +- .../environment-providers.md | 466 +- .../reference/environment-protocol.md | 640 ++ .../reference/environment-variables.md | 57 +- .../using-lightspeed/workspaces-and-skills.md | 8 +- docs/site/astro.config.mjs | 1 + .../src/routes/environment-deployment.ts | 14 +- platform/backend/src/routes/gateway.ts | 5 +- .../configurator-mcp/src/generated/tools.ts | 70 +- .../src/demo/fixtures/personal-assistant.ts | 2 +- platform/web/src/demo/fixtures/platform.ts | 7 +- .../web/src/demo/fixtures/software-factory.ts | 2 +- platform/web/src/demo/routes/admin.ts | 49 +- platform/web/src/demo/routes/environments.ts | 15 +- .../pages/AdminEnvironmentProvidersPage.tsx | 95 +- platform/web/src/pages/EnvironmentsPage.tsx | 19 +- release/metadata.env | 11 +- release/release-manifest.schema.json | 6 +- scripts/release/create-manifest.mjs | 19 +- scripts/release/smoke.sh | 2 +- scripts/release/verify-manifest.mjs | 9 +- scripts/release/verify-metadata.sh | 6 +- 310 files changed, 68930 insertions(+), 22964 deletions(-) create mode 100644 crates/environment-client/src/conformance/controller.rs create mode 100644 crates/environment-client/src/conformance/mod.rs create mode 100644 crates/environment-client/src/connection.rs delete mode 100644 crates/environment-client/src/control.rs create mode 100644 crates/environment-client/src/controller.rs create mode 100644 crates/environment-client/src/dial.rs create mode 100644 crates/environment-client/src/options.rs delete mode 100644 crates/environment-client/src/rpc.rs delete mode 100644 crates/environment-client/src/transport.rs create mode 100644 crates/environment-daemon/src/dispatch.rs create mode 100644 crates/environment-daemon/src/exec.rs create mode 100644 crates/environment-daemon/src/exec/execution.rs create mode 100644 crates/environment-daemon/src/exec/input.rs create mode 100644 crates/environment-daemon/src/exec/job_store.rs create mode 100644 crates/environment-daemon/src/exec/log.rs create mode 100644 crates/environment-daemon/src/exec/outcome.rs create mode 100644 crates/environment-daemon/src/exec/spawn_record.rs delete mode 100644 crates/environment-daemon/src/filesystem/backend/mod.rs delete mode 100644 crates/environment-daemon/src/filesystem/backend/unix.rs create mode 100644 crates/environment-daemon/src/filesystem/confinement.rs create mode 100644 crates/environment-daemon/src/filesystem/os.rs create mode 100644 crates/environment-daemon/src/filesystem/search.rs create mode 100644 crates/environment-daemon/src/filesystem/transfer/capture.rs create mode 100644 crates/environment-daemon/src/filesystem/transfer/publish.rs delete mode 100644 crates/environment-daemon/src/filesystem/transfer/session.rs create mode 100644 crates/environment-daemon/src/filesystem/walk.rs delete mode 100644 crates/environment-daemon/src/rpc.rs create mode 100644 crates/environment-daemon/tests/client_round_trip.rs create mode 100644 crates/environment-protocol/contract/controller.openrpc.json create mode 100644 crates/environment-protocol/contract/controller.schema.json create mode 100644 crates/environment-protocol/contract/data.openrpc.json create mode 100644 crates/environment-protocol/contract/data.schema.json create mode 100644 crates/environment-protocol/contract/registration.openrpc.json create mode 100644 crates/environment-protocol/contract/registration.schema.json create mode 100644 crates/environment-protocol/fixtures/bootstrap/build-facts.json create mode 100644 crates/environment-protocol/fixtures/bootstrap/discovery-document.json create mode 100644 crates/environment-protocol/fixtures/controller/ingress.ensure.json create mode 100644 crates/environment-protocol/fixtures/controller/ingress.remove.json create mode 100644 crates/environment-protocol/fixtures/controller/initialize.json create mode 100644 crates/environment-protocol/fixtures/controller/target.adopt.json create mode 100644 crates/environment-protocol/fixtures/controller/target.close.json create mode 100644 crates/environment-protocol/fixtures/controller/target.create.json create mode 100644 crates/environment-protocol/fixtures/controller/target.get.json create mode 100644 crates/environment-protocol/fixtures/controller/target.setPower.json create mode 100644 crates/environment-protocol/fixtures/controller/template.list.json delete mode 100644 crates/environment-protocol/fixtures/controller_create_target_params.json delete mode 100644 crates/environment-protocol/fixtures/controller_create_target_response.json delete mode 100644 crates/environment-protocol/fixtures/controller_initialize_params.json delete mode 100644 crates/environment-protocol/fixtures/controller_list_targets_response.json create mode 100644 crates/environment-protocol/fixtures/data/env.getIdle.json create mode 100644 crates/environment-protocol/fixtures/data/fs.createDirectory.json create mode 100644 crates/environment-protocol/fixtures/data/fs.getMetadata.json create mode 100644 crates/environment-protocol/fixtures/data/fs.globFiles.json create mode 100644 crates/environment-protocol/fixtures/data/fs.readDirectory.json create mode 100644 crates/environment-protocol/fixtures/data/fs.readFile.json create mode 100644 crates/environment-protocol/fixtures/data/fs.remove.json create mode 100644 crates/environment-protocol/fixtures/data/fs.rename.json create mode 100644 crates/environment-protocol/fixtures/data/fs.scan.json create mode 100644 crates/environment-protocol/fixtures/data/fs.searchText.json create mode 100644 crates/environment-protocol/fixtures/data/fs.writeFile.json create mode 100644 crates/environment-protocol/fixtures/data/initialize.json create mode 100644 crates/environment-protocol/fixtures/data/job.cancel.json create mode 100644 crates/environment-protocol/fixtures/data/job.read.json create mode 100644 crates/environment-protocol/fixtures/data/job.start.json create mode 100644 crates/environment-protocol/fixtures/data/process.read.json create mode 100644 crates/environment-protocol/fixtures/data/process.start.json create mode 100644 crates/environment-protocol/fixtures/data/process.terminate.json create mode 100644 crates/environment-protocol/fixtures/data/process.write.json create mode 100644 crates/environment-protocol/fixtures/data/rpc.cancelRequest.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.abort.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.advance.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.append.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.begin.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.commit.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.getStatus.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.listMissing.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.read.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.readInventory.json create mode 100644 crates/environment-protocol/fixtures/data/transfer.write.json delete mode 100644 crates/environment-protocol/fixtures/data_initialize_params.json delete mode 100644 crates/environment-protocol/fixtures/fs_read_file_response.json delete mode 100644 crates/environment-protocol/fixtures/job_read_response.json delete mode 100644 crates/environment-protocol/fixtures/job_start_params.json delete mode 100644 crates/environment-protocol/fixtures/process_read_response.json delete mode 100644 crates/environment-protocol/fixtures/process_start_params.json create mode 100644 crates/environment-protocol/fixtures/registration/registration.challenge.json create mode 100644 crates/environment-protocol/fixtures/registration/registration.closed.json create mode 100644 crates/environment-protocol/fixtures/registration/registration.goodbye.json create mode 100644 crates/environment-protocol/fixtures/registration/registration.openStream.json create mode 100644 crates/environment-protocol/fixtures/registration/registration.register.json create mode 100644 crates/environment-protocol/src/bin/export-contract.rs create mode 100644 crates/environment-protocol/src/bootstrap.rs create mode 100644 crates/environment-protocol/src/bytes.rs create mode 100644 crates/environment-protocol/src/contract.rs delete mode 100644 crates/environment-protocol/src/control/methods.rs create mode 100644 crates/environment-protocol/src/control/target.rs delete mode 100644 crates/environment-protocol/src/control/targets.rs create mode 100644 crates/environment-protocol/src/control/template.rs delete mode 100644 crates/environment-protocol/src/data/events.rs create mode 100644 crates/environment-protocol/src/data/exec.rs delete mode 100644 crates/environment-protocol/src/data/inventory.rs create mode 100644 crates/environment-protocol/src/data/job.rs delete mode 100644 crates/environment-protocol/src/data/jobs.rs delete mode 100644 crates/environment-protocol/src/data/methods.rs create mode 100644 crates/environment-protocol/src/data/scan.rs delete mode 100644 crates/environment-protocol/src/data/transfer_session.rs delete mode 100644 crates/environment-protocol/src/gateway.rs create mode 100644 crates/environment-protocol/src/handshake.rs create mode 100644 crates/environment-protocol/src/ids.rs create mode 100644 crates/environment-protocol/src/jsonrpc.rs create mode 100644 crates/environment-protocol/src/limits.rs create mode 100644 crates/environment-protocol/src/manifest.rs create mode 100644 crates/environment-protocol/src/metadata.rs create mode 100644 crates/environment-protocol/src/paths.rs create mode 100644 crates/environment-protocol/src/routes.rs create mode 100644 crates/environment-protocol/src/serde_util.rs delete mode 100644 crates/environment-protocol/src/shared.rs create mode 100644 crates/environment-protocol/src/version.rs create mode 100644 crates/environment-protocol/tests/contract.rs create mode 100644 crates/environment-protocol/tests/fixtures.rs create mode 100644 crates/environments/src/endpoint.rs create mode 100644 crates/store-pg/migrations/012_environment_endpoints.sql create mode 100644 crates/temporal-runtime/src/environments/fake_controller.rs create mode 100644 crates/temporal-workflow/src/environment_jobs.rs create mode 100644 crates/tools/src/environment/output.rs create mode 100644 crates/tools/src/fs/pattern.rs create mode 100644 docs/documentation/reference/environment-protocol.md diff --git a/Cargo.lock b/Cargo.lock index 074ab70de..f73f85622 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1304,7 +1304,7 @@ dependencies = [ name = "environment-client" version = "0.1.0" dependencies = [ - "async-trait", + "environment-client", "environment-protocol", "futures-util", "serde", @@ -1319,8 +1319,6 @@ name = "environment-daemon" version = "0.4.0" dependencies = [ "anyhow", - "api", - "async-trait", "clap", "dotenvy", "ed25519-dalek", @@ -1328,7 +1326,7 @@ dependencies = [ "environment-protocol", "flate2", "futures-util", - "glob", + "globset", "grep-regex", "grep-searcher", "harness", @@ -1351,6 +1349,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tokio-tungstenite 0.30.0", + "tokio-util", "tools", "vfs", "webpki-roots 1.0.9", @@ -1361,9 +1360,12 @@ name = "environment-protocol" version = "0.1.0" dependencies = [ "base64 0.23.1", + "environment-protocol", "schemars", "serde", "serde_json", + "serde_path_to_error", + "sha2 0.11.0", "thiserror 2.0.21", ] @@ -1376,6 +1378,7 @@ dependencies = [ "axum", "clap", "dotenvy", + "environment-client", "environment-protocol", "futures-util", "release-info", @@ -1387,6 +1390,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tokio-tungstenite 0.30.0", + "tools", "uuid", ] @@ -1854,12 +1858,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - [[package]] name = "globset" version = "0.4.20" @@ -5281,7 +5279,6 @@ dependencies = [ "bots", "channels", "chrono", - "environment-protocol", "futures", "futures-util", "harness", @@ -5934,8 +5931,9 @@ dependencies = [ "environment-client", "environment-protocol", "futures-util", - "glob", + "globset", "harness", + "ignore", "jsonschema", "regex", "reqwest", diff --git a/clients/typescript/schema/api.schema.json b/clients/typescript/schema/api.schema.json index d2bce1b0a..f36246cf9 100644 --- a/clients/typescript/schema/api.schema.json +++ b/clients/typescript/schema/api.schema.json @@ -8763,21 +8763,6 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderConnection": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/definitions/DeploymentEnvironmentProviderTransport" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "DeploymentEnvironmentProviderDeleteParams": { "properties": { "providerId": { @@ -8800,6 +8785,24 @@ ], "type": "object" }, + "DeploymentEnvironmentProviderFeaturesView": { + "description": "Optional controller methods the provider's controller advertised.", + "properties": { + "adoptTarget": { + "description": "Existing targets can be adopted (`environment/adopt`).", + "type": "boolean" + }, + "ingress": { + "description": "Public ingress can be enabled on environments.", + "type": "boolean" + } + }, + "required": [ + "adoptTarget", + "ingress" + ], + "type": "object" + }, "DeploymentEnvironmentProviderListParams": { "type": "object" }, @@ -8818,9 +8821,18 @@ "type": "object" }, "DeploymentEnvironmentProviderPutParams": { + "description": "Register or replace a provider. `credential` is the bearer token\nLightspeed presents to the provider's controller and data routes: it is\nsealed on receipt and never returned by any method. `Debug` output\nredacts it; request logging must never echo these params.", "properties": { - "controllerConnection": { - "$ref": "#/definitions/DeploymentEnvironmentProviderConnection" + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" + }, + "credential": { + "description": "Required when the provider is new; omit to keep the stored one.", + "type": [ + "string", + "null" + ] }, "displayName": { "type": [ @@ -8840,7 +8852,7 @@ }, "required": [ "providerId", - "controllerConnection" + "controllerEndpoint" ], "type": "object" }, @@ -8877,83 +8889,34 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderTransport": { - "oneOf": [ - { - "properties": { - "type": { - "const": "webSocket", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "http", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "stdio", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "ssh", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" + "DeploymentEnvironmentProviderView": { + "properties": { + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" }, - { - "properties": { - "providerType": { - "type": "string" + "controllerFeatures": { + "anyOf": [ + { + "$ref": "#/definitions/DeploymentEnvironmentProviderFeaturesView" }, - "type": { - "const": "provider", - "type": "string" + { + "type": "null" } - }, - "required": [ - "type", - "providerType" ], - "type": "object" - } - ] - }, - "DeploymentEnvironmentProviderView": { - "properties": { - "controllerConnection": { - "$ref": "#/definitions/DeploymentEnvironmentProviderConnection" + "description": "What the controller advertised at its last successful handshake;\nabsent until Lightspeed first reached it at this endpoint." }, "createdAtMs": { "format": "int64", "type": "integer" }, + "dataRouteBase": { + "description": "Base URL of the provider's data routes, as its controller advertised\nit; absent until then.", + "type": [ + "string", + "null" + ] + }, "displayName": { "type": [ "string", @@ -8976,7 +8939,7 @@ }, "required": [ "providerId", - "controllerConnection", + "controllerEndpoint", "createdAtMs", "updatedAtMs" ], @@ -9339,54 +9302,6 @@ ], "type": "object" }, - "EnvironmentConnectionTransportView": { - "oneOf": [ - { - "enum": [ - "webSocket", - "http", - "stdio", - "ssh" - ], - "type": "string" - }, - { - "additionalProperties": false, - "properties": { - "provider": { - "properties": { - "provider_type": { - "type": "string" - } - }, - "required": [ - "provider_type" - ], - "type": "object" - } - }, - "required": [ - "provider" - ], - "type": "object" - } - ] - }, - "EnvironmentConnectionView": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/definitions/EnvironmentConnectionTransportView" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "EnvironmentCreateParams": { "properties": { "bindingId": { @@ -9609,7 +9524,7 @@ "type": "string" }, "protocolVersion": { - "description": "Environment protocol version the gateway speaks. A daemon registers\nonly when it reports exactly this number.", + "description": "The envd protocol version (data plane and registration channel)\nthe gateway speaks. A daemon registers only when it reports exactly\nthis number.", "format": "uint32", "minimum": 0, "type": "integer" @@ -9635,20 +9550,16 @@ }, "EnvironmentExternalCreateParams": { "properties": { - "connection": { - "allOf": [ - { - "$ref": "#/definitions/EnvironmentConnectionView" - } - ], - "description": "Connection to an envd instance reachable from Lightspeed." - }, "displayName": { "type": [ "string", "null" ] }, + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of an envd instance reachable from\nLightspeed.", + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" @@ -9661,7 +9572,7 @@ }, "required": [ "requestId", - "connection" + "endpoint" ], "type": "object" }, @@ -9918,7 +9829,8 @@ }, "EnvironmentJobReadParams": { "properties": { - "afterSeq": { + "after": { + "description": "Byte offset to read each job's output from: a previous entry's\n`outputNext`. Omitted with `tailBytes` omitted reads from the start.", "format": "uint64", "minimum": 0, "type": [ @@ -9926,18 +9838,24 @@ "null" ] }, - "includeArtifacts": { - "default": false, - "type": "boolean" - }, "jobs": { "items": { "$ref": "#/definitions/SessionJobHandleInput" }, "type": "array" }, - "outputBytes": { - "format": "uint", + "maxBytes": { + "description": "Most output bytes per job; omit for the server's default page.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "tailBytes": { + "description": "Read each job's last `tailBytes` of output instead; excludes `after`.", + "format": "uint64", "minimum": 0, "type": [ "integer", @@ -9962,6 +9880,29 @@ }, "type": "object" }, + "EnvironmentLifecycleErrorView": { + "properties": { + "firstAtMs": { + "description": "When the current run of errors began.", + "format": "int64", + "type": "integer" + }, + "lastAtMs": { + "description": "When `message` was recorded.", + "format": "int64", + "type": "integer" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message", + "firstAtMs", + "lastAtMs" + ], + "type": "object" + }, "EnvironmentLifecycleStatusView": { "oneOf": [ { @@ -10500,8 +10441,9 @@ }, { "properties": { - "connection": { - "$ref": "#/definitions/EnvironmentConnectionView" + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL Lightspeed dials envd at.", + "type": "string" }, "type": { "const": "external", @@ -10510,7 +10452,7 @@ }, "required": [ "type", - "connection" + "endpoint" ], "type": "object" }, @@ -10592,6 +10534,36 @@ ], "type": "object" }, + "EnvironmentTemplateResourcesView": { + "description": "A template's compute, in provider-neutral units. Each field is absent\nwhen the provider does not fix it.", + "properties": { + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, "EnvironmentTemplateView": { "properties": { "bindingId": { @@ -10621,6 +10593,17 @@ "publicIngress": { "type": "boolean" }, + "resources": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentTemplateResourcesView" + }, + { + "type": "null" + } + ], + "description": "Compute targets of this template get; absent when the provider does\nnot fix it." + }, "templateId": { "type": "string" } @@ -10679,6 +10662,17 @@ "null" ] }, + "lifecycleError": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentLifecycleErrorView" + }, + { + "type": "null" + } + ], + "description": "Why the environment failed, or the latest error (or what the\nprovider reports it is waiting on) while its lifecycle converges;\nabsent once it reaches a steady state." + }, "metadata": { "additionalProperties": { "type": "string" @@ -10686,12 +10680,14 @@ "type": "object" }, "publicEndpoint": { + "description": "The public endpoint the provider last reported. It lags\n`publicIngressEnabled` while the lifecycle reconciler converges.", "type": [ "string", "null" ] }, "publicIngressEnabled": { + "description": "Lightspeed-owned public ingress intent; `publicEndpoint` is what the\nprovider realized.", "type": "boolean" }, "requestId": { @@ -15777,29 +15773,6 @@ ], "type": "object" }, - "SessionJobArtifactView": { - "properties": { - "kind": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "type": "object" - }, "SessionJobCancelEntryView": { "properties": { "error": { @@ -15862,6 +15835,49 @@ ], "type": "string" }, + "SessionJobExitView": { + "description": "How a job's root process ended.", + "oneOf": [ + { + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "type": { + "const": "code", + "type": "string" + } + }, + "required": [ + "type", + "code" + ], + "type": "object" + }, + { + "description": "Killed by this signal, named like `SIGSEGV`.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "signal": { + "type": "string" + }, + "type": { + "const": "signal", + "type": "string" + } + }, + "required": [ + "type", + "signal" + ], + "type": "object" + } + ] + }, "SessionJobHandleInput": { "properties": { "environmentId": { @@ -15897,7 +15913,8 @@ "dataBase64": { "type": "string" }, - "seq": { + "offset": { + "description": "Byte offset of the chunk's first byte in the job's combined output.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15907,12 +15924,50 @@ } }, "required": [ - "seq", + "offset", "stream", "dataBase64" ], "type": "object" }, + "SessionJobOutputGapReasonView": { + "oneOf": [ + { + "const": "overflow", + "description": "Dropped from the middle because the job's retained output exceeded\nits cap; the head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Released after it was delivered.", + "type": "string" + } + ] + }, + "SessionJobOutputGapView": { + "description": "Output bytes in `[offset, offset + omittedBytes)` that cannot be\nreturned.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/definitions/SessionJobOutputGapReasonView" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, "SessionJobOutputStreamView": { "enum": [ "stdout", @@ -15921,13 +15976,8 @@ "type": "string" }, "SessionJobReadEntryView": { + "description": "One job's summary and output page, or why it could not be read: an\nentry with `error` and no summary names a job the environment does not\nknow (never created, or garbage-collected).", "properties": { - "artifacts": { - "items": { - "$ref": "#/definitions/SessionJobArtifactView" - }, - "type": "array" - }, "error": { "type": [ "string", @@ -15945,12 +15995,27 @@ ] }, "outputChunks": { + "description": "Output in offset order.", "items": { "$ref": "#/definitions/SessionJobOutputChunkView" }, "type": "array" }, - "outputNextSeq": { + "outputEnd": { + "description": "Output bytes the job produced so far; final once it is terminal.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "outputGaps": { + "description": "Output ranges between the requested start and `outputNext` that\ncannot be returned.", + "items": { + "$ref": "#/definitions/SessionJobOutputGapView" + }, + "type": "array" + }, + "outputNext": { + "description": "Byte offset to continue reading from: pass it as `after`.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15967,7 +16032,8 @@ } }, "required": [ - "outputNextSeq" + "outputNext", + "outputEnd" ], "type": "object" }, @@ -16087,26 +16153,84 @@ "type": "object" }, "SessionJobStatusView": { - "enum": [ - "accepted", - "queued", - "running", - "succeeded", - "failed", - "cancelRequested", - "cancelled", - "timedOut", - "dependencyFailed", - "interrupted", - "lost" - ], - "type": "string" + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its timeout expired and it was stopped.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of a cancel.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The environment daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "SessionJobStoppedByView": { + "description": "Why Lightspeed stopped a running job.", + "oneOf": [ + { + "const": "timeout", + "description": "Its timeout expired: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "terminate", + "description": "A cancel without `force`: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "kill", + "description": "A cancel with `force`: killed at once.", + "type": "string" + } + ] }, "SessionJobSummaryView": { + "description": "What a job is and how it ended. Instants are the environment's wall\nclock.", "properties": { + "cancelRequestedAtMs": { + "description": "When a cancel reached the job while it ran.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "createdAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": "integer" }, "dependencies": { @@ -16115,13 +16239,26 @@ }, "type": "array" }, - "exitCode": { - "format": "int32", + "durationMs": { + "description": "Run time from start to finish, immune to clock jumps.", + "format": "uint64", + "minimum": 0, "type": [ "integer", "null" ] }, + "exit": { + "anyOf": [ + { + "$ref": "#/definitions/SessionJobExitView" + }, + { + "type": "null" + } + ], + "description": "How the job's root process ended, once it ran and exited." + }, "failure": { "type": [ "string", @@ -16129,8 +16266,7 @@ ] }, "finishedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16145,9 +16281,6 @@ "null" ] }, - "namespace": { - "type": "string" - }, "orphanedDescendants": { "description": "True when the job's root process exited while descendants it spawned\nwere still running; the host terminated them best-effort at job\ncompletion.", "type": "boolean" @@ -16159,16 +16292,18 @@ ] }, "queuedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" ] }, + "requestId": { + "description": "The start request that created the job; part of its identity.", + "type": "string" + }, "startedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16176,11 +16311,22 @@ }, "status": { "$ref": "#/definitions/SessionJobStatusView" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/definitions/SessionJobStoppedByView" + }, + { + "type": "null" + } + ], + "description": "Set when Lightspeed stopped the running job." } }, "required": [ - "namespace", "jobId", + "requestId", "status", "createdAtMs" ], diff --git a/clients/typescript/src/generated/methods.ts b/clients/typescript/src/generated/methods.ts index e9e78e345..fe535acfe 100644 --- a/clients/typescript/src/generated/methods.ts +++ b/clients/typescript/src/generated/methods.ts @@ -368,7 +368,7 @@ export const METHOD_INFO = { scope: "universe", access: {"action":"configure_resource","kind":"universe"}, summary: "Configure environment public ingress", - description: "Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health.", + description: "Records the public ingress intent of a provisioned environment; the lifecycle reconciler converges it while the environment is ready, and `publicEndpoint` shows the realized endpoint. Rejected when the provider does not offer ingress. The provider owns hostname, guest port, routing, TLS, and health; an intent its template refuses is dropped.", }, "environments/power/put": { scope: "universe", @@ -416,7 +416,7 @@ export const METHOD_INFO = { scope: "universe", access: {"action":"read","kind":"universe"}, summary: "Read environment jobs", - description: "Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling.", + description: "Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status.", }, "environments/jobs/cancel": { scope: "universe", @@ -944,19 +944,19 @@ export const METHOD_INFO = { scope: "deployment", access: {"kind":"deployment"}, summary: "Put an environment provider", - description: "Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed.", + description: "Registers or replaces one deployment provider: its controller endpoint URL and the bearer credential Lightspeed presents to the controller and data routes. The credential is write-only: required for a new provider, kept when omitted, never returned. The provider does not call this API or require access to Lightspeed.", }, "deployment/environment-providers/list": { scope: "deployment", access: {"kind":"deployment"}, summary: "List environment providers", - description: "Returns every deployment-registered deployment provider and its controller connection.", + description: "Returns every deployment-registered provider with its controller endpoint and the optional features its controller last advertised.", }, "deployment/environment-providers/read": { scope: "deployment", access: {"kind":"deployment"}, summary: "Read an environment provider", - description: "Returns one deployment-registered deployment provider and its controller connection.", + description: "Returns one deployment-registered provider with its controller endpoint and the optional features its controller last advertised.", }, "deployment/environment-providers/delete": { scope: "deployment", @@ -1330,7 +1330,7 @@ export interface MethodMap { /** * Configure environment public ingress * - * Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health. + * Records the public ingress intent of a provisioned environment; the lifecycle reconciler converges it while the environment is ready, and `publicEndpoint` shows the realized endpoint. Rejected when the provider does not offer ingress. The provider owns hostname, guest port, routing, TLS, and health; an intent its template refuses is dropped. */ "environments/ingress/put": { params: Api.EnvironmentIngressPutParams; @@ -1402,7 +1402,7 @@ export interface MethodMap { /** * Read environment jobs * - * Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling. + * Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status. */ "environments/jobs/read": { params: Api.EnvironmentJobReadParams; @@ -2194,7 +2194,7 @@ export interface MethodMap { /** * Put an environment provider * - * Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed. + * Registers or replaces one deployment provider: its controller endpoint URL and the bearer credential Lightspeed presents to the controller and data routes. The credential is write-only: required for a new provider, kept when omitted, never returned. The provider does not call this API or require access to Lightspeed. */ "deployment/environment-providers/put": { params: Api.DeploymentEnvironmentProviderPutParams; @@ -2203,7 +2203,7 @@ export interface MethodMap { /** * List environment providers * - * Returns every deployment-registered deployment provider and its controller connection. + * Returns every deployment-registered provider with its controller endpoint and the optional features its controller last advertised. */ "deployment/environment-providers/list": { params: Api.DeploymentEnvironmentProviderListParams; @@ -2212,7 +2212,7 @@ export interface MethodMap { /** * Read an environment provider * - * Returns one deployment-registered deployment provider and its controller connection. + * Returns one deployment-registered provider with its controller endpoint and the optional features its controller last advertised. */ "deployment/environment-providers/read": { params: Api.DeploymentEnvironmentProviderReadParams; @@ -2564,7 +2564,7 @@ export const rpc = { /** * Configure environment public ingress * - * Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health. + * Records the public ingress intent of a provisioned environment; the lifecycle reconciler converges it while the environment is ready, and `publicEndpoint` shows the realized endpoint. Rejected when the provider does not offer ingress. The provider owns hostname, guest port, routing, TLS, and health; an intent its template refuses is dropped. */ environmentsIngressPut(client: RpcCaller, params: Api.EnvironmentIngressPutParams): Promise { return client.call("environments/ingress/put", params); @@ -2628,7 +2628,7 @@ export const rpc = { /** * Read environment jobs * - * Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling. + * Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status. */ environmentsJobsRead(client: RpcCaller, params: Api.EnvironmentJobReadParams): Promise { return client.call("environments/jobs/read", params); @@ -3332,7 +3332,7 @@ export const rpc = { /** * Put an environment provider * - * Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed. + * Registers or replaces one deployment provider: its controller endpoint URL and the bearer credential Lightspeed presents to the controller and data routes. The credential is write-only: required for a new provider, kept when omitted, never returned. The provider does not call this API or require access to Lightspeed. */ deploymentEnvironmentProvidersPut(client: RpcCaller, params: Api.DeploymentEnvironmentProviderPutParams): Promise { return client.call("deployment/environment-providers/put", params); @@ -3340,7 +3340,7 @@ export const rpc = { /** * List environment providers * - * Returns every deployment-registered deployment provider and its controller connection. + * Returns every deployment-registered provider with its controller endpoint and the optional features its controller last advertised. */ deploymentEnvironmentProvidersList(client: RpcCaller, params: Api.DeploymentEnvironmentProviderListParams): Promise { return client.call("deployment/environment-providers/list", params); @@ -3348,7 +3348,7 @@ export const rpc = { /** * Read an environment provider * - * Returns one deployment-registered deployment provider and its controller connection. + * Returns one deployment-registered provider with its controller endpoint and the optional features its controller last advertised. */ deploymentEnvironmentProvidersRead(client: RpcCaller, params: Api.DeploymentEnvironmentProviderReadParams): Promise { return client.call("deployment/environment-providers/read", params); diff --git a/clients/typescript/src/generated/types.ts b/clients/typescript/src/generated/types.ts index 1092bbcb8..297865264 100644 --- a/clients/typescript/src/generated/types.ts +++ b/clients/typescript/src/generated/types.ts @@ -1385,7 +1385,10 @@ export type EnvironmentSourceView = type: "provisioned"; } | { - connection: EnvironmentConnectionView; + /** + * `ws://`, `wss://` or `unix://` URL Lightspeed dials envd at. + */ + endpoint: string; type: "external"; } | { @@ -1394,17 +1397,6 @@ export type EnvironmentSourceView = registrationKeyId: string; type: "registered"; }; -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentConnectionTransportView". - */ -export type EnvironmentConnectionTransportView = - | ("webSocket" | "http" | "stdio" | "ssh") - | { - provider: { - provider_type: string; - }; - }; /** * What Lightspeed does with a registered environment while its daemon is * disconnected. Registration-key policy, copied onto each environment. @@ -1422,27 +1414,6 @@ export type EnvironmentLifecycleStatusView = | "paused" | "suspended" | "offline"; -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "DeploymentEnvironmentProviderTransport". - */ -export type DeploymentEnvironmentProviderTransport = - | { - type: "webSocket"; - } - | { - type: "http"; - } - | { - type: "stdio"; - } - | { - type: "ssh"; - } - | { - providerType: string; - type: "provider"; - }; /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentProviderBindingStatusView". @@ -1466,26 +1437,53 @@ export type EnvironmentCredentialSourceView = type: "directSecret"; }; /** + * How a job's root process ended. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "SessionJobExitView". + */ +export type SessionJobExitView = + | { + code: number; + type: "code"; + } + | { + coreDumped?: boolean; + signal: string; + type: "signal"; + }; +/** + * Job lifecycle. `queued` and `running` are live; the rest are terminal. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobStatusView". */ export type SessionJobStatusView = - | "accepted" - | "queued" | "running" + | "queued" | "succeeded" | "failed" - | "cancelRequested" - | "cancelled" | "timedOut" + | "cancelled" | "dependencyFailed" - | "interrupted" - | "lost"; + | "interrupted"; +/** + * Why Lightspeed stopped a running job. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "SessionJobStoppedByView". + */ +export type SessionJobStoppedByView = "timeout" | "terminate" | "kill"; /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobOutputStreamView". */ export type SessionJobOutputStreamView = "stdout" | "stderr"; +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "SessionJobOutputGapReasonView". + */ +export type SessionJobOutputGapReasonView = "overflow" | "expired"; /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentRegistrationKeyStatusView". @@ -4736,10 +4734,24 @@ export interface EnvironmentView { * control connection. */ lastSeenAtMs?: number | null; + /** + * Why the environment failed, or the latest error (or what the + * provider reports it is waiting on) while its lifecycle converges; + * absent once it reaches a steady state. + */ + lifecycleError?: EnvironmentLifecycleErrorView | null; metadata?: { [k: string]: string; }; + /** + * The public endpoint the provider last reported. It lags + * `publicIngressEnabled` while the lifecycle reconciler converges. + */ publicEndpoint?: string | null; + /** + * Lightspeed-owned public ingress intent; `publicEndpoint` is what the + * provider realized. + */ publicIngressEnabled: boolean; requestId: string; source: EnvironmentSourceView; @@ -4779,11 +4791,18 @@ export interface EnvironmentIncarnationView { } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentConnectionView". + * via the `definition` "EnvironmentLifecycleErrorView". */ -export interface EnvironmentConnectionView { - endpoint: string; - transport: EnvironmentConnectionTransportView; +export interface EnvironmentLifecycleErrorView { + /** + * When the current run of errors began. + */ + firstAtMs: number; + /** + * When `message` was recorded. + */ + lastAtMs: number; + message: string; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -4805,8 +4824,21 @@ export interface DeploymentEnvironmentProviderDeleteResponse { * via the `definition` "DeploymentEnvironmentProviderView". */ export interface DeploymentEnvironmentProviderView { - controllerConnection: DeploymentEnvironmentProviderConnection; + /** + * `ws://`, `wss://` or `unix://` URL of the provider controller. + */ + controllerEndpoint: string; + /** + * What the controller advertised at its last successful handshake; + * absent until Lightspeed first reached it at this endpoint. + */ + controllerFeatures?: DeploymentEnvironmentProviderFeaturesView | null; createdAtMs: number; + /** + * Base URL of the provider's data routes, as its controller advertised + * it; absent until then. + */ + dataRouteBase?: string | null; displayName?: string | null; metadata?: { [k: string]: string; @@ -4815,12 +4847,20 @@ export interface DeploymentEnvironmentProviderView { updatedAtMs: number; } /** + * Optional controller methods the provider's controller advertised. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "DeploymentEnvironmentProviderConnection". + * via the `definition` "DeploymentEnvironmentProviderFeaturesView". */ -export interface DeploymentEnvironmentProviderConnection { - endpoint: string; - transport: DeploymentEnvironmentProviderTransport; +export interface DeploymentEnvironmentProviderFeaturesView { + /** + * Existing targets can be adopted (`environment/adopt`). + */ + adoptTarget: boolean; + /** + * Public ingress can be enabled on environments. + */ + ingress: boolean; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -5192,18 +5232,31 @@ export interface SessionJobHandleView { jobId: string; } /** + * What a job is and how it ended. Instants are the environment's wall + * clock. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobSummaryView". */ export interface SessionJobSummaryView { + /** + * When a cancel reached the job while it ran. + */ + cancelRequestedAtMs?: number | null; createdAtMs: number; dependencies?: string[]; - exitCode?: number | null; + /** + * Run time from start to finish, immune to clock jumps. + */ + durationMs?: number | null; + /** + * How the job's root process ended, once it ran and exited. + */ + exit?: SessionJobExitView | null; failure?: string | null; finishedAtMs?: number | null; jobId: string; name?: string | null; - namespace: string; /** * True when the job's root process exited while descendants it spawned * were still running; the host terminated them best-effort at job @@ -5212,8 +5265,16 @@ export interface SessionJobSummaryView { orphanedDescendants?: boolean; queueKey?: string | null; queuedAtMs?: number | null; + /** + * The start request that created the job; part of its identity. + */ + requestId: string; startedAtMs?: number | null; status: SessionJobStatusView; + /** + * Set when Lightspeed stopped the running job. + */ + stoppedBy?: SessionJobStoppedByView | null; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -5261,37 +5322,59 @@ export interface EnvironmentJobReadResponse { jobs?: SessionJobReadEntryView[]; } /** + * One job's summary and output page, or why it could not be read: an + * entry with `error` and no summary names a job the environment does not + * know (never created, or garbage-collected). + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobReadEntryView". */ export interface SessionJobReadEntryView { - artifacts?: SessionJobArtifactView[]; error?: string | null; handle?: SessionJobHandleView | null; + /** + * Output in offset order. + */ outputChunks?: SessionJobOutputChunkView[]; - outputNextSeq: number; + /** + * Output bytes the job produced so far; final once it is terminal. + */ + outputEnd: number; + /** + * Output ranges between the requested start and `outputNext` that + * cannot be returned. + */ + outputGaps?: SessionJobOutputGapView[]; + /** + * Byte offset to continue reading from: pass it as `after`. + */ + outputNext: number; summary?: SessionJobSummaryView | null; } -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "SessionJobArtifactView". - */ -export interface SessionJobArtifactView { - kind?: string | null; - metadata?: { - [k: string]: string; - }; - path: string; -} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobOutputChunkView". */ export interface SessionJobOutputChunkView { dataBase64: string; - seq: number; + /** + * Byte offset of the chunk's first byte in the job's combined output. + */ + offset: number; stream: SessionJobOutputStreamView; } +/** + * Output bytes in `[offset, offset + omittedBytes)` that cannot be + * returned. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "SessionJobOutputGapView". + */ +export interface SessionJobOutputGapView { + offset: number; + omittedBytes: number; + reason: SessionJobOutputGapReasonView; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfEnvironmentListResponse". @@ -5498,8 +5581,25 @@ export interface EnvironmentTemplateView { }; providerId: string; publicIngress: boolean; + /** + * Compute targets of this template get; absent when the provider does + * not fix it. + */ + resources?: EnvironmentTemplateResourcesView | null; templateId: string; } +/** + * A template's compute, in provider-neutral units. Each field is absent + * when the provider does not fix it. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentTemplateResourcesView". + */ +export interface EnvironmentTemplateResourcesView { + diskBytes?: number | null; + memoryBytes?: number | null; + vcpus?: number | null; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfEnvironmentTemplateReadResponse". @@ -5603,8 +5703,9 @@ export interface ServerInfo { export interface EnvironmentDaemonInfo { gitSha: string; /** - * Environment protocol version the gateway speaks. A daemon registers - * only when it reports exactly this number. + * The envd protocol version (data plane and registration channel) + * the gateway speaks. A daemon registers only when it reports exactly + * this number. */ protocolVersion: number; /** @@ -7426,11 +7527,23 @@ export interface DeploymentEnvironmentProviderDeleteParams { */ export interface DeploymentEnvironmentProviderListParams {} /** + * Register or replace a provider. `credential` is the bearer token + * Lightspeed presents to the provider's controller and data routes: it is + * sealed on receipt and never returned by any method. `Debug` output + * redacts it; request logging must never echo these params. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "DeploymentEnvironmentProviderPutParams". */ export interface DeploymentEnvironmentProviderPutParams { - controllerConnection: DeploymentEnvironmentProviderConnection; + /** + * `ws://`, `wss://` or `unix://` URL of the provider controller. + */ + controllerEndpoint: string; + /** + * Required when the provider is new; omit to keep the stored one. + */ + credential?: string | null; displayName?: string | null; metadata?: { [k: string]: string; @@ -7583,11 +7696,12 @@ export interface EnvironmentCredentialUnbindParams { * via the `definition` "EnvironmentExternalCreateParams". */ export interface EnvironmentExternalCreateParams { + displayName?: string | null; /** - * Connection to an envd instance reachable from Lightspeed. + * `ws://`, `wss://` or `unix://` URL of an envd instance reachable from + * Lightspeed. */ - connection: EnvironmentConnectionView; - displayName?: string | null; + endpoint: string; metadata?: { [k: string]: string; }; @@ -7669,10 +7783,20 @@ export interface SessionJobDependencyInput { * via the `definition` "EnvironmentJobReadParams". */ export interface EnvironmentJobReadParams { - afterSeq?: number | null; - includeArtifacts?: boolean; + /** + * Byte offset to read each job's output from: a previous entry's + * `outputNext`. Omitted with `tailBytes` omitted reads from the start. + */ + after?: number | null; jobs: SessionJobHandleInput[]; - outputBytes?: number | null; + /** + * Most output bytes per job; omit for the server's default page. + */ + maxBytes?: number | null; + /** + * Read each job's last `tailBytes` of output instead; excludes `after`. + */ + tailBytes?: number | null; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema diff --git a/crates/api/contract/api-reference.md b/crates/api/contract/api-reference.md index d3a9902cc..efde1c5b2 100644 --- a/crates/api/contract/api-reference.md +++ b/crates/api/contract/api-reference.md @@ -484,7 +484,7 @@ Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, **Configure environment public ingress** -Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health. +Records the public ingress intent of a provisioned environment; the lifecycle reconciler converges it while the environment is ready, and `publicEndpoint` shows the realized endpoint. Rejected when the provider does not offer ingress. The provider owns hostname, guest port, routing, TLS, and health; an intent its template refuses is dropped. - Access: `{"kind":"universe","action":"configure_resource"}` - Group: `environments` @@ -588,7 +588,7 @@ Starts a dependency-aware job group on one environment instance, injecting the e **Read environment jobs** -Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling. +Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status. - Access: `{"kind":"universe","action":"read"}` - Group: `environments` @@ -1747,7 +1747,7 @@ Revokes the key with this display prefix; revoking a revoked key keeps its first **Put an environment provider** -Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed. +Registers or replaces one deployment provider: its controller endpoint URL and the bearer credential Lightspeed presents to the controller and data routes. The credential is write-only: required for a new provider, kept when omitted, never returned. The provider does not call this API or require access to Lightspeed. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` @@ -1760,7 +1760,7 @@ Registers or replaces one deployment provider and its controller connection. The **List environment providers** -Returns every deployment-registered deployment provider and its controller connection. +Returns every deployment-registered provider with its controller endpoint and the optional features its controller last advertised. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` @@ -1773,7 +1773,7 @@ Returns every deployment-registered deployment provider and its controller conne **Read an environment provider** -Returns one deployment-registered deployment provider and its controller connection. +Returns one deployment-registered provider with its controller endpoint and the optional features its controller last advertised. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` diff --git a/crates/api/contract/api.schema.json b/crates/api/contract/api.schema.json index d2bce1b0a..f36246cf9 100644 --- a/crates/api/contract/api.schema.json +++ b/crates/api/contract/api.schema.json @@ -8763,21 +8763,6 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderConnection": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/definitions/DeploymentEnvironmentProviderTransport" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "DeploymentEnvironmentProviderDeleteParams": { "properties": { "providerId": { @@ -8800,6 +8785,24 @@ ], "type": "object" }, + "DeploymentEnvironmentProviderFeaturesView": { + "description": "Optional controller methods the provider's controller advertised.", + "properties": { + "adoptTarget": { + "description": "Existing targets can be adopted (`environment/adopt`).", + "type": "boolean" + }, + "ingress": { + "description": "Public ingress can be enabled on environments.", + "type": "boolean" + } + }, + "required": [ + "adoptTarget", + "ingress" + ], + "type": "object" + }, "DeploymentEnvironmentProviderListParams": { "type": "object" }, @@ -8818,9 +8821,18 @@ "type": "object" }, "DeploymentEnvironmentProviderPutParams": { + "description": "Register or replace a provider. `credential` is the bearer token\nLightspeed presents to the provider's controller and data routes: it is\nsealed on receipt and never returned by any method. `Debug` output\nredacts it; request logging must never echo these params.", "properties": { - "controllerConnection": { - "$ref": "#/definitions/DeploymentEnvironmentProviderConnection" + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" + }, + "credential": { + "description": "Required when the provider is new; omit to keep the stored one.", + "type": [ + "string", + "null" + ] }, "displayName": { "type": [ @@ -8840,7 +8852,7 @@ }, "required": [ "providerId", - "controllerConnection" + "controllerEndpoint" ], "type": "object" }, @@ -8877,83 +8889,34 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderTransport": { - "oneOf": [ - { - "properties": { - "type": { - "const": "webSocket", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "http", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "stdio", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "ssh", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" + "DeploymentEnvironmentProviderView": { + "properties": { + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" }, - { - "properties": { - "providerType": { - "type": "string" + "controllerFeatures": { + "anyOf": [ + { + "$ref": "#/definitions/DeploymentEnvironmentProviderFeaturesView" }, - "type": { - "const": "provider", - "type": "string" + { + "type": "null" } - }, - "required": [ - "type", - "providerType" ], - "type": "object" - } - ] - }, - "DeploymentEnvironmentProviderView": { - "properties": { - "controllerConnection": { - "$ref": "#/definitions/DeploymentEnvironmentProviderConnection" + "description": "What the controller advertised at its last successful handshake;\nabsent until Lightspeed first reached it at this endpoint." }, "createdAtMs": { "format": "int64", "type": "integer" }, + "dataRouteBase": { + "description": "Base URL of the provider's data routes, as its controller advertised\nit; absent until then.", + "type": [ + "string", + "null" + ] + }, "displayName": { "type": [ "string", @@ -8976,7 +8939,7 @@ }, "required": [ "providerId", - "controllerConnection", + "controllerEndpoint", "createdAtMs", "updatedAtMs" ], @@ -9339,54 +9302,6 @@ ], "type": "object" }, - "EnvironmentConnectionTransportView": { - "oneOf": [ - { - "enum": [ - "webSocket", - "http", - "stdio", - "ssh" - ], - "type": "string" - }, - { - "additionalProperties": false, - "properties": { - "provider": { - "properties": { - "provider_type": { - "type": "string" - } - }, - "required": [ - "provider_type" - ], - "type": "object" - } - }, - "required": [ - "provider" - ], - "type": "object" - } - ] - }, - "EnvironmentConnectionView": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/definitions/EnvironmentConnectionTransportView" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "EnvironmentCreateParams": { "properties": { "bindingId": { @@ -9609,7 +9524,7 @@ "type": "string" }, "protocolVersion": { - "description": "Environment protocol version the gateway speaks. A daemon registers\nonly when it reports exactly this number.", + "description": "The envd protocol version (data plane and registration channel)\nthe gateway speaks. A daemon registers only when it reports exactly\nthis number.", "format": "uint32", "minimum": 0, "type": "integer" @@ -9635,20 +9550,16 @@ }, "EnvironmentExternalCreateParams": { "properties": { - "connection": { - "allOf": [ - { - "$ref": "#/definitions/EnvironmentConnectionView" - } - ], - "description": "Connection to an envd instance reachable from Lightspeed." - }, "displayName": { "type": [ "string", "null" ] }, + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of an envd instance reachable from\nLightspeed.", + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" @@ -9661,7 +9572,7 @@ }, "required": [ "requestId", - "connection" + "endpoint" ], "type": "object" }, @@ -9918,7 +9829,8 @@ }, "EnvironmentJobReadParams": { "properties": { - "afterSeq": { + "after": { + "description": "Byte offset to read each job's output from: a previous entry's\n`outputNext`. Omitted with `tailBytes` omitted reads from the start.", "format": "uint64", "minimum": 0, "type": [ @@ -9926,18 +9838,24 @@ "null" ] }, - "includeArtifacts": { - "default": false, - "type": "boolean" - }, "jobs": { "items": { "$ref": "#/definitions/SessionJobHandleInput" }, "type": "array" }, - "outputBytes": { - "format": "uint", + "maxBytes": { + "description": "Most output bytes per job; omit for the server's default page.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "tailBytes": { + "description": "Read each job's last `tailBytes` of output instead; excludes `after`.", + "format": "uint64", "minimum": 0, "type": [ "integer", @@ -9962,6 +9880,29 @@ }, "type": "object" }, + "EnvironmentLifecycleErrorView": { + "properties": { + "firstAtMs": { + "description": "When the current run of errors began.", + "format": "int64", + "type": "integer" + }, + "lastAtMs": { + "description": "When `message` was recorded.", + "format": "int64", + "type": "integer" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message", + "firstAtMs", + "lastAtMs" + ], + "type": "object" + }, "EnvironmentLifecycleStatusView": { "oneOf": [ { @@ -10500,8 +10441,9 @@ }, { "properties": { - "connection": { - "$ref": "#/definitions/EnvironmentConnectionView" + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL Lightspeed dials envd at.", + "type": "string" }, "type": { "const": "external", @@ -10510,7 +10452,7 @@ }, "required": [ "type", - "connection" + "endpoint" ], "type": "object" }, @@ -10592,6 +10534,36 @@ ], "type": "object" }, + "EnvironmentTemplateResourcesView": { + "description": "A template's compute, in provider-neutral units. Each field is absent\nwhen the provider does not fix it.", + "properties": { + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, "EnvironmentTemplateView": { "properties": { "bindingId": { @@ -10621,6 +10593,17 @@ "publicIngress": { "type": "boolean" }, + "resources": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentTemplateResourcesView" + }, + { + "type": "null" + } + ], + "description": "Compute targets of this template get; absent when the provider does\nnot fix it." + }, "templateId": { "type": "string" } @@ -10679,6 +10662,17 @@ "null" ] }, + "lifecycleError": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentLifecycleErrorView" + }, + { + "type": "null" + } + ], + "description": "Why the environment failed, or the latest error (or what the\nprovider reports it is waiting on) while its lifecycle converges;\nabsent once it reaches a steady state." + }, "metadata": { "additionalProperties": { "type": "string" @@ -10686,12 +10680,14 @@ "type": "object" }, "publicEndpoint": { + "description": "The public endpoint the provider last reported. It lags\n`publicIngressEnabled` while the lifecycle reconciler converges.", "type": [ "string", "null" ] }, "publicIngressEnabled": { + "description": "Lightspeed-owned public ingress intent; `publicEndpoint` is what the\nprovider realized.", "type": "boolean" }, "requestId": { @@ -15777,29 +15773,6 @@ ], "type": "object" }, - "SessionJobArtifactView": { - "properties": { - "kind": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "type": "object" - }, "SessionJobCancelEntryView": { "properties": { "error": { @@ -15862,6 +15835,49 @@ ], "type": "string" }, + "SessionJobExitView": { + "description": "How a job's root process ended.", + "oneOf": [ + { + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "type": { + "const": "code", + "type": "string" + } + }, + "required": [ + "type", + "code" + ], + "type": "object" + }, + { + "description": "Killed by this signal, named like `SIGSEGV`.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "signal": { + "type": "string" + }, + "type": { + "const": "signal", + "type": "string" + } + }, + "required": [ + "type", + "signal" + ], + "type": "object" + } + ] + }, "SessionJobHandleInput": { "properties": { "environmentId": { @@ -15897,7 +15913,8 @@ "dataBase64": { "type": "string" }, - "seq": { + "offset": { + "description": "Byte offset of the chunk's first byte in the job's combined output.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15907,12 +15924,50 @@ } }, "required": [ - "seq", + "offset", "stream", "dataBase64" ], "type": "object" }, + "SessionJobOutputGapReasonView": { + "oneOf": [ + { + "const": "overflow", + "description": "Dropped from the middle because the job's retained output exceeded\nits cap; the head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Released after it was delivered.", + "type": "string" + } + ] + }, + "SessionJobOutputGapView": { + "description": "Output bytes in `[offset, offset + omittedBytes)` that cannot be\nreturned.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/definitions/SessionJobOutputGapReasonView" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, "SessionJobOutputStreamView": { "enum": [ "stdout", @@ -15921,13 +15976,8 @@ "type": "string" }, "SessionJobReadEntryView": { + "description": "One job's summary and output page, or why it could not be read: an\nentry with `error` and no summary names a job the environment does not\nknow (never created, or garbage-collected).", "properties": { - "artifacts": { - "items": { - "$ref": "#/definitions/SessionJobArtifactView" - }, - "type": "array" - }, "error": { "type": [ "string", @@ -15945,12 +15995,27 @@ ] }, "outputChunks": { + "description": "Output in offset order.", "items": { "$ref": "#/definitions/SessionJobOutputChunkView" }, "type": "array" }, - "outputNextSeq": { + "outputEnd": { + "description": "Output bytes the job produced so far; final once it is terminal.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "outputGaps": { + "description": "Output ranges between the requested start and `outputNext` that\ncannot be returned.", + "items": { + "$ref": "#/definitions/SessionJobOutputGapView" + }, + "type": "array" + }, + "outputNext": { + "description": "Byte offset to continue reading from: pass it as `after`.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15967,7 +16032,8 @@ } }, "required": [ - "outputNextSeq" + "outputNext", + "outputEnd" ], "type": "object" }, @@ -16087,26 +16153,84 @@ "type": "object" }, "SessionJobStatusView": { - "enum": [ - "accepted", - "queued", - "running", - "succeeded", - "failed", - "cancelRequested", - "cancelled", - "timedOut", - "dependencyFailed", - "interrupted", - "lost" - ], - "type": "string" + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its timeout expired and it was stopped.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of a cancel.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The environment daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "SessionJobStoppedByView": { + "description": "Why Lightspeed stopped a running job.", + "oneOf": [ + { + "const": "timeout", + "description": "Its timeout expired: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "terminate", + "description": "A cancel without `force`: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "kill", + "description": "A cancel with `force`: killed at once.", + "type": "string" + } + ] }, "SessionJobSummaryView": { + "description": "What a job is and how it ended. Instants are the environment's wall\nclock.", "properties": { + "cancelRequestedAtMs": { + "description": "When a cancel reached the job while it ran.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "createdAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": "integer" }, "dependencies": { @@ -16115,13 +16239,26 @@ }, "type": "array" }, - "exitCode": { - "format": "int32", + "durationMs": { + "description": "Run time from start to finish, immune to clock jumps.", + "format": "uint64", + "minimum": 0, "type": [ "integer", "null" ] }, + "exit": { + "anyOf": [ + { + "$ref": "#/definitions/SessionJobExitView" + }, + { + "type": "null" + } + ], + "description": "How the job's root process ended, once it ran and exited." + }, "failure": { "type": [ "string", @@ -16129,8 +16266,7 @@ ] }, "finishedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16145,9 +16281,6 @@ "null" ] }, - "namespace": { - "type": "string" - }, "orphanedDescendants": { "description": "True when the job's root process exited while descendants it spawned\nwere still running; the host terminated them best-effort at job\ncompletion.", "type": "boolean" @@ -16159,16 +16292,18 @@ ] }, "queuedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" ] }, + "requestId": { + "description": "The start request that created the job; part of its identity.", + "type": "string" + }, "startedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16176,11 +16311,22 @@ }, "status": { "$ref": "#/definitions/SessionJobStatusView" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/definitions/SessionJobStoppedByView" + }, + { + "type": "null" + } + ], + "description": "Set when Lightspeed stopped the running job." } }, "required": [ - "namespace", "jobId", + "requestId", "status", "createdAtMs" ], diff --git a/crates/api/contract/methods.json b/crates/api/contract/methods.json index ad1d9c68d..201b46ab2 100644 --- a/crates/api/contract/methods.json +++ b/crates/api/contract/methods.json @@ -904,7 +904,7 @@ "action": "configure_resource", "kind": "universe" }, - "description": "Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health.", + "description": "Records the public ingress intent of a provisioned environment; the lifecycle reconciler converges it while the environment is ready, and `publicEndpoint` shows the realized endpoint. Rejected when the provider does not offer ingress. The provider owns hostname, guest port, routing, TLS, and health; an intent its template refuses is dropped.", "group": "environments", "method": "environments/ingress/put", "params": { @@ -1104,7 +1104,7 @@ "action": "read", "kind": "universe" }, - "description": "Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling.", + "description": "Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status.", "group": "environments", "method": "environments/jobs/read", "params": { @@ -3289,7 +3289,7 @@ "access": { "kind": "deployment" }, - "description": "Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed.", + "description": "Registers or replaces one deployment provider: its controller endpoint URL and the bearer credential Lightspeed presents to the controller and data routes. The credential is write-only: required for a new provider, kept when omitted, never returned. The provider does not call this API or require access to Lightspeed.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/put", "params": { @@ -3313,7 +3313,7 @@ "access": { "kind": "deployment" }, - "description": "Returns every deployment-registered deployment provider and its controller connection.", + "description": "Returns every deployment-registered provider with its controller endpoint and the optional features its controller last advertised.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/list", "params": { @@ -3337,7 +3337,7 @@ "access": { "kind": "deployment" }, - "description": "Returns one deployment-registered deployment provider and its controller connection.", + "description": "Returns one deployment-registered provider with its controller endpoint and the optional features its controller last advertised.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/read", "params": { diff --git a/crates/api/contract/openrpc.json b/crates/api/contract/openrpc.json index 36c83ce67..59903a515 100644 --- a/crates/api/contract/openrpc.json +++ b/crates/api/contract/openrpc.json @@ -8763,21 +8763,6 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderConnection": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/components/schemas/DeploymentEnvironmentProviderTransport" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "DeploymentEnvironmentProviderDeleteParams": { "properties": { "providerId": { @@ -8800,6 +8785,24 @@ ], "type": "object" }, + "DeploymentEnvironmentProviderFeaturesView": { + "description": "Optional controller methods the provider's controller advertised.", + "properties": { + "adoptTarget": { + "description": "Existing targets can be adopted (`environment/adopt`).", + "type": "boolean" + }, + "ingress": { + "description": "Public ingress can be enabled on environments.", + "type": "boolean" + } + }, + "required": [ + "adoptTarget", + "ingress" + ], + "type": "object" + }, "DeploymentEnvironmentProviderListParams": { "type": "object" }, @@ -8818,9 +8821,18 @@ "type": "object" }, "DeploymentEnvironmentProviderPutParams": { + "description": "Register or replace a provider. `credential` is the bearer token\nLightspeed presents to the provider's controller and data routes: it is\nsealed on receipt and never returned by any method. `Debug` output\nredacts it; request logging must never echo these params.", "properties": { - "controllerConnection": { - "$ref": "#/components/schemas/DeploymentEnvironmentProviderConnection" + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" + }, + "credential": { + "description": "Required when the provider is new; omit to keep the stored one.", + "type": [ + "string", + "null" + ] }, "displayName": { "type": [ @@ -8840,7 +8852,7 @@ }, "required": [ "providerId", - "controllerConnection" + "controllerEndpoint" ], "type": "object" }, @@ -8877,83 +8889,34 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderTransport": { - "oneOf": [ - { - "properties": { - "type": { - "const": "webSocket", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "http", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "stdio", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "ssh", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" + "DeploymentEnvironmentProviderView": { + "properties": { + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" }, - { - "properties": { - "providerType": { - "type": "string" + "controllerFeatures": { + "anyOf": [ + { + "$ref": "#/components/schemas/DeploymentEnvironmentProviderFeaturesView" }, - "type": { - "const": "provider", - "type": "string" + { + "type": "null" } - }, - "required": [ - "type", - "providerType" ], - "type": "object" - } - ] - }, - "DeploymentEnvironmentProviderView": { - "properties": { - "controllerConnection": { - "$ref": "#/components/schemas/DeploymentEnvironmentProviderConnection" + "description": "What the controller advertised at its last successful handshake;\nabsent until Lightspeed first reached it at this endpoint." }, "createdAtMs": { "format": "int64", "type": "integer" }, + "dataRouteBase": { + "description": "Base URL of the provider's data routes, as its controller advertised\nit; absent until then.", + "type": [ + "string", + "null" + ] + }, "displayName": { "type": [ "string", @@ -8976,7 +8939,7 @@ }, "required": [ "providerId", - "controllerConnection", + "controllerEndpoint", "createdAtMs", "updatedAtMs" ], @@ -9339,54 +9302,6 @@ ], "type": "object" }, - "EnvironmentConnectionTransportView": { - "oneOf": [ - { - "enum": [ - "webSocket", - "http", - "stdio", - "ssh" - ], - "type": "string" - }, - { - "additionalProperties": false, - "properties": { - "provider": { - "properties": { - "provider_type": { - "type": "string" - } - }, - "required": [ - "provider_type" - ], - "type": "object" - } - }, - "required": [ - "provider" - ], - "type": "object" - } - ] - }, - "EnvironmentConnectionView": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/components/schemas/EnvironmentConnectionTransportView" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "EnvironmentCreateParams": { "properties": { "bindingId": { @@ -9609,7 +9524,7 @@ "type": "string" }, "protocolVersion": { - "description": "Environment protocol version the gateway speaks. A daemon registers\nonly when it reports exactly this number.", + "description": "The envd protocol version (data plane and registration channel)\nthe gateway speaks. A daemon registers only when it reports exactly\nthis number.", "format": "uint32", "minimum": 0, "type": "integer" @@ -9635,20 +9550,16 @@ }, "EnvironmentExternalCreateParams": { "properties": { - "connection": { - "allOf": [ - { - "$ref": "#/components/schemas/EnvironmentConnectionView" - } - ], - "description": "Connection to an envd instance reachable from Lightspeed." - }, "displayName": { "type": [ "string", "null" ] }, + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of an envd instance reachable from\nLightspeed.", + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" @@ -9661,7 +9572,7 @@ }, "required": [ "requestId", - "connection" + "endpoint" ], "type": "object" }, @@ -9918,7 +9829,8 @@ }, "EnvironmentJobReadParams": { "properties": { - "afterSeq": { + "after": { + "description": "Byte offset to read each job's output from: a previous entry's\n`outputNext`. Omitted with `tailBytes` omitted reads from the start.", "format": "uint64", "minimum": 0, "type": [ @@ -9926,18 +9838,24 @@ "null" ] }, - "includeArtifacts": { - "default": false, - "type": "boolean" - }, "jobs": { "items": { "$ref": "#/components/schemas/SessionJobHandleInput" }, "type": "array" }, - "outputBytes": { - "format": "uint", + "maxBytes": { + "description": "Most output bytes per job; omit for the server's default page.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "tailBytes": { + "description": "Read each job's last `tailBytes` of output instead; excludes `after`.", + "format": "uint64", "minimum": 0, "type": [ "integer", @@ -9962,6 +9880,29 @@ }, "type": "object" }, + "EnvironmentLifecycleErrorView": { + "properties": { + "firstAtMs": { + "description": "When the current run of errors began.", + "format": "int64", + "type": "integer" + }, + "lastAtMs": { + "description": "When `message` was recorded.", + "format": "int64", + "type": "integer" + }, + "message": { + "type": "string" + } + }, + "required": [ + "message", + "firstAtMs", + "lastAtMs" + ], + "type": "object" + }, "EnvironmentLifecycleStatusView": { "oneOf": [ { @@ -10500,8 +10441,9 @@ }, { "properties": { - "connection": { - "$ref": "#/components/schemas/EnvironmentConnectionView" + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL Lightspeed dials envd at.", + "type": "string" }, "type": { "const": "external", @@ -10510,7 +10452,7 @@ }, "required": [ "type", - "connection" + "endpoint" ], "type": "object" }, @@ -10592,6 +10534,36 @@ ], "type": "object" }, + "EnvironmentTemplateResourcesView": { + "description": "A template's compute, in provider-neutral units. Each field is absent\nwhen the provider does not fix it.", + "properties": { + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, "EnvironmentTemplateView": { "properties": { "bindingId": { @@ -10621,6 +10593,17 @@ "publicIngress": { "type": "boolean" }, + "resources": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentTemplateResourcesView" + }, + { + "type": "null" + } + ], + "description": "Compute targets of this template get; absent when the provider does\nnot fix it." + }, "templateId": { "type": "string" } @@ -10679,6 +10662,17 @@ "null" ] }, + "lifecycleError": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentLifecycleErrorView" + }, + { + "type": "null" + } + ], + "description": "Why the environment failed, or the latest error (or what the\nprovider reports it is waiting on) while its lifecycle converges;\nabsent once it reaches a steady state." + }, "metadata": { "additionalProperties": { "type": "string" @@ -10686,12 +10680,14 @@ "type": "object" }, "publicEndpoint": { + "description": "The public endpoint the provider last reported. It lags\n`publicIngressEnabled` while the lifecycle reconciler converges.", "type": [ "string", "null" ] }, "publicIngressEnabled": { + "description": "Lightspeed-owned public ingress intent; `publicEndpoint` is what the\nprovider realized.", "type": "boolean" }, "requestId": { @@ -15777,29 +15773,6 @@ ], "type": "object" }, - "SessionJobArtifactView": { - "properties": { - "kind": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "type": "object" - }, "SessionJobCancelEntryView": { "properties": { "error": { @@ -15862,6 +15835,49 @@ ], "type": "string" }, + "SessionJobExitView": { + "description": "How a job's root process ended.", + "oneOf": [ + { + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "type": { + "const": "code", + "type": "string" + } + }, + "required": [ + "type", + "code" + ], + "type": "object" + }, + { + "description": "Killed by this signal, named like `SIGSEGV`.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "signal": { + "type": "string" + }, + "type": { + "const": "signal", + "type": "string" + } + }, + "required": [ + "type", + "signal" + ], + "type": "object" + } + ] + }, "SessionJobHandleInput": { "properties": { "environmentId": { @@ -15897,7 +15913,8 @@ "dataBase64": { "type": "string" }, - "seq": { + "offset": { + "description": "Byte offset of the chunk's first byte in the job's combined output.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15907,12 +15924,50 @@ } }, "required": [ - "seq", + "offset", "stream", "dataBase64" ], "type": "object" }, + "SessionJobOutputGapReasonView": { + "oneOf": [ + { + "const": "overflow", + "description": "Dropped from the middle because the job's retained output exceeded\nits cap; the head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Released after it was delivered.", + "type": "string" + } + ] + }, + "SessionJobOutputGapView": { + "description": "Output bytes in `[offset, offset + omittedBytes)` that cannot be\nreturned.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/components/schemas/SessionJobOutputGapReasonView" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, "SessionJobOutputStreamView": { "enum": [ "stdout", @@ -15921,13 +15976,8 @@ "type": "string" }, "SessionJobReadEntryView": { + "description": "One job's summary and output page, or why it could not be read: an\nentry with `error` and no summary names a job the environment does not\nknow (never created, or garbage-collected).", "properties": { - "artifacts": { - "items": { - "$ref": "#/components/schemas/SessionJobArtifactView" - }, - "type": "array" - }, "error": { "type": [ "string", @@ -15945,12 +15995,27 @@ ] }, "outputChunks": { + "description": "Output in offset order.", "items": { "$ref": "#/components/schemas/SessionJobOutputChunkView" }, "type": "array" }, - "outputNextSeq": { + "outputEnd": { + "description": "Output bytes the job produced so far; final once it is terminal.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "outputGaps": { + "description": "Output ranges between the requested start and `outputNext` that\ncannot be returned.", + "items": { + "$ref": "#/components/schemas/SessionJobOutputGapView" + }, + "type": "array" + }, + "outputNext": { + "description": "Byte offset to continue reading from: pass it as `after`.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15967,7 +16032,8 @@ } }, "required": [ - "outputNextSeq" + "outputNext", + "outputEnd" ], "type": "object" }, @@ -16087,26 +16153,84 @@ "type": "object" }, "SessionJobStatusView": { - "enum": [ - "accepted", - "queued", - "running", - "succeeded", - "failed", - "cancelRequested", - "cancelled", - "timedOut", - "dependencyFailed", - "interrupted", - "lost" - ], - "type": "string" + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its timeout expired and it was stopped.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of a cancel.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The environment daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "SessionJobStoppedByView": { + "description": "Why Lightspeed stopped a running job.", + "oneOf": [ + { + "const": "timeout", + "description": "Its timeout expired: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "terminate", + "description": "A cancel without `force`: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "kill", + "description": "A cancel with `force`: killed at once.", + "type": "string" + } + ] }, "SessionJobSummaryView": { + "description": "What a job is and how it ended. Instants are the environment's wall\nclock.", "properties": { + "cancelRequestedAtMs": { + "description": "When a cancel reached the job while it ran.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "createdAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": "integer" }, "dependencies": { @@ -16115,13 +16239,26 @@ }, "type": "array" }, - "exitCode": { - "format": "int32", + "durationMs": { + "description": "Run time from start to finish, immune to clock jumps.", + "format": "uint64", + "minimum": 0, "type": [ "integer", "null" ] }, + "exit": { + "anyOf": [ + { + "$ref": "#/components/schemas/SessionJobExitView" + }, + { + "type": "null" + } + ], + "description": "How the job's root process ended, once it ran and exited." + }, "failure": { "type": [ "string", @@ -16129,8 +16266,7 @@ ] }, "finishedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16145,9 +16281,6 @@ "null" ] }, - "namespace": { - "type": "string" - }, "orphanedDescendants": { "description": "True when the job's root process exited while descendants it spawned\nwere still running; the host terminated them best-effort at job\ncompletion.", "type": "boolean" @@ -16159,16 +16292,18 @@ ] }, "queuedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" ] }, + "requestId": { + "description": "The start request that created the job; part of its identity.", + "type": "string" + }, "startedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16176,11 +16311,22 @@ }, "status": { "$ref": "#/components/schemas/SessionJobStatusView" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/components/schemas/SessionJobStoppedByView" + }, + { + "type": "null" + } + ], + "description": "Set when Lightspeed stopped the running job." } }, "required": [ - "namespace", "jobId", + "requestId", "status", "createdAtMs" ], @@ -19905,7 +20051,7 @@ "x-lightspeed-target": null }, { - "description": "Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health.", + "description": "Records the public ingress intent of a provisioned environment; the lifecycle reconciler converges it while the environment is ready, and `publicEndpoint` shows the realized endpoint. Rejected when the provider does not offer ingress. The provider owns hostname, guest port, routing, TLS, and health; an intent its template refuses is dropped.", "name": "environments/ingress/put", "paramStructure": "by-name", "params": [ @@ -20129,7 +20275,7 @@ "x-lightspeed-target": null }, { - "description": "Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling.", + "description": "Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status.", "name": "environments/jobs/read", "paramStructure": "by-name", "params": [ @@ -22579,7 +22725,7 @@ "x-lightspeed-target": null }, { - "description": "Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed.", + "description": "Registers or replaces one deployment provider: its controller endpoint URL and the bearer credential Lightspeed presents to the controller and data routes. The credential is write-only: required for a new provider, kept when omitted, never returned. The provider does not call this API or require access to Lightspeed.", "name": "deployment/environment-providers/put", "paramStructure": "by-name", "params": [ @@ -22606,7 +22752,7 @@ "x-lightspeed-target": null }, { - "description": "Returns every deployment-registered deployment provider and its controller connection.", + "description": "Returns every deployment-registered provider with its controller endpoint and the optional features its controller last advertised.", "name": "deployment/environment-providers/list", "paramStructure": "by-name", "params": [ @@ -22633,7 +22779,7 @@ "x-lightspeed-target": null }, { - "description": "Returns one deployment-registered deployment provider and its controller connection.", + "description": "Returns one deployment-registered provider with its controller endpoint and the optional features its controller last advertised.", "name": "deployment/environment-providers/read", "paramStructure": "by-name", "params": [ diff --git a/crates/api/src/deployment.rs b/crates/api/src/deployment.rs index 393efbce6..d6aecc5c0 100644 --- a/crates/api/src/deployment.rs +++ b/crates/api/src/deployment.rs @@ -272,24 +272,14 @@ pub struct DeploymentApiKeyRevokeResponse { pub api_key: DeploymentApiKeyView, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum DeploymentEnvironmentProviderTransport { - WebSocket, - Http, - Stdio, - Ssh, - Provider { - #[serde(rename = "providerType")] - provider_type: String, - }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +/// Optional controller methods the provider's controller advertised. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct DeploymentEnvironmentProviderConnection { - pub endpoint: String, - pub transport: DeploymentEnvironmentProviderTransport, +pub struct DeploymentEnvironmentProviderFeaturesView { + /// Existing targets can be adopted (`environment/adopt`). + pub adopt_target: bool, + /// Public ingress can be enabled on environments. + pub ingress: bool, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -298,22 +288,54 @@ pub struct DeploymentEnvironmentProviderView { pub provider_id: EnvironmentProviderId, #[serde(default, skip_serializing_if = "Option::is_none")] pub display_name: Option, - pub controller_connection: DeploymentEnvironmentProviderConnection, + /// `ws://`, `wss://` or `unix://` URL of the provider controller. + pub controller_endpoint: String, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, + /// What the controller advertised at its last successful handshake; + /// absent until Lightspeed first reached it at this endpoint. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub controller_features: Option, + /// Base URL of the provider's data routes, as its controller advertised + /// it; absent until then. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_route_base: Option, pub created_at_ms: i64, pub updated_at_ms: i64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +/// Register or replace a provider. `credential` is the bearer token +/// Lightspeed presents to the provider's controller and data routes: it is +/// sealed on receipt and never returned by any method. `Debug` output +/// redacts it; request logging must never echo these params. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DeploymentEnvironmentProviderPutParams { pub provider_id: EnvironmentProviderId, #[serde(default, skip_serializing_if = "Option::is_none")] pub display_name: Option, - pub controller_connection: DeploymentEnvironmentProviderConnection, + /// `ws://`, `wss://` or `unix://` URL of the provider controller. + pub controller_endpoint: String, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, + /// Required when the provider is new; omit to keep the stored one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credential: Option, +} + +impl fmt::Debug for DeploymentEnvironmentProviderPutParams { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DeploymentEnvironmentProviderPutParams") + .field("provider_id", &self.provider_id) + .field("display_name", &self.display_name) + .field("controller_endpoint", &self.controller_endpoint) + .field("metadata", &self.metadata) + .field( + "credential", + &self.credential.as_ref().map(|_| ""), + ) + .finish() + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -663,11 +685,11 @@ deployment_api_methods! { METHOD_DEPLOYMENT_API_KEYS_REVOKE => revoke_api_key(DeploymentApiKeyRevokeParams) -> DeploymentApiKeyRevokeResponse => ["Revoke a scoped API key", "Revokes the key with this display prefix; revoking a revoked key keeps its first revocation time. An unknown prefix is not found."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_PUT => put_environment_provider(DeploymentEnvironmentProviderPutParams) -> DeploymentEnvironmentProviderPutResponse => - ["Put an environment provider", "Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed."], access: MethodAccess::Deployment, + ["Put an environment provider", "Registers or replaces one deployment provider: its controller endpoint URL and the bearer credential Lightspeed presents to the controller and data routes. The credential is write-only: required for a new provider, kept when omitted, never returned. The provider does not call this API or require access to Lightspeed."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_LIST => list_environment_providers(DeploymentEnvironmentProviderListParams) -> DeploymentEnvironmentProviderListResponse => - ["List environment providers", "Returns every deployment-registered deployment provider and its controller connection."], access: MethodAccess::Deployment, + ["List environment providers", "Returns every deployment-registered provider with its controller endpoint and the optional features its controller last advertised."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_READ => read_environment_provider(DeploymentEnvironmentProviderReadParams) -> DeploymentEnvironmentProviderReadResponse => - ["Read an environment provider", "Returns one deployment-registered deployment provider and its controller connection."], access: MethodAccess::Deployment, + ["Read an environment provider", "Returns one deployment-registered provider with its controller endpoint and the optional features its controller last advertised."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_DELETE => delete_environment_provider(DeploymentEnvironmentProviderDeleteParams) -> DeploymentEnvironmentProviderDeleteResponse => ["Delete an environment provider", "Deletes a deployment provider only when no universe binding references it."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_PROVIDER_BINDINGS_PUT => put_environment_provider_binding(DeploymentProviderBindingPutParams) -> DeploymentProviderBindingPutResponse => diff --git a/crates/api/src/environments.rs b/crates/api/src/environments.rs index d53a85c5d..6273cf3c5 100644 --- a/crates/api/src/environments.rs +++ b/crates/api/src/environments.rs @@ -173,8 +173,9 @@ pub struct EnvironmentIdlePolicyPutResponse { #[serde(rename_all = "camelCase")] pub struct EnvironmentExternalCreateParams { pub request_id: EnvironmentProvisionRequestId, - /// Connection to an envd instance reachable from Lightspeed. - pub connection: EnvironmentConnectionView, + /// `ws://`, `wss://` or `unix://` URL of an envd instance reachable from + /// Lightspeed. + pub endpoint: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub display_name: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] @@ -218,7 +219,8 @@ pub enum EnvironmentSourceView { binding_id: EnvironmentProviderBindingId, }, External { - connection: EnvironmentConnectionView, + /// `ws://`, `wss://` or `unix://` URL Lightspeed dials envd at. + endpoint: String, }, /// An envd that dialed the gateway outbound and was admitted by a /// registration key. The key is the environment's group; the daemon id @@ -354,23 +356,6 @@ pub struct EnvironmentRegistrationKeyRevokeResponse { pub closed_environment_ids: Vec, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentConnectionView { - pub endpoint: String, - pub transport: EnvironmentConnectionTransportView, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub enum EnvironmentConnectionTransportView { - WebSocket, - Http, - Stdio, - Ssh, - Provider { provider_type: String }, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentIncarnationView { @@ -403,7 +388,11 @@ pub struct EnvironmentView { #[serde(default, skip_serializing_if = "Option::is_none")] pub idle_policy: Option, pub incarnation: EnvironmentIncarnationView, + /// Lightspeed-owned public ingress intent; `publicEndpoint` is what the + /// provider realized. pub public_ingress_enabled: bool, + /// The public endpoint the provider last reported. It lags + /// `publicIngressEnabled` while the lifecycle reconciler converges. #[serde(default, skip_serializing_if = "Option::is_none")] pub public_endpoint: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] @@ -412,10 +401,25 @@ pub struct EnvironmentView { /// control connection. #[serde(default, skip_serializing_if = "Option::is_none")] pub last_seen_at_ms: Option, + /// Why the environment failed, or the latest error (or what the + /// provider reports it is waiting on) while its lifecycle converges; + /// absent once it reaches a steady state. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lifecycle_error: Option, pub created_at_ms: i64, pub updated_at_ms: i64, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentLifecycleErrorView { + pub message: String, + /// When the current run of errors began. + pub first_at_ms: i64, + /// When `message` was recorded. + pub last_at_ms: i64, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum EnvironmentProviderBindingStatusView { @@ -469,10 +473,27 @@ pub struct EnvironmentTemplateView { pub description: Option, pub public_ingress: bool, pub deprecated: bool, + /// Compute targets of this template get; absent when the provider does + /// not fix it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resources: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } +/// A template's compute, in provider-neutral units. Each field is absent +/// when the provider does not fix it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentTemplateResourcesView { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vcpus: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub disk_bytes: Option, +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentTemplateListParams { @@ -632,6 +653,9 @@ pub struct SessionJobStartedView { pub queue_key: Option, } +/// One job's summary and output page, or why it could not be read: an +/// entry with `error` and no summary names a job the environment does not +/// know (never created, or garbage-collected). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SessionJobReadEntryView { @@ -639,11 +663,17 @@ pub struct SessionJobReadEntryView { pub handle: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub summary: Option, + /// Output in offset order. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub output_chunks: Vec, - pub output_next_seq: u64, + /// Byte offset to continue reading from: pass it as `after`. + pub output_next: u64, + /// Output bytes the job produced so far; final once it is terminal. + pub output_end: u64, + /// Output ranges between the requested start and `outputNext` that + /// cannot be returned. #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub artifacts: Vec, + pub output_gaps: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, } @@ -667,25 +697,38 @@ pub enum SessionJobCancelScopeView { Dependents, } +/// What a job is and how it ended. Instants are the environment's wall +/// clock. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SessionJobSummaryView { - pub namespace: String, pub job_id: String, + /// The start request that created the job; part of its identity. + pub request_id: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, pub status: SessionJobStatusView, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub dependencies: Vec, - pub created_at_ms: u64, + pub created_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queued_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at_ms: Option, + /// When a cancel reached the job while it ran. #[serde(default, skip_serializing_if = "Option::is_none")] - pub queued_at_ms: Option, + pub cancel_requested_at_ms: Option, + /// Run time from start to finish, immune to clock jumps. #[serde(default, skip_serializing_if = "Option::is_none")] - pub started_at_ms: Option, + pub duration_ms: Option, + /// How the job's root process ended, once it ran and exited. #[serde(default, skip_serializing_if = "Option::is_none")] - pub finished_at_ms: Option, + pub exit: Option, + /// Set when Lightspeed stopped the running job. #[serde(default, skip_serializing_if = "Option::is_none")] - pub exit_code: Option, + pub stopped_by: Option, /// True when the job's root process exited while descendants it spawned /// were still running; the host terminated them best-effort at job /// completion. @@ -697,45 +740,91 @@ pub struct SessionJobSummaryView { pub queue_key: Option, } +/// Job lifecycle. `queued` and `running` are live; the rest are terminal. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum SessionJobStatusView { - Accepted, + /// Waiting for dependencies or its queue. Queued, Running, + /// Exited with code 0. Succeeded, + /// Exited otherwise, or could not start. Failed, - CancelRequested, - Cancelled, + /// Its timeout expired and it was stopped. TimedOut, + /// Stopped or never started because of a cancel. + Cancelled, + /// A dependency did not finish as the dependency policy requires. DependencyFailed, + /// The environment daemon stopped while the job was queued or running. Interrupted, - Lost, +} + +/// How a job's root process ended. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde( + tag = "type", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum SessionJobExitView { + /// Normal exit with this status code. + Code { code: i32 }, + /// Killed by this signal, named like `SIGSEGV`. + Signal { + signal: String, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + core_dumped: bool, + }, +} + +/// Why Lightspeed stopped a running job. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum SessionJobStoppedByView { + /// Its timeout expired: terminate, a grace period, then kill. + Timeout, + /// A cancel without `force`: terminate, a grace period, then kill. + Terminate, + /// A cancel with `force`: killed at once. + Kill, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SessionJobOutputChunkView { - pub seq: u64, + /// Byte offset of the chunk's first byte in the job's combined output. + pub offset: u64, pub stream: SessionJobOutputStreamView, pub data_base64: String, } +/// Output bytes in `[offset, offset + omittedBytes)` that cannot be +/// returned. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub enum SessionJobOutputStreamView { - Stdout, - Stderr, +pub struct SessionJobOutputGapView { + pub offset: u64, + pub omitted_bytes: u64, + pub reason: SessionJobOutputGapReasonView, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct SessionJobArtifactView { - pub path: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub kind: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, +pub enum SessionJobOutputGapReasonView { + /// Dropped from the middle because the job's retained output exceeded + /// its cap; the head and the newest bytes are kept. + Overflow, + /// Released after it was delivered. + Expired, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum SessionJobOutputStreamView { + Stdout, + Stderr, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -759,12 +848,16 @@ pub struct EnvironmentJobCreateResponse { #[serde(rename_all = "camelCase")] pub struct EnvironmentJobReadParams { pub jobs: Vec, + /// Most output bytes per job; omit for the server's default page. #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_bytes: Option, + pub max_bytes: Option, + /// Byte offset to read each job's output from: a previous entry's + /// `outputNext`. Omitted with `tailBytes` omitted reads from the start. #[serde(default, skip_serializing_if = "Option::is_none")] - pub after_seq: Option, - #[serde(default)] - pub include_artifacts: bool, + pub after: Option, + /// Read each job's last `tailBytes` of output instead; excludes `after`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tail_bytes: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/crates/api/src/handshake.rs b/crates/api/src/handshake.rs index 33a9c7c2a..c20746ba4 100644 --- a/crates/api/src/handshake.rs +++ b/crates/api/src/handshake.rs @@ -96,8 +96,9 @@ pub struct ServerInfo { pub struct EnvironmentDaemonInfo { pub version: String, pub git_sha: String, - /// Environment protocol version the gateway speaks. A daemon registers - /// only when it reports exactly this number. + /// The envd protocol version (data plane and registration channel) + /// the gateway speaks. A daemon registers only when it reports exactly + /// this number. pub protocol_version: u32, /// Rust target triples the daemon is published for. pub targets: Vec, diff --git a/crates/api/src/rpc.rs b/crates/api/src/rpc.rs index 08c2d47e5..e4320e92f 100644 --- a/crates/api/src/rpc.rs +++ b/crates/api/src/rpc.rs @@ -405,7 +405,7 @@ api_methods! { METHOD_ENVIRONMENTS_EXTERNAL_CREATE => create_external_environment(EnvironmentExternalCreateParams) -> EnvironmentExternalCreateResponse => ["Register an external environment", "Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, attributed to the caller. Reachability is checked on demand."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_INGRESS_PUT => put_environment_ingress(EnvironmentIngressPutParams) -> EnvironmentIngressPutResponse => - ["Configure environment public ingress", "Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + ["Configure environment public ingress", "Records the public ingress intent of a provisioned environment; the lifecycle reconciler converges it while the environment is ready, and `publicEndpoint` shows the realized endpoint. Rejected when the provider does not offer ingress. The provider owns hostname, guest port, routing, TLS, and health; an intent its template refuses is dropped."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_POWER_PUT => put_environment_power(EnvironmentPowerPutParams) -> EnvironmentPowerPutResponse => ["Set environment power intent", "Records the desired power state (running, paused, suspended, or stopped) of a provisioned environment; the lifecycle reconciler converges the provider target asynchronously. Powered-down environments wake transparently on their next use. Rejected when the provider does not support the state."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_IDLE_POLICY_PUT => put_environment_idle_policy(EnvironmentIdlePolicyPutParams) -> EnvironmentIdlePolicyPutResponse => @@ -421,7 +421,7 @@ api_methods! { METHOD_ENVIRONMENTS_JOBS_CREATE => create_environment_jobs(EnvironmentJobCreateParams) -> EnvironmentJobCreateResponse => ["Create environment jobs", "Starts a dependency-aware job group on one environment instance, injecting the environment's configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance rather than a session. A powered-down environment is woken on use: the call fails with environment_not_ready while the wake is in progress; retry with backoff."], access: MethodAccess::Universe(UniverseAction::UseResource), METHOD_ENVIRONMENTS_JOBS_READ => read_environment_jobs(EnvironmentJobReadParams) -> EnvironmentJobReadResponse => - ["Read environment jobs", "Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling."], access: MethodAccess::Universe(UniverseAction::Read), + ["Read environment jobs", "Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status."], access: MethodAccess::Universe(UniverseAction::Read), METHOD_ENVIRONMENTS_JOBS_CANCEL => cancel_environment_jobs(EnvironmentJobCancelParams) -> EnvironmentJobCancelResponse => ["Cancel environment jobs", "Requests cancellation for selected jobs, optionally including dependents. Force is provider-specific escalation; inspect each per-job result."], access: MethodAccess::Universe(UniverseAction::UseResource), METHOD_ENVIRONMENTS_REGISTRATION_KEYS_CREATE => create_environment_registration_key(EnvironmentRegistrationKeyCreateParams) -> EnvironmentRegistrationKeyCreateResponse => diff --git a/crates/api/src/tests.rs b/crates/api/src/tests.rs index 7aad6f5e0..f74b8695d 100644 --- a/crates/api/src/tests.rs +++ b/crates/api/src/tests.rs @@ -171,6 +171,22 @@ fn auth_provider_create_params_redact_credential_in_debug_output() { ); } +#[test] +fn environment_provider_put_params_redact_credential_in_debug_output() { + let params: DeploymentEnvironmentProviderPutParams = serde_json::from_value(json!({ + "providerId": "incus-local", + "controllerEndpoint": "wss://incus.example.com/control", + "credential": "provider-bearer-secret" + })) + .expect("deserialize environment provider put params"); + + let debug = format!("{params:?}"); + + assert!(!debug.contains("provider-bearer-secret"), "{debug}"); + assert!(debug.contains("")); + assert_eq!(params.credential.as_deref(), Some("provider-bearer-secret")); +} + #[test] fn request_ids_accept_number_or_string() { let numeric: JsonRpcRequest = serde_json::from_value(json!({ @@ -3262,6 +3278,7 @@ fn test_environment_instance() -> EnvironmentView { public_endpoint: None, metadata: BTreeMap::new(), last_seen_at_ms: None, + lifecycle_error: None, created_at_ms: 10, updated_at_ms: 10, } @@ -3303,10 +3320,7 @@ fn test_external_environment() -> EnvironmentView { environment_id: "evi_enrolled".to_owned(), request_id: "request-enrolled".to_owned(), source: EnvironmentSourceView::External { - connection: EnvironmentConnectionView { - endpoint: "ws://envd.test:19091".to_owned(), - transport: EnvironmentConnectionTransportView::WebSocket, - }, + endpoint: "ws://envd.test:19091".to_owned(), }, display_name: Some("External".to_owned()), status: EnvironmentLifecycleStatusView::Ready, @@ -3325,6 +3339,7 @@ fn test_external_environment() -> EnvironmentView { public_endpoint: None, metadata: BTreeMap::new(), last_seen_at_ms: None, + lifecycle_error: None, created_at_ms: 10, updated_at_ms: 10, } @@ -3403,11 +3418,13 @@ fn test_deployment_environment_provider(provider_id: &str) -> DeploymentEnvironm DeploymentEnvironmentProviderView { provider_id: provider_id.to_owned(), display_name: Some("Local Incus".to_owned()), - controller_connection: DeploymentEnvironmentProviderConnection { - endpoint: "ws://127.0.0.1:19090/control".to_owned(), - transport: DeploymentEnvironmentProviderTransport::WebSocket, - }, + controller_endpoint: "ws://127.0.0.1:19090/control".to_owned(), metadata: BTreeMap::new(), + controller_features: Some(DeploymentEnvironmentProviderFeaturesView { + adopt_target: true, + ingress: true, + }), + data_route_base: Some("ws://127.0.0.1:19090/data".to_owned()), created_at_ms: 10, updated_at_ms: 20, } @@ -3572,10 +3589,7 @@ async fn deployment_environment_provider_methods_dispatch() { json!({ "providerId": "incus-local", "displayName": "Local Incus", - "controllerConnection": { - "endpoint": "ws://127.0.0.1:19090/control", - "transport": { "type": "webSocket" } - } + "controllerEndpoint": "ws://127.0.0.1:19090/control" }), ), (METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_LIST, json!({})), diff --git a/crates/cli/src/env_cli.rs b/crates/cli/src/env_cli.rs index e27cbf9fd..600dd1f4e 100644 --- a/crates/cli/src/env_cli.rs +++ b/crates/cli/src/env_cli.rs @@ -556,7 +556,7 @@ struct CreateArgs { struct RegisterArgs { #[command(flatten)] common: ResourceArgs, - /// A ws:// or wss:// endpoint reachable from the runtime. + /// A ws://, wss:// or unix:// endpoint of envd reachable from the runtime. endpoint: String, /// Supply the same request ID when retrying registration. #[arg(long)] @@ -639,20 +639,18 @@ async fn create(args: CreateArgs) -> Result<()> { crate::output::show(args.common.json, &response.environment) } async fn register(args: RegisterArgs) -> Result<()> { - let endpoint = reqwest::Url::parse(&args.endpoint)?; anyhow::ensure!( - matches!(endpoint.scheme(), "ws" | "wss"), - "envd endpoint must use ws:// or wss://" + ["ws://", "wss://", "unix://"] + .iter() + .any(|scheme| args.endpoint.starts_with(scheme)), + "envd endpoint must use ws://, wss:// or unix://" ); let response: api::EnvironmentExternalCreateResponse = HttpAgentApi::new(args.common.api_url) .request( api::METHOD_ENVIRONMENTS_EXTERNAL_CREATE, api::EnvironmentExternalCreateParams { request_id: args.request_id, - connection: api::EnvironmentConnectionView { - endpoint: args.endpoint, - transport: api::EnvironmentConnectionTransportView::WebSocket, - }, + endpoint: args.endpoint, display_name: args.display_name, metadata: args.metadata.map(), }, diff --git a/crates/cli/tests/resource_commands.rs b/crates/cli/tests/resource_commands.rs index 182c2f60c..8fed2de50 100644 --- a/crates/cli/tests/resource_commands.rs +++ b/crates/cli/tests/resource_commands.rs @@ -556,11 +556,9 @@ fn provisioning_and_template_commands_send_current_typed_requests() { assert_eq!(params["metadata"], json!({"project":"audit"})); json!({"type":"provisioned","providerId":"incus","bindingId":"local"}) } else { - assert_eq!( - params["connection"], - json!({"endpoint":"wss://env.example/ws","transport":"webSocket"}) - ); - json!({"type":"external","connection":params["connection"]}) + assert_eq!(params["endpoint"], "wss://env.example/ws"); + assert!(params.get("connection").is_none(), "{params}"); + json!({"type":"external","endpoint":params["endpoint"]}) }; assert_eq!(params["requestId"], "retry-1"); json!({"environment":{"environmentId":"env1","requestId":params["requestId"],"source":source,"status":"provisioning","desiredPower":"running","incarnation":{"incarnationId":"i1","createdAtMs":0,"updatedAtMs":0},"publicIngressEnabled":false,"createdAtMs":0,"updatedAtMs":0}}) diff --git a/crates/environment-client/Cargo.toml b/crates/environment-client/Cargo.toml index b868be79b..a1e1ddd70 100644 --- a/crates/environment-client/Cargo.toml +++ b/crates/environment-client/Cargo.toml @@ -3,15 +3,20 @@ name = "environment-client" version = "0.1.0" edition = "2024" +[features] +# `conformance::controller`: a suite that drives any controller endpoint +# through `ControllerClient` and checks the controller-plane contract. +conformance = [] + [dependencies] -async-trait = "0.1" -futures-util = "0.3" +futures-util = { version = "0.3", features = ["sink"] } environment-protocol = { path = "../environment-protocol" } serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" -tokio = { version = "1", features = ["net"] } -tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] } +tokio = { version = "1", features = ["macros", "net", "rt", "sync", "time"] } +tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] } [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt", "time"] } +environment-client = { path = ".", features = ["conformance"] } +tokio = { version = "1", features = ["io-util", "macros", "rt", "time"] } diff --git a/crates/environment-client/src/conformance/controller.rs b/crates/environment-client/src/conformance/controller.rs new file mode 100644 index 000000000..ac61c60bb --- /dev/null +++ b/crates/environment-client/src/conformance/controller.rs @@ -0,0 +1,631 @@ +//! Controller-plane conformance: drives any controller endpoint through +//! [`ControllerClient`] and checks the contract every provider shares. +//! +//! The suite checks that: +//! +//! - `initialize` names the `controller` plane, refuses a wrong +//! `protocolVersion` with `versionMismatch` (leaving the connection open +//! and uninitialized), refuses requests before it with `notInitialized`, +//! and refuses a repeat with `invalidRequest`; +//! - a missing or wrong bearer credential is refused with HTTP 401 before +//! the upgrade; +//! - `routes.data` passes the downgrade rule for the controller URL; +//! - `target/get` never creates and answers `notFound` for an unknown +//! owner or id; a foreign owner on a known `targetId` is +//! `failedPrecondition` (`ownerMismatch`); +//! - `target/create` answers promptly and is idempotent per owner; +//! - `target/close` by owner without a `targetId` closes, is idempotent, +//! and answers `closed` without `targetId` for an owner with nothing; +//! - optional methods answer `capabilityUnavailable` when not advertised. +//! +//! The suite creates one target from the configured template and closes it +//! again; every other check uses fresh owners that hold nothing. +//! +//! ```no_run +//! # async fn example(options: environment_client::ConnectOptions, +//! # template_id: environment_protocol::ids::TemplateId, +//! # owner: std::sync::Arc environment_protocol::control::target::TargetOwner + Send + Sync>) { +//! use environment_client::conformance::controller::{ControllerConformance, run}; +//! +//! let suite = ControllerConformance::new("wss://provider.example/control", options, template_id, owner); +//! run(&suite).await.into_result().expect("controller conforms"); +//! # } +//! ``` + +use std::{ + fmt, + sync::Arc, + time::{Duration, Instant}, +}; + +use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, + bytes::SecretString, + control::{ + handshake::{ControllerInitializeParams, ControllerInitializeResponse}, + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + AdoptTargetParams, CloseTargetParams, CreateTargetParams, GetTargetParams, + TargetObservation, TargetOwner, TargetPhase, TargetRef, + }, + }, + error::{Capability, ErrorData, PreconditionReason, ProtocolErrorKind}, + handshake::Plane, + ids::{ProviderNativeRef, ProviderTargetId, TemplateId}, + manifest::{ControllerInitialize, TargetGet}, +}; +use thiserror::Error; + +use crate::{ + ClientError, ConnectOptions, ControllerClient, connection::Connection, controller::initialize, +}; + +/// What to run the suite against. +#[derive(Clone)] +pub struct ControllerConformance { + /// The controller URL, as registered for the provider. + pub endpoint: String, + /// Connect options carrying the provider credential as `bearer_token`. + pub options: ConnectOptions, + /// A template the suite may create one target from. + pub template_id: TemplateId, + /// Returns a fresh owner on every call, valid for the provider's + /// binding and never used before. + pub owner: Arc TargetOwner + Send + Sync>, + /// Bound on every answer: target methods answer promptly and continue + /// slow work in the background. Default 15 s. + pub prompt_bound: Duration, + /// How long the created target may take to carry a `targetId`, and its + /// close to reach `closed`. Default 10 min. + pub settle_timeout: Duration, + /// Pause between observations while waiting. Default 2 s. + pub poll_interval: Duration, +} + +impl ControllerConformance { + pub fn new( + endpoint: impl Into, + options: ConnectOptions, + template_id: TemplateId, + owner: Arc TargetOwner + Send + Sync>, + ) -> Self { + Self { + endpoint: endpoint.into(), + options, + template_id, + owner, + prompt_bound: Duration::from_secs(15), + settle_timeout: Duration::from_secs(600), + poll_interval: Duration::from_secs(2), + } + } +} + +impl fmt::Debug for ControllerConformance { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ControllerConformance") + .field("endpoint", &self.endpoint) + .field("template_id", &self.template_id) + .field("prompt_bound", &self.prompt_bound) + .field("settle_timeout", &self.settle_timeout) + .finish_non_exhaustive() + } +} + +/// The outcome of one check. +#[derive(Clone, Debug)] +pub struct CheckResult { + /// Stable check name, such as `target.get.foreignOwner`. + pub name: &'static str, + /// Why the check failed. + pub failure: Option, +} + +/// Every check that ran, in order. +#[derive(Clone, Debug, Default)] +pub struct ConformanceReport { + pub checks: Vec, +} + +impl ConformanceReport { + pub fn passed(&self) -> bool { + self.checks.iter().all(|check| check.failure.is_none()) + } + + pub fn failures(&self) -> impl Iterator { + self.checks.iter().filter(|check| check.failure.is_some()) + } + + /// `Ok` when every check passed. + pub fn into_result(self) -> Result { + if self.passed() { + return Ok(self); + } + Err(ConformanceFailure { + failures: self + .failures() + .map(|check| (check.name, check.failure.clone().unwrap_or_default())) + .collect(), + }) + } + + fn record(&mut self, name: &'static str, outcome: Result<(), String>) -> bool { + let passed = outcome.is_ok(); + self.checks.push(CheckResult { + name, + failure: outcome.err(), + }); + passed + } +} + +/// The failed checks of a run. +#[derive(Clone, Debug, Error)] +#[error("controller conformance failed: {}", describe(failures))] +pub struct ConformanceFailure { + /// Check name and why it failed. + pub failures: Vec<(&'static str, String)>, +} + +fn describe(failures: &[(&'static str, String)]) -> String { + failures + .iter() + .map(|(name, failure)| format!("{name}: {failure}")) + .collect::>() + .join("; ") +} + +type Check = Result<(), String>; + +/// Run every check against `suite.endpoint`. Checks that need the +/// handshake are skipped (and reported) when it fails. +pub async fn run(suite: &ControllerConformance) -> ConformanceReport { + let mut report = ConformanceReport::default(); + let connected = ControllerClient::connect_initialized( + &suite.endpoint, + with_timeout(suite.options.clone(), suite.prompt_bound), + ) + .await; + let (client, initialize) = match connected { + Ok(connected) => connected, + Err(error) => { + report.record( + "initialize", + Err(format!("connect_initialized failed: {error}")), + ); + return report; + } + }; + report.record("initialize", check_initialize(&initialize)); + report.record( + "initialize.repeat", + check_repeat_initialize(suite, &client).await, + ); + report.record("initialize.order", check_initialize_order(suite).await); + report.record("credential", check_credential(suite).await); + report.record( + "routes.data", + initialize + .routes + .data + .check_for_controller(&suite.endpoint) + .map_err(|error| error.to_string()), + ); + report.record( + "target.get.unknown", + check_get_unknown(suite, &client).await, + ); + report.record( + "target.close.nothing", + check_close_nothing(suite, &client).await, + ); + report.record( + "capabilities.optional", + check_optional_methods(suite, &client, &initialize).await, + ); + + let owner = (suite.owner)(); + let created = check_create(suite, &client, &owner).await; + let created_target = created.as_ref().ok().cloned().flatten(); + report.record("target.create", created.map(|_| ())); + if let Some(target_id) = created_target { + report.record( + "target.get.foreignOwner", + check_foreign_owner(suite, &client, &target_id).await, + ); + } + report.record( + "target.close.byOwner", + check_close_by_owner(suite, &client, &owner).await, + ); + let _ = client.close().await; + report +} + +fn with_timeout(mut options: ConnectOptions, bound: Duration) -> ConnectOptions { + options.request_timeout = Some(bound); + options +} + +fn check_initialize(initialize: &ControllerInitializeResponse) -> Check { + if initialize.plane != Plane::Controller { + return Err(format!("plane is {}", initialize.plane)); + } + if initialize.protocol_version != CONTROLLER_PROTOCOL_VERSION { + return Err(format!( + "protocolVersion is {}", + initialize.protocol_version + )); + } + Ok(()) +} + +async fn check_repeat_initialize( + suite: &ControllerConformance, + client: &ControllerClient, +) -> Check { + let repeated = client + .call::(&ControllerInitializeParams { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + client_info: suite.options.client_info.clone(), + }) + .await; + expect_kind(repeated.map(|_| ()), ProtocolErrorKind::InvalidRequest) +} + +/// `notInitialized` before `initialize`; `versionMismatch` for another +/// version, after which the same connection still initializes. +async fn check_initialize_order(suite: &ControllerConformance) -> Check { + let options = with_timeout(suite.options.clone(), suite.prompt_bound); + let connection = Connection::connect(&suite.endpoint, &options, Plane::Controller) + .await + .map_err(|error| format!("connect: {error}"))?; + let outcome = async { + let early = connection + .call::( + &GetTargetParams { + target: TargetRef { + owner: (suite.owner)(), + target_id: None, + }, + }, + None, + ) + .await; + expect_kind(early.map(|_| ()), ProtocolErrorKind::NotInitialized) + .map_err(|error| format!("target/get before initialize: {error}"))?; + + let wrong = CONTROLLER_PROTOCOL_VERSION + 1; + let params = serde_json::to_value(ControllerInitializeParams { + protocol_version: wrong, + client_info: suite.options.client_info.clone(), + }) + .map_err(|error| error.to_string())?; + let response = connection + .request(ControllerInitialize::NAME, params, Some(suite.prompt_bound)) + .await + .map_err(|error| format!("initialize with version {wrong}: {error}"))?; + match response.result { + Err(error) => match error.data { + ErrorData::VersionMismatch { expected, received } + if expected == CONTROLLER_PROTOCOL_VERSION && received == wrong => {} + data => { + return Err(format!( + "initialize with version {wrong} answered {data:?}, expected \ + versionMismatch {{expected: {CONTROLLER_PROTOCOL_VERSION}, received: {wrong}}}" + )); + } + }, + Ok(_) => return Err(format!("initialize with version {wrong} succeeded")), + } + initialize(&connection, &options) + .await + .map(|_| ()) + .map_err(|error| format!("initialize after a mismatch: {error}")) + } + .await; + let _ = connection.close().await; + outcome +} + +async fn check_credential(suite: &ControllerConformance) -> Check { + let mut missing = suite.options.clone(); + missing.bearer_token = None; + let mut wrong = suite.options.clone(); + wrong.bearer_token = Some(SecretString::new("lightspeed-conformance-wrong-credential")); + for (label, options) in [("missing", missing), ("wrong", wrong)] { + match ControllerClient::connect_initialized(&suite.endpoint, options).await { + Err(ClientError::UpgradeRejected { status: 401 }) => {} + Err(error) => { + return Err(format!( + "{label} credential: expected HTTP 401, got {error}" + )); + } + Ok((client, _)) => { + let _ = client.close().await; + return Err(format!("{label} credential was accepted")); + } + } + } + Ok(()) +} + +async fn check_get_unknown(suite: &ControllerConformance, client: &ControllerClient) -> Check { + let owner = (suite.owner)(); + let by_owner = get(client, &owner, None).await; + expect_kind(by_owner.map(|_| ()), ProtocolErrorKind::NotFound) + .map_err(|error| format!("target/get of an owner with nothing: {error}"))?; + let unknown = ProviderTargetId::new("lightspeed-conformance-unknown-target") + .map_err(|error| error.to_string())?; + let by_id = get(client, &owner, Some(unknown)).await; + expect_kind(by_id.map(|_| ()), ProtocolErrorKind::NotFound) + .map_err(|error| format!("target/get of an unknown targetId: {error}"))?; + let again = get(client, &owner, None).await; + expect_kind(again.map(|_| ()), ProtocolErrorKind::NotFound) + .map_err(|error| format!("target/get created a target: {error}")) +} + +async fn check_close_nothing(suite: &ControllerConformance, client: &ControllerClient) -> Check { + let owner = (suite.owner)(); + let observation = close(client, &owner) + .await + .map_err(|error| format!("target/close of an owner with nothing: {error}"))?; + if observation.state.phase != TargetPhase::Closed || observation.target_id.is_some() { + return Err(format!( + "target/close of an owner with nothing answered phase {:?} with targetId {:?}; \ + expected closed without targetId", + observation.state.phase, observation.target_id + )); + } + check_owner_echo(&observation, &owner) +} + +async fn check_optional_methods( + suite: &ControllerConformance, + client: &ControllerClient, + initialize: &ControllerInitializeResponse, +) -> Check { + if !initialize.capabilities.adopt_target { + let source = ProviderNativeRef::new("lightspeed-conformance-source") + .map_err(|error| error.to_string())?; + let adopted = client + .adopt_target(&AdoptTargetParams { + owner: (suite.owner)(), + source_target: source, + }) + .await; + expect_capability(adopted.map(|_| ()), Capability::AdoptTarget) + .map_err(|error| format!("target/adopt without adoptTarget: {error}"))?; + } + if !initialize.capabilities.ingress { + let target = TargetRef { + owner: (suite.owner)(), + target_id: None, + }; + let ensured = client + .ensure_ingress(&EnsureIngressParams { + target: target.clone(), + }) + .await; + expect_capability(ensured.map(|_| ()), Capability::Ingress) + .map_err(|error| format!("ingress/ensure without ingress: {error}"))?; + let removed = client.remove_ingress(&RemoveIngressParams { target }).await; + expect_capability(removed.map(|_| ()), Capability::Ingress) + .map_err(|error| format!("ingress/remove without ingress: {error}"))?; + } + Ok(()) +} + +/// Create twice for one owner, each answered within the prompt bound, then +/// wait for the target's id. Returns that id. +async fn check_create( + suite: &ControllerConformance, + client: &ControllerClient, + owner: &TargetOwner, +) -> Result, String> { + let params = CreateTargetParams { + owner: owner.clone(), + template_id: suite.template_id.clone(), + }; + let first = timed(suite, "target/create", client.create_target(¶ms)).await?; + let second = timed( + suite, + "repeated target/create", + client.create_target(¶ms), + ) + .await?; + check_owner_echo(&first.target, owner)?; + check_owner_echo(&second.target, owner)?; + if let (Some(first_id), Some(second_id)) = (&first.target.target_id, &second.target.target_id) + && first_id != second_id + { + return Err(format!( + "target/create is not idempotent per owner: {first_id} then {second_id}" + )); + } + let mut target_id = second.target.target_id.or(first.target.target_id); + let deadline = Instant::now() + suite.settle_timeout; + while target_id.is_none() { + if Instant::now() >= deadline { + return Err(format!( + "the target carried no targetId within {:?}", + suite.settle_timeout + )); + } + tokio::time::sleep(suite.poll_interval).await; + let observation = get(client, owner, None) + .await + .map_err(|error| format!("target/get of the created target: {error}"))?; + target_id = observation.target_id; + } + let repeated = timed( + suite, + "target/create after the id", + client.create_target(¶ms), + ) + .await?; + if repeated.target.target_id != target_id { + return Err(format!( + "target/create after the id answered targetId {:?}, expected {:?}", + repeated.target.target_id, target_id + )); + } + Ok(target_id) +} + +async fn check_foreign_owner( + suite: &ControllerConformance, + client: &ControllerClient, + target_id: &ProviderTargetId, +) -> Check { + let foreign = (suite.owner)(); + match get(client, &foreign, Some(target_id.clone())).await { + Err(ClientError::Protocol(error)) + if matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::OwnerMismatch + } + ) => + { + Ok(()) + } + Err(error) => Err(format!( + "expected failedPrecondition (ownerMismatch), got {error}" + )), + Ok(observation) => Err(format!( + "a foreign owner observed the target (phase {:?})", + observation.state.phase + )), + } +} + +/// Close by owner without a `targetId` until `closed`, then once more. +async fn check_close_by_owner( + suite: &ControllerConformance, + client: &ControllerClient, + owner: &TargetOwner, +) -> Check { + let deadline = Instant::now() + suite.settle_timeout; + loop { + let observation = timed(suite, "target/close", async { + client + .close_target(&CloseTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: None, + }, + }) + .await + }) + .await? + .target; + check_owner_echo(&observation, owner)?; + match observation.state.phase { + TargetPhase::Closed => break, + TargetPhase::Closing => {} + phase => return Err(format!("target/close answered phase {phase:?}")), + } + if Instant::now() >= deadline { + return Err(format!( + "the target did not reach closed within {:?}", + suite.settle_timeout + )); + } + tokio::time::sleep(suite.poll_interval).await; + } + let again = close(client, owner) + .await + .map_err(|error| format!("repeated target/close: {error}"))?; + if again.state.phase != TargetPhase::Closed { + return Err(format!( + "repeated target/close answered phase {:?}", + again.state.phase + )); + } + Ok(()) +} + +async fn get( + client: &ControllerClient, + owner: &TargetOwner, + target_id: Option, +) -> Result { + client + .get_target(&GetTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id, + }, + }) + .await + .map(|response| response.target) +} + +async fn close( + client: &ControllerClient, + owner: &TargetOwner, +) -> Result { + client + .close_target(&CloseTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: None, + }, + }) + .await + .map(|response| response.target) +} + +/// Await `call` and require it to answer within the prompt bound. +async fn timed( + suite: &ControllerConformance, + label: &str, + call: impl Future>, +) -> Result { + let started = Instant::now(); + let outcome = call.await.map_err(|error| format!("{label}: {error}"))?; + let elapsed = started.elapsed(); + if elapsed > suite.prompt_bound { + return Err(format!( + "{label} took {elapsed:?}, over the {:?} prompt bound", + suite.prompt_bound + )); + } + Ok(outcome) +} + +fn check_owner_echo(observation: &TargetObservation, owner: &TargetOwner) -> Check { + if &observation.owner != owner { + return Err(format!( + "the observation echoes owner {:?}, expected {:?}", + observation.owner, owner + )); + } + Ok(()) +} + +fn expect_kind(outcome: Result<(), ClientError>, kind: ProtocolErrorKind) -> Check { + match outcome { + Err(ClientError::Protocol(error)) if error.kind() == kind => Ok(()), + Err(error) => Err(format!("expected {kind}, got {error}")), + Ok(()) => Err(format!("expected {kind}, the call succeeded")), + } +} + +fn expect_capability(outcome: Result<(), ClientError>, capability: Capability) -> Check { + match outcome { + Err(ClientError::Protocol(error)) + if error.data == (ErrorData::CapabilityUnavailable { capability }) => + { + Ok(()) + } + Err(error) => Err(format!( + "expected capabilityUnavailable ({capability:?}), got {error}" + )), + Ok(()) => Err(format!( + "expected capabilityUnavailable ({capability:?}), the call succeeded" + )), + } +} diff --git a/crates/environment-client/src/conformance/mod.rs b/crates/environment-client/src/conformance/mod.rs new file mode 100644 index 000000000..ec2c07fe7 --- /dev/null +++ b/crates/environment-client/src/conformance/mod.rs @@ -0,0 +1,8 @@ +//! Conformance suites that drive a real endpoint through this crate's +//! clients and check the protocol contract. Enabled by the `conformance` +//! feature. +//! +//! - [`controller`]: any provider controller, through +//! [`ControllerClient`](crate::ControllerClient). + +pub mod controller; diff --git a/crates/environment-client/src/connection.rs b/crates/environment-client/src/connection.rs new file mode 100644 index 000000000..82134b217 --- /dev/null +++ b/crates/environment-client/src/connection.rs @@ -0,0 +1,774 @@ +//! The multiplexed JSON-RPC connection under both clients. +//! +//! One writer task owns the socket's sink and one reader task its stream. +//! A call registers a oneshot under a fresh integer id, queues its frame +//! for the writer and waits; the reader routes each response to its call +//! by id, in whatever order the server answers. A call that is dropped or +//! runs past its deadline removes its entry, so a late response is +//! discarded, and on the data plane an abortable call also sends +//! `$/cancelRequest`. While any call is pending the writer pings every +//! keepalive interval. +//! +//! The server of a data or controller connection only answers. A request +//! or notification from it, a binary frame, a frame over the message +//! limit or an unparseable frame ends the connection: every pending call +//! fails with [`ClientError::InvalidMessage`] and the client closes the +//! socket with close code 1002 (1009 for an oversized frame). + +use std::{ + collections::HashMap, + sync::{ + Arc, Mutex, + atomic::{AtomicI64, Ordering}, + }, + time::Duration, +}; + +use environment_protocol::{ + error::ErrorData, + handshake::Plane, + jsonrpc::{ + self, CancelRequestParams, Id, Message as RpcMessage, Notification, Request, Response, + }, + limits::MAX_MESSAGE_BYTES, + manifest::{self, CancelRequest, RequestMethod}, +}; +use futures_util::{ + SinkExt, StreamExt, + future::BoxFuture, + stream::{SplitSink, SplitStream}, +}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::{mpsc, oneshot}, + time::{Instant, MissedTickBehavior}, +}; +use tokio_tungstenite::{ + WebSocketStream, + tungstenite::{ + self, Message, + protocol::{CloseFrame, frame::coding::CloseCode}, + }, +}; + +use crate::{ + dial::{self, Dialed}, + error::ClientError, + options::ConnectOptions, +}; + +/// Serves one request of an in-process client: the request as the server +/// would receive it, answered with the response it would send. An `Err` +/// fails that one call with the given error, as a test's stand-in for a +/// transport failure; the client stays usable. +pub type InProcessHandler = + Arc BoxFuture<'static, Result> + Send + Sync>; + +/// How long [`Connection::close`] waits for the close frame to be written. +const CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long the reader keeps reading for the server's close reply once +/// the writer has stopped. +const CLOSE_GRACE: Duration = Duration::from_secs(5); + +type Reply = Result; + +/// Calls waiting for a response, and why the connection ended once it +/// has. +#[derive(Default)] +struct Shared { + state: Mutex, +} + +#[derive(Default)] +struct State { + pending: HashMap>, + terminated: Option, +} + +impl Shared { + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn register(&self, id: i64) -> Result, ClientError> { + let mut state = self.state(); + if let Some(error) = &state.terminated { + return Err(error.clone()); + } + let (sender, receiver) = oneshot::channel(); + state.pending.insert(id, sender); + Ok(receiver) + } + + /// Hand a response to its call; a response nobody waits for any more is + /// discarded. + fn resolve(&self, id: i64, response: Response) { + if let Some(sender) = self.state().pending.remove(&id) { + let _ = sender.send(Ok(response)); + } + } + + /// Stop waiting for `id`. True when the call was still unanswered. + fn forget(&self, id: i64) -> bool { + self.state().pending.remove(&id).is_some() + } + + fn has_pending(&self) -> bool { + !self.state().pending.is_empty() + } + + /// End the connection with `error`: every pending call and every later + /// call fails with it. Only the first cause is kept. + fn terminate(&self, error: ClientError) { + let mut state = self.state(); + if state.terminated.is_none() { + state.terminated = Some(error.clone()); + } + for (_, sender) in state.pending.drain() { + let _ = sender.send(Err(error.clone())); + } + } + + fn terminated(&self) -> Option { + self.state().terminated.clone() + } +} + +enum Outgoing { + Text(String), + /// Send `frame` (if any), then stop writing. `ack` learns whether the + /// frame was written. + Close { + frame: Option, + ack: Option>>, + }, +} + +enum Backend { + Socket { + outgoing: mpsc::UnboundedSender, + }, + InProcess { + handler: InProcessHandler, + }, +} + +/// One connection of either plane. +pub(crate) struct Connection { + plane: Plane, + shared: Arc, + backend: Backend, + next_id: AtomicI64, + request_timeout: Option, +} + +impl Connection { + /// Dial `endpoint` (no `initialize` yet). + pub(crate) async fn connect( + endpoint: &str, + options: &ConnectOptions, + plane: Plane, + ) -> Result { + Ok(match dial::dial(endpoint, options).await? { + Dialed::Tcp(socket) => Self::over_socket(*socket, plane, options), + Dialed::Unix(socket) => Self::over_socket(*socket, plane, options), + }) + } + + /// Upgrade an already connected byte stream (no `initialize` yet). + pub(crate) async fn connect_over( + stream: S, + options: &ConnectOptions, + plane: Plane, + ) -> Result + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let socket = dial::upgrade(stream, options).await?; + Ok(Self::over_socket(socket, plane, options)) + } + + fn over_socket(socket: WebSocketStream, plane: Plane, options: &ConnectOptions) -> Self + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let (sink, stream) = socket.split(); + let shared = Arc::new(Shared::default()); + let (outgoing, outgoing_receiver) = mpsc::unbounded_channel(); + let (writer_done, writer_stopped) = oneshot::channel(); + let keepalive = options.keepalive_interval.max(Duration::from_millis(1)); + tokio::spawn(write_loop( + sink, + outgoing_receiver, + shared.clone(), + keepalive, + writer_done, + )); + tokio::spawn(read_loop( + stream, + shared.clone(), + outgoing.downgrade(), + writer_stopped, + )); + Self { + plane, + shared, + backend: Backend::Socket { outgoing }, + next_id: AtomicI64::new(1), + request_timeout: options.request_timeout, + } + } + + pub(crate) fn in_process(plane: Plane, handler: InProcessHandler) -> Self { + Self { + plane, + shared: Arc::new(Shared::default()), + backend: Backend::InProcess { handler }, + next_id: AtomicI64::new(1), + request_timeout: None, + } + } + + pub(crate) fn is_closed(&self) -> bool { + self.shared.terminated().is_some() + } + + /// Close the connection: pending calls fail with `Closed`, a close + /// frame is written (waiting at most a few seconds), and every later + /// call fails. Closing again does nothing. + pub(crate) async fn close(&self) -> Result<(), ClientError> { + self.shared.terminate(ClientError::Closed { + code: Some(u16::from(CloseCode::Normal)), + reason: None, + }); + let Backend::Socket { outgoing } = &self.backend else { + return Ok(()); + }; + let (ack, written) = oneshot::channel(); + let close = Outgoing::Close { + frame: Some(CloseFrame { + code: CloseCode::Normal, + reason: "".into(), + }), + ack: Some(ack), + }; + if outgoing.send(close).is_err() { + return Ok(()); + } + match tokio::time::timeout(CLOSE_TIMEOUT, written).await { + Ok(Ok(Err(message))) => Err(ClientError::Transport(message)), + _ => Ok(()), + } + } + + /// A typed call of this connection's plane, with the connection's + /// default deadline unless `timeout` overrides it. + pub(crate) async fn call( + &self, + params: &M::Params, + timeout: Option, + ) -> Result { + if M::PLANE != self.plane { + return Err(ClientError::Encode(format!( + "{} is a {} method; this client serves the {} plane", + M::NAME, + M::PLANE, + self.plane + ))); + } + let response = self + .request( + M::NAME, + params_value(params)?, + timeout.or(self.request_timeout), + ) + .await?; + decode(response) + } + + /// Run `initialize`: a `versionMismatch` answer, a response naming + /// another plane or another `protocolVersion` fail with the typed + /// handshake errors before the response is decoded strictly. + pub(crate) async fn initialize( + &self, + method: &str, + params: &P, + local_version: u32, + timeout: Duration, + ) -> Result { + let response = self + .request(method, params_value(params)?, Some(timeout)) + .await?; + let id = response.id.clone(); + let value = match response.result { + Ok(value) => value, + Err(error) => { + return Err(match error.data { + ErrorData::VersionMismatch { expected, .. } => ClientError::VersionMismatch { + plane: self.plane, + local: local_version, + remote: expected, + }, + _ => ClientError::Protocol(error), + }); + } + }; + if let Some(actual) = value + .get("plane") + .and_then(|plane| serde_json::from_value::(plane.clone()).ok()) + && actual != self.plane + { + return Err(ClientError::WrongPlane { + expected: self.plane, + actual, + }); + } + if let Some(remote) = value.get("protocolVersion").and_then(Value::as_u64) + && remote != u64::from(local_version) + { + return Err(ClientError::VersionMismatch { + plane: self.plane, + local: local_version, + remote: u32::try_from(remote).unwrap_or(u32::MAX), + }); + } + decode(Response { + id, + result: Ok(value), + }) + } + + /// Send one request and wait for its response, at most `timeout`. + pub(crate) async fn request( + &self, + method: &str, + params: Value, + timeout: Option, + ) -> Reply { + let started = Instant::now(); + if let Some(error) = self.shared.terminated() { + return Err(error); + } + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let request = Request { + id: Id::Number(id), + method: method.to_owned(), + params, + }; + let text = request.to_text(); + check_size(&text)?; + match &self.backend { + Backend::Socket { outgoing } => { + let mut receiver = self.shared.register(id)?; + let _pending = PendingCall { + shared: &self.shared, + outgoing, + id, + cancel: self.plane == Plane::Data + && manifest::find(Plane::Data, method).is_some_and(|spec| spec.abortable), + }; + if outgoing.send(Outgoing::Text(text)).is_err() { + return Err(self.ended()); + } + match timeout { + None => receiver.await.unwrap_or_else(|_| Err(self.ended())), + Some(timeout) => match tokio::time::timeout(timeout, &mut receiver).await { + Ok(reply) => reply.unwrap_or_else(|_| Err(self.ended())), + // A response that raced the deadline still counts. + Err(_) => receiver.try_recv().unwrap_or_else(|_| { + Err(ClientError::DeadlineExceeded { + elapsed_ms: elapsed_ms(started), + }) + }), + }, + } + } + Backend::InProcess { handler } => { + let call = handler(request); + let response = match timeout { + None => call.await?, + Some(timeout) => { + tokio::time::timeout(timeout, call).await.map_err(|_| { + ClientError::DeadlineExceeded { + elapsed_ms: elapsed_ms(started), + } + })?? + } + }; + // The response crosses the same encoding a socket would + // apply, the server's message limit included. + match jsonrpc::parse(&response.to_text()) { + Ok(RpcMessage::Response(response)) if response.id == Some(Id::Number(id)) => { + Ok(response) + } + Ok(_) | Err(_) => Err(ClientError::InvalidMessage(format!( + "the in-process handler did not answer request {id}" + ))), + } + } + } + } + + fn ended(&self) -> ClientError { + self.shared.terminated().unwrap_or(ClientError::Closed { + code: None, + reason: None, + }) + } +} + +/// A registered call. Dropping it (the call finished, ran past its +/// deadline, or its future was dropped) removes the entry; an entry that +/// was still unanswered gets `$/cancelRequest` when the method is +/// abortable. +struct PendingCall<'a> { + shared: &'a Shared, + outgoing: &'a mpsc::UnboundedSender, + id: i64, + cancel: bool, +} + +impl Drop for PendingCall<'_> { + fn drop(&mut self) { + if !self.shared.forget(self.id) || !self.cancel || self.shared.terminated().is_some() { + return; + } + if let Ok(notification) = Notification::new( + CancelRequest::NAME, + &CancelRequestParams { + id: Id::Number(self.id), + }, + ) { + let _ = self.outgoing.send(Outgoing::Text(notification.to_text())); + } + } +} + +fn elapsed_ms(started: Instant) -> u64 { + u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) +} + +fn check_size(text: &str) -> Result<(), ClientError> { + let actual_bytes = text.len() as u64; + if actual_bytes > MAX_MESSAGE_BYTES { + return Err(ClientError::PayloadTooLarge { + limit_bytes: MAX_MESSAGE_BYTES, + actual_bytes, + }); + } + Ok(()) +} + +fn params_value(params: &P) -> Result { + match serde_json::to_value(params) { + Ok(value @ Value::Object(_)) => Ok(value), + Ok(_) => Err(ClientError::Encode( + "params must encode as a JSON object".to_owned(), + )), + Err(error) => Err(ClientError::Encode(error.to_string())), + } +} + +fn decode(response: Response) -> Result { + response.decode().map_err(|error| match error { + jsonrpc::ResponseError::Failed(error) => ClientError::Protocol(error), + jsonrpc::ResponseError::Undecodable(message) => ClientError::Decode(message), + }) +} + +async fn write_loop( + mut sink: SplitSink, Message>, + mut outgoing: mpsc::UnboundedReceiver, + shared: Arc, + keepalive: Duration, + // Dropped when the writer stops, which starts the reader's close grace. + _done: oneshot::Sender<()>, +) where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut ticker = tokio::time::interval_at(Instant::now() + keepalive, keepalive); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + tokio::select! { + command = outgoing.recv() => match command { + Some(Outgoing::Text(text)) => { + if let Err(error) = sink.send(Message::text(text)).await { + shared.terminate(ClientError::Transport(error.to_string())); + return; + } + } + Some(Outgoing::Close { frame, ack }) => { + let written = match frame { + Some(frame) => sink + .send(Message::Close(Some(frame))) + .await + .map_err(|error| error.to_string()), + None => Ok(()), + }; + if let Some(ack) = ack { + let _ = ack.send(written); + } + return; + } + // Every client handle is gone. + None => { + let _ = sink + .send(Message::Close(Some(CloseFrame { + code: CloseCode::Normal, + reason: "".into(), + }))) + .await; + return; + } + }, + _ = ticker.tick() => { + if shared.has_pending() + && let Err(error) = sink.send(Message::Ping(Default::default())).await + { + shared.terminate(ClientError::Transport(error.to_string())); + return; + } + } + } + } +} + +async fn read_loop( + mut stream: SplitStream>, + shared: Arc, + outgoing: mpsc::WeakUnboundedSender, + mut writer_stopped: oneshot::Receiver<()>, +) where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut grace_until: Option = None; + let violation = loop { + let frame = tokio::select! { + frame = stream.next() => frame, + _ = &mut writer_stopped, if grace_until.is_none() => { + grace_until = Some(Instant::now() + CLOSE_GRACE); + continue; + } + _ = tokio::time::sleep_until(grace_until.unwrap_or_else(Instant::now)), + if grace_until.is_some() => break None, + }; + match frame { + None => break None, + Some(Err(tungstenite::Error::ConnectionClosed | tungstenite::Error::AlreadyClosed)) => { + break None; + } + Some(Err(tungstenite::Error::Capacity(error))) => { + break Some(( + CloseCode::Size, + format!("frame over the {MAX_MESSAGE_BYTES}-byte message limit: {error}"), + )); + } + Some(Err(error)) => { + shared.terminate(ClientError::Transport(error.to_string())); + break None; + } + Some(Ok(Message::Text(text))) => match jsonrpc::parse(&text) { + Ok(RpcMessage::Response(response)) => match &response.id { + Some(Id::Number(id)) => shared.resolve(*id, response), + // This client never sends string ids: nothing waits. + Some(Id::String(_)) => {} + None => { + let error = response + .result + .err() + .map(|error| error.to_string()) + .unwrap_or_default(); + break Some(( + CloseCode::Protocol, + format!("the server answered with a null id: {error}"), + )); + } + }, + Ok(RpcMessage::Request(request)) => { + break Some(( + CloseCode::Protocol, + format!("the server sent a request ({})", request.method), + )); + } + Ok(RpcMessage::Notification(notification)) => { + break Some(( + CloseCode::Protocol, + format!("the server sent a notification ({})", notification.method), + )); + } + Err(rejection) => { + break Some(( + CloseCode::Protocol, + format!("unacceptable frame: {}", rejection.error), + )); + } + }, + Some(Ok(Message::Binary(_))) => { + break Some(( + CloseCode::Protocol, + "binary frames are not part of the protocol".to_owned(), + )); + } + Some(Ok(Message::Close(frame))) => { + // Keep reading: the next read flushes the close reply and + // then ends the stream. + shared.terminate(ClientError::Closed { + code: frame.as_ref().map(|frame| u16::from(frame.code)), + reason: frame + .map(|frame| frame.reason.to_string()) + .filter(|reason| !reason.is_empty()), + }); + } + Some(Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_))) => {} + } + }; + let frame = match violation { + Some((code, message)) => { + shared.terminate(ClientError::InvalidMessage(message.clone())); + let mut reason = message; + truncate_reason(&mut reason); + Some(CloseFrame { + code, + reason: reason.into(), + }) + } + None => { + shared.terminate(ClientError::Closed { + code: None, + reason: None, + }); + None + } + }; + if let Some(outgoing) = outgoing.upgrade() { + let _ = outgoing.send(Outgoing::Close { frame, ack: None }); + } +} + +/// A close reason fits in a control frame: at most 123 bytes. +fn truncate_reason(reason: &mut String) { + const MAX: usize = 123; + if reason.len() > MAX { + let mut end = MAX; + while !reason.is_char_boundary(end) { + end -= 1; + } + reason.truncate(end); + } +} + +#[cfg(test)] +mod tests { + use environment_protocol::{ + data::idle::{GetIdleParams, GetIdleResponse}, + error::ProtocolError, + manifest::{EnvGetIdle, TargetGet}, + }; + + use super::*; + + fn idle_handler(response: Value) -> InProcessHandler { + Arc::new(move |request: Request| { + let response = response.clone(); + Box::pin(async move { Ok(Response::success(request.id, &response)) }) + }) + } + + #[tokio::test(flavor = "current_thread")] + async fn in_process_calls_decode_strictly() { + let connection = Connection::in_process( + Plane::Data, + idle_handler(serde_json::json!({"idleForMs": 5})), + ); + let idle = connection + .call::(&GetIdleParams {}, None) + .await + .unwrap(); + assert_eq!( + idle, + GetIdleResponse { + idle_for_ms: 5, + ..GetIdleResponse::default() + } + ); + let broken = Connection::in_process( + Plane::Data, + idle_handler(serde_json::json!({"idleForMs": "soon"})), + ); + assert!(matches!( + broken.call::(&GetIdleParams {}, None).await, + Err(ClientError::Decode(_)) + )); + } + + #[tokio::test(flavor = "current_thread")] + async fn server_errors_keep_their_kind() { + let connection = Connection::in_process( + Plane::Data, + Arc::new(|request: Request| { + Box::pin(async move { + Ok(Response::failure( + Some(request.id), + ProtocolError::not_initialized(), + )) + }) + }), + ); + let error = connection + .call::(&GetIdleParams {}, None) + .await + .unwrap_err(); + assert_eq!( + error.protocol_kind(), + Some(environment_protocol::error::ProtocolErrorKind::NotInitialized) + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn methods_of_another_plane_are_refused_before_sending() { + let connection = Connection::in_process(Plane::Data, idle_handler(serde_json::json!({}))); + let params = serde_json::from_value(serde_json::json!({ + "target": {"owner": { + "universeId": "u", "bindingId": "b", + "environmentId": "e", "incarnationId": "i" + }} + })) + .unwrap(); + assert!(matches!( + connection.call::(¶ms, None).await, + Err(ClientError::Encode(_)) + )); + } + + #[tokio::test(flavor = "current_thread")] + async fn closed_connections_refuse_calls() { + let connection = Connection::in_process( + Plane::Data, + idle_handler(serde_json::json!({"idleForMs": 0})), + ); + connection.close().await.unwrap(); + assert!(connection.is_closed()); + assert!(matches!( + connection.call::(&GetIdleParams {}, None).await, + Err(ClientError::Closed { + code: Some(1000), + .. + }) + )); + } + + #[test] + fn oversized_requests_are_refused_with_their_size() { + let text = "x".repeat(MAX_MESSAGE_BYTES as usize + 1); + assert!(matches!( + check_size(&text), + Err(ClientError::PayloadTooLarge { limit_bytes: MAX_MESSAGE_BYTES, actual_bytes }) + if actual_bytes == MAX_MESSAGE_BYTES + 1 + )); + } +} diff --git a/crates/environment-client/src/control.rs b/crates/environment-client/src/control.rs deleted file mode 100644 index 94f39013e..000000000 --- a/crates/environment-client/src/control.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! Typed controller-plane client methods. - -use environment_protocol::control::{ - handshake::{ControllerInitializeParams, ControllerInitializeResponse}, - ingress::{EnsureIngressParams, IngressResponse, RemoveIngressParams}, - methods::{ - ADOPT_TARGET_METHOD, CLOSE_TARGET_METHOD, CREATE_TARGET_METHOD, ENSURE_INGRESS_METHOD, - GET_TARGET_METHOD, INITIALIZE_METHOD, LIST_TARGETS_METHOD, LIST_TEMPLATES_METHOD, - REMOVE_INGRESS_METHOD, SET_TARGET_POWER_METHOD, - }, - targets::{ - AdoptTargetParams, AdoptTargetResponse, CloseTargetParams, CloseTargetResponse, - CreateTargetParams, CreateTargetResponse, GetTargetParams, GetTargetResponse, - ListTargetsParams, ListTargetsResponse, ListTemplatesParams, ListTemplatesResponse, - SetTargetPowerParams, SetTargetPowerResponse, - }, -}; - -use crate::{ - error::EnvironmentClientResult, - rpc::{JsonRpcClient, JsonRpcTransport}, - transport::{WebSocketConnectOptions, WebSocketTransport}, -}; - -pub struct EnvironmentProviderClient { - rpc: JsonRpcClient, -} - -impl EnvironmentProviderClient -where - T: JsonRpcTransport, -{ - pub fn new(transport: T) -> Self { - Self { - rpc: JsonRpcClient::new(transport), - } - } - - pub fn from_rpc(rpc: JsonRpcClient) -> Self { - Self { rpc } - } - - pub fn into_rpc(self) -> JsonRpcClient { - self.rpc - } - - /// Gracefully close the underlying transport. - pub async fn close(&mut self) -> EnvironmentClientResult<()> { - self.rpc.close().await - } - - pub async fn initialize( - &mut self, - params: &ControllerInitializeParams, - ) -> EnvironmentClientResult { - self.rpc.request(INITIALIZE_METHOD, params).await - } - - pub async fn list_targets( - &mut self, - params: &ListTargetsParams, - ) -> EnvironmentClientResult { - self.rpc.request(LIST_TARGETS_METHOD, params).await - } - - pub async fn list_templates( - &mut self, - params: &ListTemplatesParams, - ) -> EnvironmentClientResult { - self.rpc.request(LIST_TEMPLATES_METHOD, params).await - } - - pub async fn create_target( - &mut self, - params: &CreateTargetParams, - ) -> EnvironmentClientResult { - self.rpc.request(CREATE_TARGET_METHOD, params).await - } - - pub async fn adopt_target( - &mut self, - params: &AdoptTargetParams, - ) -> EnvironmentClientResult { - self.rpc.request(ADOPT_TARGET_METHOD, params).await - } - - pub async fn get_target( - &mut self, - params: &GetTargetParams, - ) -> EnvironmentClientResult { - self.rpc.request(GET_TARGET_METHOD, params).await - } - - pub async fn close_target( - &mut self, - params: &CloseTargetParams, - ) -> EnvironmentClientResult { - self.rpc.request(CLOSE_TARGET_METHOD, params).await - } - - pub async fn set_target_power( - &mut self, - params: &SetTargetPowerParams, - ) -> EnvironmentClientResult { - self.rpc.request(SET_TARGET_POWER_METHOD, params).await - } - - pub async fn ensure_ingress( - &mut self, - params: &EnsureIngressParams, - ) -> EnvironmentClientResult { - self.rpc.request(ENSURE_INGRESS_METHOD, params).await - } - - pub async fn remove_ingress( - &mut self, - params: &RemoveIngressParams, - ) -> EnvironmentClientResult { - self.rpc.request(REMOVE_INGRESS_METHOD, params).await - } -} - -impl EnvironmentProviderClient { - pub async fn connect( - endpoint: &str, - options: WebSocketConnectOptions, - ) -> EnvironmentClientResult { - Ok(Self::new( - WebSocketTransport::connect(endpoint, options).await?, - )) - } -} diff --git a/crates/environment-client/src/controller.rs b/crates/environment-client/src/controller.rs new file mode 100644 index 000000000..4a0b577c2 --- /dev/null +++ b/crates/environment-client/src/controller.rs @@ -0,0 +1,169 @@ +//! The controller-plane client: the runtime's connection to one provider +//! controller. + +use std::{sync::Arc, time::Duration}; + +use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, + control::handshake::{ControllerInitializeParams, ControllerInitializeResponse}, + handshake::Plane, + jsonrpc::{Request, Response}, + manifest::{ + ControllerInitialize, IngressEnsure, IngressRemove, RequestMethod, TargetAdopt, + TargetClose, TargetCreate, TargetGet, TargetSetPower, TemplateList, + }, +}; +use futures_util::future::BoxFuture; + +use crate::{ + connection::{Connection, InProcessHandler}, + data::typed_calls, + error::ClientError, + options::ConnectOptions, +}; + +/// A controller-plane connection after a successful `initialize`. +/// +/// Same shape as [`DataClient`](crate::DataClient): cheap to clone, calls +/// take `&self` and may run concurrently, a dropped call is abandoned. The +/// controller plane has no `$/cancelRequest`; a mutation abandoned by its +/// caller still runs to completion on the provider. +#[derive(Clone)] +pub struct ControllerClient { + inner: Arc, +} + +struct Inner { + connection: Connection, + initialize: ControllerInitializeResponse, +} + +impl ControllerClient { + /// Dial the controller URL and run `initialize` with + /// [`CONTROLLER_PROTOCOL_VERSION`]. Pass the provider credential as + /// [`ConnectOptions::bearer_token`]. Fails with + /// [`ClientError::VersionMismatch`] or [`ClientError::WrongPlane`] + /// before the connection is used. The caller still checks + /// `routes.data` against the controller URL + /// (`DataRouteBase::check_for_controller`). + pub async fn connect_initialized( + endpoint: &str, + options: ConnectOptions, + ) -> Result<(Self, ControllerInitializeResponse), ClientError> { + let connection = Connection::connect(endpoint, &options, Plane::Controller).await?; + let initialized = initialize(&connection, &options).await; + match initialized { + Ok(initialize) => Ok(( + Self { + inner: Arc::new(Inner { + connection, + initialize: initialize.clone(), + }), + }, + initialize, + )), + Err(error) => { + let _ = connection.close().await; + Err(error) + } + } + } + + /// A client whose requests `handler` answers in process, for tests that + /// need no socket (see [`DataClient::in_process`](crate::DataClient::in_process)). + pub fn in_process( + initialize: ControllerInitializeResponse, + handler: impl Fn(Request) -> BoxFuture<'static, Result> + + Send + + Sync + + 'static, + ) -> Self { + let handler: InProcessHandler = Arc::new(handler); + Self { + inner: Arc::new(Inner { + connection: Connection::in_process(Plane::Controller, handler), + initialize, + }), + } + } + + /// The controller's `initialize` response: capabilities and routes. + pub fn initialize_response(&self) -> &ControllerInitializeResponse { + &self.inner.initialize + } + + /// Call any controller-plane method, with the connection's default + /// deadline ([`ConnectOptions::request_timeout`]). + pub async fn call( + &self, + params: &M::Params, + ) -> Result { + self.inner.connection.call::(params, None).await + } + + /// [`call`](Self::call) with this deadline instead of the default one. + pub async fn call_with_timeout( + &self, + params: &M::Params, + timeout: Duration, + ) -> Result { + self.inner.connection.call::(params, Some(timeout)).await + } + + /// Close the connection for every clone. Idempotent. + pub async fn close(&self) -> Result<(), ClientError> { + self.inner.connection.close().await + } + + /// True once the connection ended, by either side. + pub fn is_closed(&self) -> bool { + self.inner.connection.is_closed() + } + + typed_calls! { + /// `template/list`. + list_templates => TemplateList; + /// `target/create`. + create_target => TargetCreate; + /// `target/adopt`. + adopt_target => TargetAdopt; + /// `target/get`. + get_target => TargetGet; + /// `target/setPower`. + set_target_power => TargetSetPower; + /// `target/close`. + close_target => TargetClose; + /// `ingress/ensure`. + ensure_ingress => IngressEnsure; + /// `ingress/remove`. + remove_ingress => IngressRemove; + } +} + +impl std::fmt::Debug for ControllerClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ControllerClient") + .field("server_info", &self.inner.initialize.server_info) + .field("closed", &self.is_closed()) + .finish_non_exhaustive() + } +} + +/// Run the controller `initialize` on a dialed connection. +pub(crate) async fn initialize( + connection: &Connection, + options: &ConnectOptions, +) -> Result { + connection + .initialize( + ControllerInitialize::NAME, + &ControllerInitializeParams { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + client_info: options.client_info.clone(), + }, + CONTROLLER_PROTOCOL_VERSION, + options.initialize_timeout, + ) + .await +} diff --git a/crates/environment-client/src/data.rs b/crates/environment-client/src/data.rs index ffdd96a6a..f13d82286 100644 --- a/crates/environment-client/src/data.rs +++ b/crates/environment-client/src/data.rs @@ -1,257 +1,255 @@ -//! Typed data-plane client methods. - -use environment_protocol::data::methods::{FS_CAPTURE_METHOD, FS_MATERIALIZE_METHOD}; -use environment_protocol::data::transfer::*; -use environment_protocol::data::{ - fs::{ - CopyParams, CopyResponse, CreateDirectoryParams, CreateDirectoryResponse, - GetMetadataParams, GetMetadataResponse, GlobFilesParams, GlobFilesResponse, - ReadDirectoryParams, ReadDirectoryResponse, ReadFileParams, ReadFileResponse, RemoveParams, - RemoveResponse, SearchTextParams, SearchTextResponse, WriteFileParams, WriteFileResponse, - }, - handshake::{InitializeParams, InitializeResponse, InitializedParams}, - idle::{IdleParams, IdleResponse}, - jobs::{ - CancelJobsParams, CancelJobsResponse, ListJobsParams, ListJobsResponse, ReadJobsParams, - ReadJobsResponse, StartJobsParams, StartJobsResponse, - }, - methods::{ - ENV_IDLE_METHOD, FS_COPY_METHOD, FS_CREATE_DIRECTORY_METHOD, FS_GET_METADATA_METHOD, - FS_GLOB_FILES_METHOD, FS_READ_DIRECTORY_METHOD, FS_READ_FILE_METHOD, FS_REMOVE_METHOD, - FS_SEARCH_TEXT_METHOD, FS_WRITE_FILE_METHOD, INITIALIZE_METHOD, INITIALIZED_METHOD, - JOB_CANCEL_METHOD, JOB_LIST_METHOD, JOB_READ_METHOD, JOB_START_METHOD, PROCESS_READ_METHOD, - PROCESS_RESIZE_METHOD, PROCESS_START_METHOD, PROCESS_TERMINATE_METHOD, - PROCESS_WRITE_METHOD, - }, - process::{ - ReadProcessParams, ReadProcessResponse, ResizeProcessParams, ResizeProcessResponse, - StartProcessParams, StartProcessResponse, TerminateProcessParams, TerminateProcessResponse, - WriteProcessParams, WriteProcessResponse, +//! The data-plane client: the runtime's connection to one envd. + +use std::{sync::Arc, time::Duration}; + +use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + data::handshake::{InitializeParams, InitializeResponse}, + handshake::Plane, + jsonrpc::{Request, Response}, + manifest::{ + DataInitialize, EnvGetIdle, FsCreateDirectory, FsGetMetadata, FsGlobFiles, FsReadDirectory, + FsReadFile, FsRemove, FsRename, FsScan, FsSearchText, FsWriteFile, JobCancel, JobRead, + JobStart, ProcessRead, ProcessStart, ProcessTerminate, ProcessWrite, RequestMethod, + TransferAbort, TransferAdvance, TransferAppend, TransferBegin, TransferCommit, + TransferGetStatus, TransferListMissing, TransferRead, TransferReadInventory, TransferWrite, }, }; +use futures_util::future::BoxFuture; +use tokio::io::{AsyncRead, AsyncWrite}; use crate::{ - error::EnvironmentClientResult, - rpc::{JsonRpcClient, JsonRpcNotification, JsonRpcTransport}, - transport::{WebSocketConnectOptions, WebSocketTransport}, + connection::{Connection, InProcessHandler}, + error::ClientError, + options::ConnectOptions, }; -pub struct EnvironmentDataClient { - rpc: JsonRpcClient, +/// Typed sugar: one method per manifest entry, each a [`call`] of its +/// marker. +/// +/// [`call`]: DataClient::call +macro_rules! typed_calls { + ($( $(#[$doc:meta])* $name:ident => $marker:ident; )+) => { + $( + $(#[$doc])* + pub async fn $name( + &self, + params: &<$marker as RequestMethod>::Params, + ) -> Result<<$marker as RequestMethod>::Response, ClientError> { + self.call::<$marker>(params).await + } + )+ + }; +} +pub(crate) use typed_calls; + +/// A data-plane connection after a successful `initialize`. +/// +/// Cheap to clone; clones share one socket. Calls take `&self`, may run +/// concurrently and are answered in any order. Dropping a call's future +/// abandons it (an abortable one is sent `$/cancelRequest`). The socket +/// closes when [`close`](Self::close) is called or the last clone is +/// dropped. +#[derive(Clone)] +pub struct DataClient { + inner: Arc, } -impl EnvironmentDataClient -where - T: JsonRpcTransport, -{ - pub fn new(transport: T) -> Self { +struct Inner { + connection: Connection, + initialize: InitializeResponse, +} + +impl DataClient { + /// Dial `endpoint` (`ws://`, `wss://` or `unix:///path`) and run + /// `initialize` with [`ENVD_PROTOCOL_VERSION`]. Fails with + /// [`ClientError::VersionMismatch`] or [`ClientError::WrongPlane`] + /// before the connection is used; sends nothing else. + pub async fn connect_initialized( + endpoint: &str, + options: ConnectOptions, + ) -> Result<(Self, InitializeResponse), ClientError> { + let connection = Connection::connect(endpoint, &options, Plane::Data).await?; + Self::initialize(connection, &options).await + } + + /// [`connect_initialized`](Self::connect_initialized) over a byte + /// stream that is already connected to a data server, such as one end + /// of [`tokio::io::duplex`] or an accepted Unix socket. The upgrade + /// targets `/` with `Host: localhost`. + pub async fn connect_initialized_over( + stream: S, + options: ConnectOptions, + ) -> Result<(Self, InitializeResponse), ClientError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let connection = Connection::connect_over(stream, &options, Plane::Data).await?; + Self::initialize(connection, &options).await + } + + /// A client whose requests `handler` answers in process, for tests that + /// need no socket. `initialize` is what + /// [`initialize_response`](Self::initialize_response) reports; the + /// handler sees every later request, with the id this client chose. + /// Requests and responses pass the same size limit and strict decoding + /// as on a socket; calls have no default deadline. A handler `Err` fails + /// that one call with the error, a stand-in for a lost response. + pub fn in_process( + initialize: InitializeResponse, + handler: impl Fn(Request) -> BoxFuture<'static, Result> + + Send + + Sync + + 'static, + ) -> Self { + let handler: InProcessHandler = Arc::new(handler); Self { - rpc: JsonRpcClient::new(transport), + inner: Arc::new(Inner { + connection: Connection::in_process(Plane::Data, handler), + initialize, + }), } } - pub fn from_rpc(rpc: JsonRpcClient) -> Self { - Self { rpc } - } - - pub fn into_rpc(self) -> JsonRpcClient { - self.rpc - } - - /// Gracefully close the underlying transport. - pub async fn close(&mut self) -> EnvironmentClientResult<()> { - self.rpc.close().await - } - - pub async fn initialize( - &mut self, - params: &InitializeParams, - ) -> EnvironmentClientResult { - self.rpc.request(INITIALIZE_METHOD, params).await - } - - pub async fn initialized(&mut self, params: &InitializedParams) -> EnvironmentClientResult<()> { - self.rpc.notify(INITIALIZED_METHOD, params).await - } - - pub async fn scan( - &mut self, - params: &environment_protocol::data::inventory::ScanParams, - ) -> EnvironmentClientResult { - self.rpc - .request(environment_protocol::data::methods::FS_SCAN_METHOD, params) - .await - } - - pub async fn transfer( - &mut self, - params: &environment_protocol::data::transfer_session::TransferRequest, - ) -> EnvironmentClientResult - { - self.rpc - .request( - environment_protocol::data::methods::FS_TRANSFER_METHOD, - params, + async fn initialize( + connection: Connection, + options: &ConnectOptions, + ) -> Result<(Self, InitializeResponse), ClientError> { + let params = InitializeParams { + protocol_version: ENVD_PROTOCOL_VERSION, + client_info: options.client_info.clone(), + }; + let initialized = connection + .initialize::<_, InitializeResponse>( + DataInitialize::NAME, + ¶ms, + ENVD_PROTOCOL_VERSION, + options.initialize_timeout, ) - .await - } - - pub async fn capture( - &mut self, - params: &CaptureParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_CAPTURE_METHOD, params).await - } - - pub async fn materialize( - &mut self, - params: &MaterializeParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_MATERIALIZE_METHOD, params).await - } - - pub async fn read_file( - &mut self, - params: &ReadFileParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_READ_FILE_METHOD, params).await - } - - pub async fn write_file( - &mut self, - params: &WriteFileParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_WRITE_FILE_METHOD, params).await - } - - pub async fn create_directory( - &mut self, - params: &CreateDirectoryParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_CREATE_DIRECTORY_METHOD, params).await - } - - pub async fn get_metadata( - &mut self, - params: &GetMetadataParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_GET_METADATA_METHOD, params).await - } - - pub async fn read_directory( - &mut self, - params: &ReadDirectoryParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_READ_DIRECTORY_METHOD, params).await - } - - pub async fn remove( - &mut self, - params: &RemoveParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_REMOVE_METHOD, params).await - } - - pub async fn copy(&mut self, params: &CopyParams) -> EnvironmentClientResult { - self.rpc.request(FS_COPY_METHOD, params).await - } - - pub async fn search_text( - &mut self, - params: &SearchTextParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_SEARCH_TEXT_METHOD, params).await - } - - pub async fn glob_files( - &mut self, - params: &GlobFilesParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_GLOB_FILES_METHOD, params).await - } - - pub async fn start_process( - &mut self, - params: &StartProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_START_METHOD, params).await - } - - pub async fn read_process( - &mut self, - params: &ReadProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_READ_METHOD, params).await - } - - pub async fn write_process( - &mut self, - params: &WriteProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_WRITE_METHOD, params).await - } - - pub async fn terminate_process( - &mut self, - params: &TerminateProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_TERMINATE_METHOD, params).await - } - - pub async fn resize_process( - &mut self, - params: &ResizeProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_RESIZE_METHOD, params).await - } - - pub async fn start_jobs( - &mut self, - params: &StartJobsParams, - ) -> EnvironmentClientResult { - self.rpc.request(JOB_START_METHOD, params).await - } - - pub async fn list_jobs( - &mut self, - params: &ListJobsParams, - ) -> EnvironmentClientResult { - self.rpc.request(JOB_LIST_METHOD, params).await - } - - pub async fn read_jobs( - &mut self, - params: &ReadJobsParams, - ) -> EnvironmentClientResult { - self.rpc.request(JOB_READ_METHOD, params).await - } - - pub async fn cancel_jobs( - &mut self, - params: &CancelJobsParams, - ) -> EnvironmentClientResult { - self.rpc.request(JOB_CANCEL_METHOD, params).await - } - - /// Daemon activity report for idle power policy. Not itself activity. - pub async fn idle(&mut self, params: &IdleParams) -> EnvironmentClientResult { - self.rpc.request(ENV_IDLE_METHOD, params).await + .await; + match initialized { + Ok(initialize) => Ok(( + Self { + inner: Arc::new(Inner { + connection, + initialize: initialize.clone(), + }), + }, + initialize, + )), + Err(error) => { + let _ = connection.close().await; + Err(error) + } + } } - pub async fn next_notification( - &mut self, - ) -> EnvironmentClientResult> { - self.rpc.next_notification().await + /// The daemon's `initialize` response: instance id, capabilities, + /// ceilings and default cwd. + pub fn initialize_response(&self) -> &InitializeResponse { + &self.inner.initialize + } + + /// Call any data-plane method, with the connection's default deadline + /// ([`ConnectOptions::request_timeout`]). + pub async fn call( + &self, + params: &M::Params, + ) -> Result { + self.inner.connection.call::(params, None).await + } + + /// [`call`](Self::call) with this deadline instead of the default one. + /// When it passes, an abortable method is sent `$/cancelRequest` and the + /// call fails with [`ClientError::DeadlineExceeded`]. + pub async fn call_with_timeout( + &self, + params: &M::Params, + timeout: Duration, + ) -> Result { + self.inner.connection.call::(params, Some(timeout)).await + } + + /// Close the connection for every clone: pending calls fail with + /// [`ClientError::Closed`]. Idempotent. + pub async fn close(&self) -> Result<(), ClientError> { + self.inner.connection.close().await + } + + /// True once the connection ended, by either side. + pub fn is_closed(&self) -> bool { + self.inner.connection.is_closed() + } + + typed_calls! { + /// `env/getIdle`. + get_idle => EnvGetIdle; + /// `fs/readFile`. + read_file => FsReadFile; + /// `fs/writeFile`. + write_file => FsWriteFile; + /// `fs/createDirectory`. + create_directory => FsCreateDirectory; + /// `fs/getMetadata`. + get_metadata => FsGetMetadata; + /// `fs/readDirectory`. + read_directory => FsReadDirectory; + /// `fs/remove`. + remove => FsRemove; + /// `fs/rename`. + rename => FsRename; + /// `fs/searchText`. + search_text => FsSearchText; + /// `fs/globFiles`. + glob_files => FsGlobFiles; + /// `fs/scan`. + scan => FsScan; + /// `transfer/begin`. + transfer_begin => TransferBegin; + /// `transfer/advance`. + transfer_advance => TransferAdvance; + /// `transfer/getStatus`. + transfer_get_status => TransferGetStatus; + /// `transfer/readInventory`. + transfer_read_inventory => TransferReadInventory; + /// `transfer/append`. + transfer_append => TransferAppend; + /// `transfer/listMissing`. + transfer_list_missing => TransferListMissing; + /// `transfer/read`. + transfer_read => TransferRead; + /// `transfer/write`. + transfer_write => TransferWrite; + /// `transfer/commit`. + transfer_commit => TransferCommit; + /// `transfer/abort`. + transfer_abort => TransferAbort; + /// `process/start`. + start_process => ProcessStart; + /// `process/read`. + read_process => ProcessRead; + /// `process/write`. + write_process => ProcessWrite; + /// `process/terminate`. + terminate_process => ProcessTerminate; + /// `job/start`. + start_jobs => JobStart; + /// `job/read`. + read_jobs => JobRead; + /// `job/cancel`. + cancel_jobs => JobCancel; } } -impl EnvironmentDataClient { - pub async fn connect( - endpoint: &str, - options: WebSocketConnectOptions, - ) -> EnvironmentClientResult { - Ok(Self::new( - WebSocketTransport::connect(endpoint, options).await?, - )) +impl std::fmt::Debug for DataClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("DataClient") + .field( + "daemon_instance_id", + &self.inner.initialize.daemon_instance_id, + ) + .field("closed", &self.is_closed()) + .finish_non_exhaustive() } } diff --git a/crates/environment-client/src/dial.rs b/crates/environment-client/src/dial.rs new file mode 100644 index 000000000..544fe68d8 --- /dev/null +++ b/crates/environment-client/src/dial.rs @@ -0,0 +1,244 @@ +//! Endpoint dialing: `ws://`, `wss://` and `unix:///path`, with the +//! protocol's message limits and the caller's headers. + +use std::time::Duration; + +use environment_protocol::limits::MAX_MESSAGE_BYTES; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::UnixStream, +}; +use tokio_tungstenite::{ + WebSocketStream, client_async_with_config, connect_async_with_config, + tungstenite::{ + self, + client::IntoClientRequest, + handshake::client::Request, + http::{ + HeaderName, HeaderValue, + header::{AUTHORIZATION, USER_AGENT}, + }, + protocol::WebSocketConfig, + }, +}; + +use crate::{error::ClientError, options::ConnectOptions}; + +/// The WebSocket settings of every client socket: one explicit limit for +/// messages and frames, the protocol's [`MAX_MESSAGE_BYTES`]. +pub(crate) fn websocket_config() -> WebSocketConfig { + WebSocketConfig::default() + .max_message_size(Some(MAX_MESSAGE_BYTES as usize)) + .max_frame_size(Some(MAX_MESSAGE_BYTES as usize)) +} + +enum Target<'a> { + /// `ws://` or `wss://`: dialed as given. + Url, + /// `unix:///path`: the socket path; the upgrade targets `/` with + /// `Host: localhost`. + Unix(&'a str), +} + +fn target(endpoint: &str) -> Result, ClientError> { + if endpoint.starts_with("ws://") || endpoint.starts_with("wss://") { + return Ok(Target::Url); + } + if let Some(path) = endpoint.strip_prefix("unix://") { + if path.len() > 1 && path.starts_with('/') && !path.contains(['?', '#']) { + return Ok(Target::Unix(path)); + } + return Err(configuration( + endpoint, + "a unix endpoint names an absolute socket path: unix:///path/to/socket", + )); + } + Err(configuration( + endpoint, + "the endpoint scheme must be ws://, wss:// or unix://", + )) +} + +fn configuration(endpoint: &str, message: impl Into) -> ClientError { + ClientError::Configuration { + endpoint: endpoint.to_owned(), + message: message.into(), + } +} + +fn upgrade_request( + endpoint: &str, + url: &str, + options: &ConnectOptions, +) -> Result { + let mut request = url + .into_client_request() + .map_err(|error| configuration(endpoint, error.to_string()))?; + let headers = request.headers_mut(); + let user_agent = HeaderValue::from_str(&options.effective_user_agent()) + .map_err(|_| configuration(endpoint, "the user agent is not a valid header value"))?; + headers.insert(USER_AGENT, user_agent); + if let Some(token) = &options.bearer_token { + let mut value = HeaderValue::from_str(&format!("Bearer {}", token.expose())) + .map_err(|_| configuration(endpoint, "the bearer token is not a valid header value"))?; + value.set_sensitive(true); + headers.insert(AUTHORIZATION, value); + } + for (name, value) in &options.headers { + let header = HeaderName::from_bytes(name.as_bytes()) + .map_err(|_| configuration(endpoint, format!("invalid header name {name:?}")))?; + let value = HeaderValue::from_str(value).map_err(|_| { + configuration(endpoint, format!("header {name:?} has an invalid value")) + })?; + headers.insert(header, value); + } + Ok(request) +} + +fn dial_error(endpoint: &str, error: tungstenite::Error) -> ClientError { + match error { + tungstenite::Error::Http(response) => ClientError::UpgradeRejected { + status: response.status().as_u16(), + }, + tungstenite::Error::Url(error) => configuration(endpoint, error.to_string()), + error => ClientError::Connect { + endpoint: endpoint.to_owned(), + message: error.to_string(), + }, + } +} + +async fn bounded( + endpoint: &str, + timeout: Duration, + dial: impl Future>, +) -> Result { + tokio::time::timeout(timeout, dial) + .await + .map_err(|_| ClientError::ConnectTimeout { + endpoint: endpoint.to_owned(), + })? +} + +/// A dialed socket of either transport. +pub(crate) enum Dialed { + Tcp(Box>>), + Unix(Box>), +} + +/// Dial `endpoint` and complete the WebSocket upgrade within +/// `options.connect_timeout`. +pub(crate) async fn dial(endpoint: &str, options: &ConnectOptions) -> Result { + match target(endpoint)? { + Target::Url => { + let request = upgrade_request(endpoint, endpoint, options)?; + bounded(endpoint, options.connect_timeout, async { + connect_async_with_config(request, Some(websocket_config()), true) + .await + .map(|(socket, _)| Dialed::Tcp(Box::new(socket))) + .map_err(|error| dial_error(endpoint, error)) + }) + .await + } + Target::Unix(path) => { + let request = upgrade_request(endpoint, "ws://localhost/", options)?; + bounded(endpoint, options.connect_timeout, async { + let stream = + UnixStream::connect(path) + .await + .map_err(|error| ClientError::Connect { + endpoint: endpoint.to_owned(), + message: error.to_string(), + })?; + client_async_with_config(request, stream, Some(websocket_config())) + .await + .map(|(socket, _)| Dialed::Unix(Box::new(socket))) + .map_err(|error| dial_error(endpoint, error)) + }) + .await + } + } +} + +/// Complete the WebSocket upgrade over an already connected byte stream +/// (request target `/`, `Host: localhost`) within +/// `options.connect_timeout`. +pub(crate) async fn upgrade( + stream: S, + options: &ConnectOptions, +) -> Result, ClientError> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + const ENDPOINT: &str = "ws://localhost/"; + let request = upgrade_request(ENDPOINT, ENDPOINT, options)?; + bounded(ENDPOINT, options.connect_timeout, async { + client_async_with_config(request, stream, Some(websocket_config())) + .await + .map(|(socket, _)| socket) + .map_err(|error| dial_error(ENDPOINT, error)) + }) + .await +} + +#[cfg(test)] +mod tests { + use environment_protocol::handshake::ImplementationInfo; + + use super::*; + + fn options() -> ConnectOptions { + ConnectOptions::new( + ImplementationInfo::new("lightspeed-test") + .and_then(|info| info.with_version("1.0.0")) + .unwrap(), + ) + } + + #[test] + fn endpoints_select_their_dialer() { + assert!(matches!(target("ws://127.0.0.1:1/x"), Ok(Target::Url))); + assert!(matches!(target("wss://example.test"), Ok(Target::Url))); + assert!(matches!( + target("unix:///run/envd.sock"), + Ok(Target::Unix("/run/envd.sock")) + )); + for bad in [ + "http://example.test", + "unix://relative.sock", + "unix:///", + "unix:///run/envd.sock?x=1", + "stdio://", + ] { + assert!( + matches!(target(bad), Err(ClientError::Configuration { .. })), + "{bad}" + ); + } + } + + #[test] + fn upgrade_requests_carry_agent_credential_and_headers() { + let request = upgrade_request( + "ws://h/", + "ws://h/", + &options() + .with_bearer_token("secret") + .with_header("x-lightspeed-test", "1"), + ) + .unwrap(); + let headers = request.headers(); + assert_eq!(headers[USER_AGENT], "lightspeed-test/1.0.0"); + assert_eq!(headers[AUTHORIZATION], "Bearer secret"); + assert!(headers[AUTHORIZATION].is_sensitive()); + assert_eq!(headers["x-lightspeed-test"], "1"); + assert!(matches!( + upgrade_request( + "ws://h/", + "ws://h/", + &options().with_header("bad name", "1") + ), + Err(ClientError::Configuration { .. }) + )); + } +} diff --git a/crates/environment-client/src/error.rs b/crates/environment-client/src/error.rs index 60843fc7d..3008d7c30 100644 --- a/crates/environment-client/src/error.rs +++ b/crates/environment-client/src/error.rs @@ -1,24 +1,223 @@ -//! Client-side environment protocol errors. +//! Client-side errors. -use environment_protocol::error::EnvironmentProtocolError; +use environment_protocol::{ + error::{ProtocolError, ProtocolErrorKind}, + handshake::Plane, +}; use thiserror::Error; -pub type EnvironmentClientResult = Result; +/// Why a connect or call failed. +/// +/// A server's error answer is [`ClientError::Protocol`] with the server's +/// error kind intact; everything else is a fact about the connection or a +/// contract breach the client detected itself. The client never +/// synthesizes a protocol error kind. +#[derive(Clone, Debug, Error)] +pub enum ClientError { + /// The connection settings cannot be used: an endpoint that is not + /// `ws://`, `wss://` or `unix:///path`, or a header that is not valid + /// HTTP. Retrying does not help. + #[error("invalid connection settings for {endpoint}: {message}")] + Configuration { endpoint: String, message: String }, -#[derive(Debug, Error)] -pub enum EnvironmentClientError { - #[error("failed to serialize JSON-RPC message: {0}")] - Serialize(#[from] serde_json::Error), + /// TCP or Unix connect, TLS or the WebSocket upgrade failed below HTTP. + #[error("connect to {endpoint}: {message}")] + Connect { endpoint: String, message: String }, - #[error("transport closed")] - TransportClosed, + /// Connect, TLS and upgrade did not finish within + /// [`ConnectOptions::connect_timeout`](crate::ConnectOptions::connect_timeout). + #[error("connect to {endpoint} timed out")] + ConnectTimeout { endpoint: String }, - #[error("transport error: {0}")] - Transport(String), + /// The server answered the upgrade with an HTTP status instead of + /// switching protocols: 401 for a missing or wrong credential, 503 for + /// a route that is not ready yet. + #[error("upgrade refused with HTTP {status}")] + UpgradeRejected { status: u16 }, + + /// The server speaks another protocol version on this plane: it + /// answered `initialize` with `versionMismatch`, or with a different + /// `protocolVersion`. `local` is this client's version. + #[error("{plane} protocol mismatch: this client speaks {local}, the server {remote}")] + VersionMismatch { + plane: Plane, + local: u32, + remote: u32, + }, + + /// The server's `initialize` response names another plane: the + /// endpoint serves something else. + #[error("expected the {expected} plane, the server serves {actual}")] + WrongPlane { expected: Plane, actual: Plane }, + + /// The server answered the call with this error. + #[error("{0}")] + Protocol(ProtocolError), + + /// The call's deadline passed before its response arrived. On the data + /// plane an abortable call was also sent `$/cancelRequest`. + #[error("deadline exceeded after {elapsed_ms} ms")] + DeadlineExceeded { elapsed_ms: u64 }, + + /// The connection is closed: by the server (with its close code and + /// reason when it sent a close frame), by the transport ending, or by + /// [`close`](crate::DataClient::close). + #[error("connection closed{}", describe_close(*code, reason.as_deref()))] + Closed { + code: Option, + reason: Option, + }, - #[error("invalid JSON-RPC message: {0}")] + /// The encoded request is larger than the protocol's message limit; it + /// was not sent. + #[error("request of {actual_bytes} bytes exceeds the {limit_bytes}-byte limit")] + PayloadTooLarge { limit_bytes: u64, actual_bytes: u64 }, + + /// The request could not be encoded. + #[error("encode request: {0}")] + Encode(String), + + /// The server's result does not decode as the method's response type: + /// the server broke the contract. + #[error("decode response: {0}")] + Decode(String), + + /// The server sent something no client accepts on this plane: a + /// request or notification, a binary frame, a frame over the message + /// limit, or a frame that is not a valid response. The client closed + /// the connection and failed every pending call with this error. + #[error("invalid message from server: {0}")] InvalidMessage(String), - #[error("environment protocol error: {0:?}")] - Protocol(EnvironmentProtocolError), + /// The socket failed after the connection was established. + #[error("transport: {0}")] + Transport(String), +} + +impl ClientError { + /// The server's error kind: the kind of a [`Protocol`](Self::Protocol) + /// error, and `versionMismatch` for + /// [`VersionMismatch`](Self::VersionMismatch). `None` for failures the + /// client observed itself. + pub fn protocol_kind(&self) -> Option { + match self { + Self::Protocol(error) => Some(error.kind()), + Self::VersionMismatch { .. } => Some(ProtocolErrorKind::VersionMismatch), + _ => None, + } + } + + /// The server's error, for [`Protocol`](Self::Protocol). + pub fn protocol_error(&self) -> Option<&ProtocolError> { + match self { + Self::Protocol(error) => Some(error), + _ => None, + } + } + + /// True when the same call may succeed later without any change: a + /// retryable protocol kind (`busy`, `unavailable`, `aborted`), a + /// connection failure, a closed connection, a deadline, or an upgrade + /// refused with 429 or a 5xx status. Version and plane mismatches, + /// contract breaches and refusals are not. + pub fn is_retryable(&self) -> bool { + match self { + Self::Protocol(error) => error.is_retryable(), + Self::Connect { .. } + | Self::ConnectTimeout { .. } + | Self::Closed { .. } + | Self::Transport(_) + | Self::DeadlineExceeded { .. } => true, + Self::UpgradeRejected { status } => *status == 429 || (500..600).contains(status), + Self::Configuration { .. } + | Self::VersionMismatch { .. } + | Self::WrongPlane { .. } + | Self::PayloadTooLarge { .. } + | Self::Encode(_) + | Self::Decode(_) + | Self::InvalidMessage(_) => false, + } + } +} + +fn describe_close(code: Option, reason: Option<&str>) -> String { + match (code, reason) { + (Some(code), Some(reason)) if !reason.is_empty() => format!(" ({code}: {reason})"), + (Some(code), _) => format!(" ({code})"), + (None, _) => String::new(), + } +} + +#[cfg(test)] +mod tests { + use environment_protocol::error::{AbortReason, ProtocolError}; + + use super::*; + + #[test] + fn protocol_errors_keep_their_kind_and_retryability() { + let aborted = ClientError::Protocol(ProtocolError::aborted( + Some(AbortReason::Cancelled), + "cancelled", + )); + assert_eq!(aborted.protocol_kind(), Some(ProtocolErrorKind::Aborted)); + assert!(aborted.is_retryable()); + let missing = ClientError::Protocol(ProtocolError::not_found("no such process")); + assert_eq!(missing.protocol_kind(), Some(ProtocolErrorKind::NotFound)); + assert!(!missing.is_retryable()); + assert_eq!(missing.to_string(), "notFound: no such process"); + } + + #[test] + fn connection_failures_are_retryable_and_breaches_are_not() { + assert!( + ClientError::Closed { + code: None, + reason: None + } + .is_retryable() + ); + assert!(ClientError::DeadlineExceeded { elapsed_ms: 10 }.is_retryable()); + assert!(ClientError::UpgradeRejected { status: 503 }.is_retryable()); + assert!(ClientError::UpgradeRejected { status: 429 }.is_retryable()); + assert!(!ClientError::UpgradeRejected { status: 401 }.is_retryable()); + assert!(!ClientError::Decode("x".to_owned()).is_retryable()); + assert!( + !ClientError::VersionMismatch { + plane: Plane::Data, + local: 3, + remote: 2 + } + .is_retryable() + ); + assert_eq!( + ClientError::VersionMismatch { + plane: Plane::Controller, + local: 3, + remote: 4 + } + .protocol_kind(), + Some(ProtocolErrorKind::VersionMismatch) + ); + } + + #[test] + fn close_display_names_code_and_reason() { + assert_eq!( + ClientError::Closed { + code: Some(1008), + reason: Some("route closed".to_owned()) + } + .to_string(), + "connection closed (1008: route closed)" + ); + assert_eq!( + ClientError::Closed { + code: None, + reason: None + } + .to_string(), + "connection closed" + ); + } } diff --git a/crates/environment-client/src/lib.rs b/crates/environment-client/src/lib.rs index 4717b0236..8cb949316 100644 --- a/crates/environment-client/src/lib.rs +++ b/crates/environment-client/src/lib.rs @@ -1,16 +1,66 @@ -//! Typed client for environment protocol data-plane and controller-plane calls. +//! Clients of the environment protocol's data and controller planes. //! -//! The crate owns transport and JSON-RPC mechanics while reusing pure protocol -//! records from `environment-protocol`. +//! [`DataClient`] talks to one envd, [`ControllerClient`] to one provider +//! controller. Both are opened with `connect_initialized`, which dials +//! (`ws://`, `wss://` or `unix:///path`), runs `initialize` and fails with a +//! typed [`ClientError::VersionMismatch`] or [`ClientError::WrongPlane`] +//! before anything else is sent. Calls are typed by the protocol's method +//! markers ([`environment_protocol::manifest`]): +//! +//! ```no_run +//! # async fn example() -> Result<(), environment_client::ClientError> { +//! use environment_client::{ConnectOptions, DataClient}; +//! use environment_protocol::{ +//! data::idle::GetIdleParams, handshake::ImplementationInfo, manifest::EnvGetIdle, +//! }; +//! +//! let info = ImplementationInfo::new("lightspeed-runtime").expect("valid name"); +//! let (client, initialize) = +//! DataClient::connect_initialized("ws://127.0.0.1:7070", ConnectOptions::new(info)).await?; +//! let idle = client.call::(&GetIdleParams {}).await?; +//! # let _ = (initialize, idle); +//! # Ok(()) +//! # } +//! ``` +//! +//! # Connection behaviour +//! +//! - **Multiplexed.** Calls take `&self`, may be issued concurrently on +//! one connection and are matched to responses by id. Clients are cheap +//! to clone. +//! - **Cancel-safe.** A call whose future is dropped, or whose deadline +//! passes, is forgotten; its late response is discarded. On the data +//! plane an abandoned abortable call (`process/read`, `job/read`, +//! `fs/searchText`, `fs/globFiles`, `fs/scan`) is also sent +//! `$/cancelRequest`. +//! - **Deadlines.** [`ConnectOptions`] bounds connect and `initialize` +//! (10 s each by default) and can set a default deadline for every call; +//! `call_with_timeout` overrides it per call. +//! - **Keepalive.** While any call is pending the client sends a WebSocket +//! ping every [`ConnectOptions::keepalive_interval`] (20 s by default), +//! so relays never idle-close a long wait. +//! - **Limits.** Messages and frames are limited to +//! [`MAX_MESSAGE_BYTES`](environment_protocol::limits::MAX_MESSAGE_BYTES) +//! in both directions; a larger request fails with +//! [`ClientError::PayloadTooLarge`] without being sent. +//! - **Servers only answer.** A request or notification from the server, a +//! binary frame or an unparseable frame closes the connection and fails +//! every pending call with [`ClientError::InvalidMessage`]. +//! +//! Tests that need no socket build clients with `in_process`. + +mod connection; +mod controller; +mod data; +mod dial; +mod error; +mod options; -pub mod control; -pub mod data; -pub mod error; -pub mod rpc; -pub mod transport; +#[cfg(feature = "conformance")] +pub mod conformance; -pub use control::EnvironmentProviderClient; -pub use data::EnvironmentDataClient; -pub use error::{EnvironmentClientError, EnvironmentClientResult}; -pub use rpc::{JsonRpcClient, JsonRpcNotification, JsonRpcTransport}; -pub use transport::{WebSocketConnectOptions, WebSocketTransport}; +pub use connection::InProcessHandler; +pub use controller::ControllerClient; +pub use data::DataClient; +pub use error::ClientError; +pub use options::{ConnectOptions, DEFAULT_CONNECT_TIMEOUT, DEFAULT_INITIALIZE_TIMEOUT}; diff --git a/crates/environment-client/src/options.rs b/crates/environment-client/src/options.rs new file mode 100644 index 000000000..3c482d9df --- /dev/null +++ b/crates/environment-client/src/options.rs @@ -0,0 +1,112 @@ +//! Connection settings. + +use std::{fmt, time::Duration}; + +use environment_protocol::{ + bytes::SecretString, handshake::ImplementationInfo, limits::KEEPALIVE_INTERVAL_MS, +}; + +/// Default bound of connect, TLS and WebSocket upgrade together. +pub const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +/// Default bound of the `initialize` round trip. +pub const DEFAULT_INITIALIZE_TIMEOUT: Duration = Duration::from_secs(10); + +/// How to dial and greet an endpoint. +#[derive(Clone)] +pub struct ConnectOptions { + /// Sent as `clientInfo` in `initialize`, and as the default + /// `User-Agent` (`name/version`). + pub client_info: ImplementationInfo, + /// Sent as `Authorization: Bearer ` on the upgrade request. + pub bearer_token: Option, + /// Overrides the `User-Agent` derived from `client_info`. + pub user_agent: Option, + /// Extra upgrade request headers. + pub headers: Vec<(String, String)>, + /// Bound of TCP or Unix connect, TLS and the WebSocket upgrade + /// together. Default [`DEFAULT_CONNECT_TIMEOUT`]. + pub connect_timeout: Duration, + /// Bound of the `initialize` round trip. Default + /// [`DEFAULT_INITIALIZE_TIMEOUT`]. + pub initialize_timeout: Duration, + /// Default deadline of every call made through the client; `None` + /// means calls wait as long as the caller does (tool paths rely on + /// their activity deadline). Background paths set one. Default `None`. + pub request_timeout: Option, + /// Ping interval while any call is pending (the waiting peer keeps + /// intermediaries from idle-closing a long wait). Default + /// [`KEEPALIVE_INTERVAL_MS`]. + pub keepalive_interval: Duration, +} + +impl ConnectOptions { + pub fn new(client_info: ImplementationInfo) -> Self { + Self { + client_info, + bearer_token: None, + user_agent: None, + headers: Vec::new(), + connect_timeout: DEFAULT_CONNECT_TIMEOUT, + initialize_timeout: DEFAULT_INITIALIZE_TIMEOUT, + request_timeout: None, + keepalive_interval: Duration::from_millis(KEEPALIVE_INTERVAL_MS), + } + } + + pub fn with_bearer_token(mut self, token: impl Into) -> Self { + self.bearer_token = Some(token.into()); + self + } + + pub fn with_user_agent(mut self, user_agent: impl Into) -> Self { + self.user_agent = Some(user_agent.into()); + self + } + + pub fn with_header(mut self, name: impl Into, value: impl Into) -> Self { + self.headers.push((name.into(), value.into())); + self + } + + pub fn with_request_timeout(mut self, timeout: Duration) -> Self { + self.request_timeout = Some(timeout); + self + } + + /// The `User-Agent` sent on the upgrade request. + pub(crate) fn effective_user_agent(&self) -> String { + if let Some(user_agent) = &self.user_agent { + return user_agent.clone(); + } + match self.client_info.version() { + Some(version) => format!("{}/{version}", self.client_info.name()), + None => self.client_info.name().to_owned(), + } + } +} + +impl fmt::Debug for ConnectOptions { + /// Header values and the bearer token are left out: they may carry + /// credentials. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ConnectOptions") + .field("client_info", &self.client_info) + .field("bearer_token", &self.bearer_token) + .field("user_agent", &self.user_agent) + .field( + "headers", + &self + .headers + .iter() + .map(|(name, _)| name.as_str()) + .collect::>(), + ) + .field("connect_timeout", &self.connect_timeout) + .field("initialize_timeout", &self.initialize_timeout) + .field("request_timeout", &self.request_timeout) + .field("keepalive_interval", &self.keepalive_interval) + .finish() + } +} diff --git a/crates/environment-client/src/rpc.rs b/crates/environment-client/src/rpc.rs deleted file mode 100644 index 1a74572c8..000000000 --- a/crates/environment-client/src/rpc.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! Minimal JSON-RPC 2.0 client machinery. - -use std::collections::VecDeque; - -use async_trait::async_trait; -use environment_protocol::error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}; -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::{Value, json}; - -use crate::error::{EnvironmentClientError, EnvironmentClientResult}; - -#[async_trait] -pub trait JsonRpcTransport: Send { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()>; - async fn recv(&mut self) -> EnvironmentClientResult>; - - /// Gracefully close the transport when it has an application-level - /// shutdown handshake. Stateless/in-process transports need no action. - async fn close(&mut self) -> EnvironmentClientResult<()> { - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct JsonRpcNotification { - pub method: String, - pub params: Value, -} - -pub struct JsonRpcClient { - transport: T, - next_id: u64, - notifications: VecDeque, -} - -impl JsonRpcClient -where - T: JsonRpcTransport, -{ - pub fn new(transport: T) -> Self { - Self { - transport, - next_id: 1, - notifications: VecDeque::new(), - } - } - - pub fn into_inner(self) -> T { - self.transport - } - - pub async fn close(&mut self) -> EnvironmentClientResult<()> { - self.transport.close().await - } - - pub async fn request(&mut self, method: &str, params: &P) -> EnvironmentClientResult - where - P: Serialize + Sync, - R: DeserializeOwned, - { - let id = self.next_request_id(); - self.transport - .send(json!({ - "jsonrpc": "2.0", - "id": id, - "method": method, - "params": serde_json::to_value(params)?, - })) - .await?; - - loop { - let message = self - .transport - .recv() - .await? - .ok_or(EnvironmentClientError::TransportClosed)?; - if let Some(notification) = parse_notification(&message)? { - self.notifications.push_back(notification); - continue; - } - - let Some(response_id) = message.get("id").and_then(Value::as_u64) else { - return Err(EnvironmentClientError::InvalidMessage( - "response missing numeric id".to_owned(), - )); - }; - if response_id != id { - return Err(EnvironmentClientError::InvalidMessage(format!( - "unexpected response id {response_id}, expected {id}" - ))); - } - - if let Some(error) = message.get("error") { - return Err(EnvironmentClientError::Protocol(parse_error(error)?)); - } - - let result = message.get("result").ok_or_else(|| { - EnvironmentClientError::InvalidMessage("response missing result".to_owned()) - })?; - return serde_json::from_value(result.clone()).map_err(Into::into); - } - } - - pub async fn notify

(&mut self, method: &str, params: &P) -> EnvironmentClientResult<()> - where - P: Serialize + Sync, - { - self.transport - .send(json!({ - "jsonrpc": "2.0", - "method": method, - "params": serde_json::to_value(params)?, - })) - .await - } - - pub async fn next_notification( - &mut self, - ) -> EnvironmentClientResult> { - if let Some(notification) = self.notifications.pop_front() { - return Ok(Some(notification)); - } - - loop { - let Some(message) = self.transport.recv().await? else { - return Ok(None); - }; - if let Some(notification) = parse_notification(&message)? { - return Ok(Some(notification)); - } - } - } - - fn next_request_id(&mut self) -> u64 { - let id = self.next_id; - self.next_id += 1; - id - } -} - -fn parse_notification(message: &Value) -> EnvironmentClientResult> { - if message.get("id").is_some() { - return Ok(None); - } - - let Some(method) = message.get("method").and_then(Value::as_str) else { - return Ok(None); - }; - let params = message.get("params").cloned().unwrap_or(Value::Null); - Ok(Some(JsonRpcNotification { - method: method.to_owned(), - params, - })) -} - -fn parse_error(value: &Value) -> EnvironmentClientResult { - if let Ok(error) = serde_json::from_value::(value.clone()) { - return Ok(error); - } - - let message = value - .get("message") - .and_then(Value::as_str) - .unwrap_or("JSON-RPC error"); - Ok(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - message, - )) -} diff --git a/crates/environment-client/src/transport.rs b/crates/environment-client/src/transport.rs deleted file mode 100644 index 80418bca5..000000000 --- a/crates/environment-client/src/transport.rs +++ /dev/null @@ -1,114 +0,0 @@ -//! Transport implementations for environment protocol clients. - -use async_trait::async_trait; -use futures_util::{SinkExt, StreamExt}; -use serde_json::Value; -use tokio::net::TcpStream; -use tokio_tungstenite::{ - MaybeTlsStream, WebSocketStream, connect_async, - tungstenite::{ - Message, - client::IntoClientRequest, - http::{ - HeaderName, HeaderValue, - header::{AUTHORIZATION, USER_AGENT}, - }, - }, -}; - -use crate::{ - error::{EnvironmentClientError, EnvironmentClientResult}, - rpc::JsonRpcTransport, -}; - -pub struct WebSocketTransport { - stream: WebSocketStream>, - closed: bool, -} - -impl WebSocketTransport { - pub async fn connect( - endpoint: &str, - options: WebSocketConnectOptions, - ) -> EnvironmentClientResult { - let mut request = endpoint - .into_client_request() - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - - if let Some(user_agent) = options.user_agent { - let value = HeaderValue::from_str(&user_agent) - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - request.headers_mut().insert(USER_AGENT, value); - } - - if let Some(token) = options.bearer_token { - let value = HeaderValue::from_str(&format!("Bearer {token}")) - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - request.headers_mut().insert(AUTHORIZATION, value); - } - - for (name, value) in options.headers { - let name = HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - let value = HeaderValue::from_str(&value) - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - request.headers_mut().insert(name, value); - } - - let (stream, _) = connect_async(request) - .await - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - Ok(Self { - stream, - closed: false, - }) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct WebSocketConnectOptions { - pub bearer_token: Option, - pub user_agent: Option, - pub headers: Vec<(String, String)>, -} - -#[async_trait] -impl JsonRpcTransport for WebSocketTransport { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()> { - self.stream - .send(Message::Text(message.to_string().into())) - .await - .map_err(|error| EnvironmentClientError::Transport(error.to_string())) - } - - async fn recv(&mut self) -> EnvironmentClientResult> { - loop { - let Some(message) = self.stream.next().await else { - return Ok(None); - }; - let message = - message.map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - match message { - Message::Text(text) => { - return serde_json::from_str(&text).map(Some).map_err(Into::into); - } - Message::Binary(bytes) => { - return serde_json::from_slice(&bytes).map(Some).map_err(Into::into); - } - Message::Close(_) => return Ok(None), - Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => {} - } - } - } - - async fn close(&mut self) -> EnvironmentClientResult<()> { - if self.closed { - return Ok(()); - } - self.closed = true; - self.stream - .close(None) - .await - .map_err(|error| EnvironmentClientError::Transport(error.to_string())) - } -} diff --git a/crates/environment-client/tests/client.rs b/crates/environment-client/tests/client.rs index 0c8d27424..d03b08f77 100644 --- a/crates/environment-client/tests/client.rs +++ b/crates/environment-client/tests/client.rs @@ -1,395 +1,644 @@ -use std::collections::{BTreeMap, VecDeque}; +//! Socket behaviour of the data and controller clients against scripted +//! WebSocket servers. + +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; -use async_trait::async_trait; use environment_client::{ - EnvironmentClientError, EnvironmentClientResult, EnvironmentDataClient, - EnvironmentProviderClient, JsonRpcTransport, WebSocketConnectOptions, + ClientError, ConnectOptions, ControllerClient, DataClient, + conformance::controller::{ControllerConformance, run}, }; use environment_protocol::{ - control::{ - handshake::ControllerInitializeParams, - targets::{AdoptTargetParams, ProviderBindingContext}, - }, + CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION, + bytes::ByteChunk, + control::{handshake::ControllerInitializeResponse, target::TargetOwner}, data::{ - fs::ReadFileParams, - jobs::{JobDependencyPolicy, JobStartSpec, ListJobsParams, StartJobsParams}, - methods::PROCESS_OUTPUT_METHOD, + exec::ReturnOn, + fs::{WriteCondition, WriteFileParams}, + handshake::InitializeResponse, + idle::{GetIdleParams, GetIdleResponse}, + process::ReadProcessParams, }, - error::EnvironmentProtocolErrorCode, - shared::{ByteChunk, CURRENT_PROTOCOL_VERSION, EnvironmentPath, JobId}, + error::{ProtocolError, ProtocolErrorKind}, + handshake::{ImplementationInfo, Plane}, + ids::{BindingId, EnvironmentId, IncarnationId, ProcessId, TemplateId, UniverseId}, + jsonrpc::{self, CancelRequestParams, Id, Message as RpcMessage, Request, Response}, + limits::MAX_MESSAGE_BYTES, + manifest::{CancelRequest, EnvGetIdle}, + paths::EnvironmentPath, }; +use futures_util::{SinkExt, StreamExt}; use serde_json::{Value, json}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::{TcpListener, UnixListener}, + sync::mpsc, +}; +use tokio_tungstenite::{ + WebSocketStream, accept_async, accept_hdr_async, + tungstenite::{ + Message, + handshake::server::{ + ErrorResponse, Request as UpgradeRequest, Response as UpgradeResponse, + }, + http::StatusCode, + }, +}; -#[tokio::test(flavor = "current_thread")] -async fn websocket_client_close_sends_a_close_frame() { - use futures_util::StreamExt as _; - use tokio::net::TcpListener; - use tokio_tungstenite::{accept_async, tungstenite::Message}; +fn fixture(path: &str) -> Value { + let text = match path { + "data" => include_str!("../../environment-protocol/fixtures/data/initialize.json"), + "controller" => { + include_str!("../../environment-protocol/fixtures/controller/initialize.json") + } + other => panic!("no fixture {other}"), + }; + serde_json::from_str::(text).expect("fixture")["response"].clone() +} + +fn options() -> ConnectOptions { + ConnectOptions::new( + ImplementationInfo::new("lightspeed-client-test") + .and_then(|info| info.with_version("0.0.0")) + .expect("client info"), + ) +} +/// Serve one WebSocket connection on a fresh loopback port with `serve`. +async fn server(serve: F) -> String +where + F: FnOnce(WebSocketStream) -> Fut + Send + 'static, + Fut: Future + Send, +{ let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); - let address = listener.local_addr().expect("listener address"); - let server = tokio::spawn(async move { + let url = format!("ws://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { let (stream, _) = listener.accept().await.expect("accept"); - let mut websocket = accept_async(stream).await.expect("websocket handshake"); - while let Some(message) = websocket.next().await { - if matches!(message.expect("websocket message"), Message::Close(_)) { - return true; - } - } - false + serve(accept_async(stream).await.expect("upgrade")).await; }); + url +} - let mut client = EnvironmentDataClient::connect( - &format!("ws://{address}"), - WebSocketConnectOptions::default(), - ) - .await - .expect("connect client"); - client.close().await.expect("close client"); +/// The next frame that is a request, skipping pings. +async fn next_request(socket: &mut WebSocketStream) -> Option +where + S: AsyncRead + AsyncWrite + Unpin, +{ + loop { + match socket.next().await?.ok()? { + Message::Text(text) => match jsonrpc::parse(&text).expect("valid frame") { + RpcMessage::Request(request) => return Some(request), + RpcMessage::Notification(_) | RpcMessage::Response(_) => {} + }, + Message::Close(_) => return None, + _ => {} + } + } +} - let saw_close = tokio::time::timeout(std::time::Duration::from_secs(2), server) +async fn send(socket: &mut WebSocketStream, response: Response) +where + S: AsyncRead + AsyncWrite + Unpin, +{ + socket + .send(Message::text(response.to_text())) .await - .expect("server observed close before timeout") - .expect("server task"); - assert!(saw_close, "client dropped without a WebSocket close frame"); + .expect("send response"); } -#[derive(Default)] -struct MockTransport { - sent: Vec, - recv: VecDeque, +/// Answer the first request, `initialize`, with `result`. +async fn answer_initialize(socket: &mut WebSocketStream, result: Value) +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let request = next_request(socket).await.expect("initialize"); + assert_eq!(request.method, "initialize"); + assert_eq!( + request.params["protocolVersion"], + json!(ENVD_PROTOCOL_VERSION) + ); + send(socket, Response::success(request.id, &result)).await; } -impl MockTransport { - fn with_recv(messages: impl IntoIterator) -> Self { - Self { - sent: Vec::new(), - recv: messages.into_iter().collect(), - } +fn idle(ms: u64) -> GetIdleResponse { + GetIdleResponse { + idle_for_ms: ms, + ..GetIdleResponse::default() } } -#[async_trait] -impl JsonRpcTransport for MockTransport { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()> { - self.sent.push(message); - Ok(()) - } - - async fn recv(&mut self) -> EnvironmentClientResult> { - Ok(self.recv.pop_front()) +fn read_params() -> ReadProcessParams { + ReadProcessParams { + process_id: ProcessId::new("p1").expect("process id"), + after: None, + tail_bytes: None, + ack: None, + max_bytes: 1024, + wait_ms: 60_000, + return_on: ReturnOn::AnyOutput, } } -#[tokio::test] -async fn data_client_sends_typed_request_and_decodes_response() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "data": "aGk=" +#[tokio::test(flavor = "current_thread")] +async fn connect_initialized_returns_the_handshake_and_close_sends_a_close_frame() { + let (saw_close, mut closed) = mpsc::unbounded_channel(); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + while let Some(Ok(message)) = socket.next().await { + if matches!(message, Message::Close(_)) { + let _ = saw_close.send(()); + return; + } } - })]); - let mut client = EnvironmentDataClient::new(transport); - - let response = client - .read_file(&ReadFileParams { - path: EnvironmentPath::new("README.md").expect("path"), - offset: None, - max_bytes: None, - }) + }) + .await; + let (client, initialize) = DataClient::connect_initialized(&url, options()) .await - .expect("response"); - - assert_eq!(response.data, ByteChunk::from(b"hi".as_slice())); + .expect("connect"); + let expected: InitializeResponse = serde_json::from_value(fixture("data")).expect("fixture"); + assert_eq!(initialize, expected); + assert_eq!(client.initialize_response(), &expected); + client.close().await.expect("close"); + assert!(client.is_closed()); + tokio::time::timeout(Duration::from_secs(5), closed.recv()) + .await + .expect("server saw the close frame in time") + .expect("close frame"); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::Closed { .. }) + )); +} - let transport = client.into_rpc().into_inner(); - assert_eq!( - transport.sent, - vec![json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "fs/readFile", - "params": { - "path": "README.md" +#[tokio::test(flavor = "current_thread")] +async fn a_version_mismatch_answer_is_typed() { + let url = server(|mut socket| async move { + let request = next_request(&mut socket).await.expect("initialize"); + let received = request.params["protocolVersion"].as_u64().unwrap() as u32; + send( + &mut socket, + Response::failure( + Some(request.id), + ProtocolError::version_mismatch(2, received), + ), + ) + .await; + while socket.next().await.is_some() {} + }) + .await; + let error = DataClient::connect_initialized(&url, options()) + .await + .expect_err("mismatch"); + assert!( + matches!( + error, + ClientError::VersionMismatch { + plane: Plane::Data, + local: ENVD_PROTOCOL_VERSION, + remote: 2 } - })] + ), + "{error:?}" ); + assert_eq!( + error.protocol_kind(), + Some(ProtocolErrorKind::VersionMismatch) + ); + assert!(!error.is_retryable()); } -#[tokio::test] -async fn data_client_sends_typed_job_start_request() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "jobs": [ - { - "createdAtMs": 1, - "jobId": "job-1", - "namespace": "session_1", - "status": "queued" - } - ] - } - })]); - let mut client = EnvironmentDataClient::new(transport); +#[tokio::test(flavor = "current_thread")] +async fn a_response_with_another_version_is_a_mismatch() { + let mut response = fixture("data"); + response["protocolVersion"] = json!(ENVD_PROTOCOL_VERSION + 1); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, response).await; + while socket.next().await.is_some() {} + }) + .await; + assert!(matches!( + DataClient::connect_initialized(&url, options()).await, + Err(ClientError::VersionMismatch { remote, .. }) if remote == ENVD_PROTOCOL_VERSION + 1 + )); +} - let response = client - .start_jobs(&StartJobsParams { - namespace: "session_1".to_owned(), - request_id: "request-1".to_owned(), - jobs: vec![JobStartSpec { - job_id: JobId::new("job-1"), - name: None, - argv: vec![ - "/bin/sh".to_owned(), - "-c".to_owned(), - "printf ok".to_owned(), - ], - cwd: None, - env: Default::default(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(1_000), - depends_on: Vec::new(), - dependency_policy: JobDependencyPolicy::AllSucceeded, - queue_key: None, - }], +#[tokio::test(flavor = "current_thread")] +async fn a_controller_answering_a_data_client_is_the_wrong_plane() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("controller")).await; + while socket.next().await.is_some() {} + }) + .await; + assert!(matches!( + DataClient::connect_initialized(&url, options()).await, + Err(ClientError::WrongPlane { + expected: Plane::Data, + actual: Plane::Controller }) - .await - .expect("response"); - - assert_eq!(response.jobs[0].job_id.as_str(), "job-1"); + )); +} - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "job/start"); - assert_eq!(transport.sent[0]["params"]["jobs"][0]["jobId"], "job-1"); - assert_eq!(transport.sent[0]["params"]["jobs"][0]["argv"][0], "/bin/sh"); +#[tokio::test(flavor = "current_thread")] +async fn controller_clients_check_their_own_plane_and_version() { + let url = server(|mut socket| async move { + let request = next_request(&mut socket).await.expect("initialize"); + assert_eq!( + request.params["protocolVersion"], + json!(CONTROLLER_PROTOCOL_VERSION) + ); + send( + &mut socket, + Response::success(request.id, &fixture("controller")), + ) + .await; + while socket.next().await.is_some() {} + }) + .await; + let (client, initialize) = ControllerClient::connect_initialized(&url, options()) + .await + .expect("connect"); + let expected: ControllerInitializeResponse = + serde_json::from_value(fixture("controller")).expect("fixture"); + assert_eq!(initialize, expected); + assert!(initialize.capabilities.ingress); + client.close().await.expect("close"); } -#[tokio::test] -async fn data_client_sends_typed_job_list_request() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "jobs": [ - { - "createdAtMs": 2, - "jobId": "job-2", - "namespace": "session_1", - "status": "running" - } - ] +#[tokio::test(flavor = "current_thread")] +async fn pipelined_calls_are_matched_by_id() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let first = next_request(&mut socket).await.expect("first"); + let second = next_request(&mut socket).await.expect("second"); + // Answer in reverse order; each result names its own id. + for request in [second, first] { + let Id::Number(id) = request.id else { + panic!("numeric ids") + }; + send(&mut socket, Response::success(request.id, &idle(id as u64))).await; } - })]); - let mut client = EnvironmentDataClient::new(transport); - - let response = client - .list_jobs(&ListJobsParams { - namespace: "session_1".to_owned(), - limit: Some(5), - }) + while socket.next().await.is_some() {} + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) .await - .expect("response"); - - assert_eq!(response.jobs[0].job_id.as_str(), "job-2"); - - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "job/list"); - assert_eq!(transport.sent[0]["params"]["namespace"], "session_1"); - assert_eq!(transport.sent[0]["params"]["limit"], 5); + .expect("connect"); + let (first, second) = tokio::join!( + client.call::(&GetIdleParams {}), + client.get_idle(&GetIdleParams {}), + ); + let (first, second) = (first.expect("first"), second.expect("second")); + assert_ne!(first.idle_for_ms, second.idle_for_ms); + assert!( + first.idle_for_ms < second.idle_for_ms, + "first sent, first id" + ); } -#[tokio::test] -async fn data_client_stashes_notifications_seen_while_waiting_for_response() { - let transport = MockTransport::with_recv([ - json!({ - "jsonrpc": "2.0", - "method": PROCESS_OUTPUT_METHOD, - "params": { - "chunk": "b2sK", - "processId": "proc-1", - "seq": 1, - "stream": "stdout" - } - }), - json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "data": "aGk=" - } - }), - ]); - let mut client = EnvironmentDataClient::new(transport); - - client - .read_file(&ReadFileParams { - path: EnvironmentPath::new("README.md").expect("path"), - offset: None, - max_bytes: None, - }) +#[tokio::test(flavor = "current_thread")] +async fn a_server_notification_fails_pending_calls_and_closes() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let _ = next_request(&mut socket).await.expect("request"); + socket + .send(Message::text( + r#"{"jsonrpc":"2.0","method":"process/output","params":{}}"#, + )) + .await + .expect("send notification"); + while socket.next().await.is_some() {} + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) .await - .expect("response"); + .expect("connect"); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::InvalidMessage(_)) + )); + assert!(client.is_closed()); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::InvalidMessage(_)) + )); +} - let notification = client - .next_notification() +#[tokio::test(flavor = "current_thread")] +async fn a_binary_frame_is_a_protocol_violation() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let _ = next_request(&mut socket).await.expect("request"); + socket + .send(Message::binary(b"{}".to_vec())) + .await + .expect("send binary"); + while socket.next().await.is_some() {} + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) .await - .expect("notification read") - .expect("notification"); - assert_eq!(notification.method, PROCESS_OUTPUT_METHOD); - assert_eq!(notification.params["processId"], "proc-1"); + .expect("connect"); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::InvalidMessage(_)) + )); } -#[tokio::test] -async fn data_client_maps_protocol_error_payloads() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "error": { - "code": "notFound", - "message": "missing" +#[tokio::test(flavor = "current_thread")] +async fn a_deadline_cancels_an_abortable_call() { + let (cancelled, mut cancels) = mpsc::unbounded_channel(); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let read = next_request(&mut socket).await.expect("process/read"); + assert_eq!(read.method, "process/read"); + while let Some(Ok(message)) = socket.next().await { + if let Message::Text(text) = message + && let Ok(RpcMessage::Notification(notification)) = jsonrpc::parse(&text) + { + assert_eq!(notification.method, CancelRequest::NAME); + let params: CancelRequestParams = notification.params().expect("params"); + let _ = cancelled.send((params.id, read.id.clone())); + // The late answer is discarded by the client. + send( + &mut socket, + Response::failure(Some(read.id.clone()), ProtocolError::cancelled()), + ) + .await; + } } - })]); - let mut client = EnvironmentDataClient::new(transport); - + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) + .await + .expect("connect"); let error = client - .read_file(&ReadFileParams { - path: EnvironmentPath::new("missing.txt").expect("path"), - offset: None, - max_bytes: None, - }) + .call_with_timeout::( + &read_params(), + Duration::from_millis(50), + ) .await - .expect_err("environment protocol error"); - - match error { - EnvironmentClientError::Protocol(error) => { - assert_eq!(error.code, EnvironmentProtocolErrorCode::NotFound) - } - other => panic!("unexpected error {other:?}"), - } + .expect_err("deadline"); + assert!( + matches!(error, ClientError::DeadlineExceeded { .. }), + "{error:?}" + ); + let (cancelled_id, read_id) = tokio::time::timeout(Duration::from_secs(5), cancels.recv()) + .await + .expect("cancel in time") + .expect("cancel"); + assert_eq!(cancelled_id, read_id); + assert!( + !client.is_closed(), + "a deadline leaves the connection usable" + ); } -#[tokio::test] -async fn controller_client_sends_typed_initialize_request() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "capabilities": { - "closeTarget": true, - "createTarget": true, - "getTarget": true, - "listTargets": true - }, - "implementation": { - "name": "test-controller", - "version": "0.1.0" - }, - "protocolVersion": 2 +#[tokio::test(flavor = "current_thread")] +async fn a_dropped_abortable_call_is_cancelled() { + let (cancelled, mut cancels) = mpsc::unbounded_channel(); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let read = next_request(&mut socket).await.expect("process/read"); + while let Some(Ok(message)) = socket.next().await { + if let Message::Text(text) = message + && let Ok(RpcMessage::Notification(notification)) = jsonrpc::parse(&text) + { + let params: CancelRequestParams = notification.params().expect("params"); + let _ = cancelled.send(params.id == read.id); + } } - })]); - let mut client = EnvironmentProviderClient::new(transport); - - let response = client - .initialize(&ControllerInitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-test".to_owned(), - }) + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) .await - .expect("response"); - - assert!(response.capabilities.create_target); - assert_eq!(response.implementation.name, "test-controller"); - - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "controller/initialize"); - assert_eq!(transport.sent[0]["params"]["clientName"], "lightspeed-test"); + .expect("connect"); + let params = read_params(); + let read = client.read_process(¶ms); + assert!( + tokio::time::timeout(Duration::from_millis(50), read) + .await + .is_err(), + "the read waits" + ); + let matched = tokio::time::timeout(Duration::from_secs(5), cancels.recv()) + .await + .expect("cancel in time") + .expect("cancel"); + assert!(matched, "the cancel names the dropped request"); } -#[tokio::test] -async fn controller_client_sends_typed_adoption_request() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "target": { - "targetId": "target-adopted", - "status": "starting", - "scope": { "type": "default" }, - "capabilities": {} +#[tokio::test(flavor = "current_thread")] +async fn pending_calls_keep_the_connection_alive_with_pings() { + let pings = Arc::new(AtomicUsize::new(0)); + let counted = pings.clone(); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let request = next_request(&mut socket).await.expect("request"); + let quiet = tokio::time::sleep(Duration::from_millis(300)); + tokio::pin!(quiet); + loop { + tokio::select! { + _ = &mut quiet => break, + message = socket.next() => match message { + Some(Ok(Message::Ping(_))) => { counted.fetch_add(1, Ordering::SeqCst); } + Some(Ok(_)) => {} + _ => return, + }, } } - })]); - let mut client = EnvironmentProviderClient::new(transport); + send(&mut socket, Response::success(request.id, &idle(0))).await; + while socket.next().await.is_some() {} + }) + .await; + let mut options = options(); + options.keepalive_interval = Duration::from_millis(40); + let (client, _) = DataClient::connect_initialized(&url, options) + .await + .expect("connect"); + client.get_idle(&GetIdleParams {}).await.expect("idle"); + assert!( + pings.load(Ordering::SeqCst) >= 3, + "pinged {} times", + pings.load(Ordering::SeqCst) + ); +} - let response = client - .adopt_target(&AdoptTargetParams { - request_id: "adopt-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-1".to_owned(), - binding_id: "primary".to_owned(), - }, - source_target: "legacy/hand-built-vm".to_owned(), +#[tokio::test(flavor = "current_thread")] +async fn the_default_request_timeout_applies_to_every_call() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + while socket.next().await.is_some() {} + }) + .await; + let (client, _) = DataClient::connect_initialized( + &url, + options().with_request_timeout(Duration::from_millis(50)), + ) + .await + .expect("connect"); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::DeadlineExceeded { .. }) + )); +} + +#[tokio::test(flavor = "current_thread")] +async fn oversized_requests_fail_without_being_sent() { + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let client = DataClient::in_process( + serde_json::from_value(fixture("data")).expect("fixture"), + move |request: Request| { + seen.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { Ok(Response::success(request.id, &json!({}))) }) + }, + ); + let error = client + .write_file(&WriteFileParams { + path: EnvironmentPath::new("/workspace/big").expect("path"), + data: ByteChunk::new(vec![0_u8; MAX_MESSAGE_BYTES as usize]), + create: WriteCondition::default(), + if_match: None, + executable: false, }) .await - .expect("response"); - assert_eq!(response.target.target_id.as_str(), "target-adopted"); + .expect_err("too large"); + assert!(matches!( + error, + ClientError::PayloadTooLarge { limit_bytes: MAX_MESSAGE_BYTES, actual_bytes } + if actual_bytes > MAX_MESSAGE_BYTES + )); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "controller/adoptTarget"); +#[tokio::test(flavor = "current_thread")] +async fn in_process_clients_answer_through_the_handler() { + let client = DataClient::in_process( + serde_json::from_value(fixture("data")).expect("fixture"), + |request: Request| { + Box::pin(async move { + assert_eq!(request.method, "env/getIdle"); + Ok(Response::success(request.id, &idle(7))) + }) + }, + ); assert_eq!( - transport.sent[0]["params"]["sourceTarget"], - "legacy/hand-built-vm" + client.get_idle(&GetIdleParams {}).await.expect("idle"), + idle(7) ); } +#[allow(clippy::result_large_err)] +fn refuse_upgrade( + _: &UpgradeRequest, + _: UpgradeResponse, +) -> Result { + let mut response = ErrorResponse::new(None); + *response.status_mut() = StatusCode::UNAUTHORIZED; + Err(response) +} + #[tokio::test(flavor = "current_thread")] -async fn data_client_transfer_methods_preserve_binary_payloads() { - use environment_protocol::data::transfer::*; - let entries = vec![TransferEntry { - path: "".into(), - content: TransferContent::File { - data: ByteChunk::from(vec![0, 255, 128]), - executable: true, - }, - }]; - let limits = TransferLimits { - max_entries: 1, - max_depth: 0, - max_file_bytes: 3, - max_total_bytes: 3, - max_duration_ms: 1000, - }; - let transport = MockTransport::with_recv([ - json!({"jsonrpc":"2.0", "id":1, "result": {"destination":"/file", "entries":1, "bytes":3, "retiredDirectory":null}}), - json!({"jsonrpc":"2.0", "id":2, "result": {"source":"/file", "entries":entries, "bytes":3}}), - ]); - let mut client = EnvironmentDataClient::new(transport); - let request = MaterializeParams { - destination: EnvironmentPath::new("/file").unwrap(), - entries: entries.clone(), - limits, - on_existing: TransferOnExisting::Error, - }; - assert_eq!(client.materialize(&request).await.unwrap().bytes, 3); - let captured = client - .capture(&CaptureParams { - source: request.destination.clone(), - limits, - }) +async fn an_upgrade_refusal_reports_its_status() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let url = format!("ws://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let _ = accept_hdr_async(stream, refuse_upgrade).await; + }); + let error = DataClient::connect_initialized(&url, options()) .await - .unwrap(); - assert_eq!(captured.entries, entries); - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "fs/materialize"); + .expect_err("refused"); + assert!( + matches!(error, ClientError::UpgradeRejected { status: 401 }), + "{error:?}" + ); + assert!(!error.is_retryable()); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_silent_server_hits_the_connect_deadline() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let url = format!("ws://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + let (_stream, _) = listener.accept().await.expect("accept"); + tokio::time::sleep(Duration::from_secs(30)).await; + }); + let mut options = options(); + options.connect_timeout = Duration::from_millis(100); + assert!(matches!( + DataClient::connect_initialized(&url, options).await, + Err(ClientError::ConnectTimeout { .. }) + )); +} + +#[tokio::test(flavor = "current_thread")] +async fn unix_endpoints_upgrade_over_the_socket() { + let path = std::path::PathBuf::from(format!( + "/tmp/lightspeed-client-{}-{}.sock", + std::process::id(), + line!() + )); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).expect("bind unix socket"); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let mut socket = accept_async(stream).await.expect("upgrade"); + answer_initialize(&mut socket, fixture("data")).await; + while socket.next().await.is_some() {} + }); + let endpoint = format!("unix://{}", path.display()); + let (client, _) = DataClient::connect_initialized(&endpoint, options()) + .await + .expect("connect over unix"); + client.close().await.expect("close"); + let _ = std::fs::remove_file(&path); +} + +#[tokio::test(flavor = "current_thread")] +async fn clients_connect_over_an_existing_byte_stream() { + let (client_end, server_end) = tokio::io::duplex(64 * 1024); + tokio::spawn(async move { + let mut socket = accept_async(server_end).await.expect("upgrade"); + answer_initialize(&mut socket, fixture("data")).await; + let request = next_request(&mut socket).await.expect("request"); + send(&mut socket, Response::success(request.id, &idle(3))).await; + while socket.next().await.is_some() {} + }); + let (client, _) = DataClient::connect_initialized_over(client_end, options()) + .await + .expect("connect over duplex"); assert_eq!( - transport.sent[0]["params"], - serde_json::to_value(request).unwrap() + client.get_idle(&GetIdleParams {}).await.expect("idle"), + idle(3) ); - assert_eq!(transport.sent[1]["method"], "fs/capture"); +} + +#[tokio::test(flavor = "current_thread")] +async fn conformance_reports_an_unreachable_controller() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let url = format!("ws://{}", listener.local_addr().expect("address")); + drop(listener); + let owner = Arc::new(|| TargetOwner { + universe_id: UniverseId::new("u1").unwrap(), + binding_id: BindingId::new("b1").unwrap(), + environment_id: EnvironmentId::new("e1").unwrap(), + incarnation_id: IncarnationId::new("i1").unwrap(), + }); + let report = run(&ControllerConformance::new( + url, + options().with_bearer_token("credential"), + TemplateId::new("t1").unwrap(), + owner, + )) + .await; + assert!(!report.passed()); + assert_eq!(report.checks.len(), 1); + assert_eq!(report.checks[0].name, "initialize"); + assert!(report.into_result().is_err()); } diff --git a/crates/environment-daemon/Cargo.toml b/crates/environment-daemon/Cargo.toml index c415affcf..5f35ededb 100644 --- a/crates/environment-daemon/Cargo.toml +++ b/crates/environment-daemon/Cargo.toml @@ -6,13 +6,12 @@ edition = "2024" [dependencies] hex = "0.4" anyhow = "1" -api = { path = "../api" } clap = { version = "4", features = ["derive", "env"] } dotenvy = "0.15" release-info = { path = "../release-info" } futures-util = "0.3" flate2 = "1" -glob = "0.3" +globset = "0.4" grep-regex = "0.1" grep-searcher = "0.1" environment-protocol = { path = "../environment-protocol" } @@ -25,7 +24,8 @@ sha2 = "0.11" tar = { version = "0.4", default-features = false } tempfile = "3" thiserror = "2" -tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "process", "rt", "signal", "sync", "time"] } +tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "process", "rt", "rt-multi-thread", "signal", "sync", "time"] } +tokio-util = "0.7" tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] } ed25519-dalek = { version = "3", features = ["rand_core"] } rand = "0.10" @@ -40,7 +40,6 @@ rustix = { version = "1", features = ["fs", "process"] } libc = "0.2" [dev-dependencies] -async-trait = "0.1" store-fs = { path = "../store-fs" } vfs = { path = "../vfs" } harness = { path = "../harness" } diff --git a/crates/environment-daemon/src/config.rs b/crates/environment-daemon/src/config.rs index cb3fd0ed9..e9abbe912 100644 --- a/crates/environment-daemon/src/config.rs +++ b/crates/environment-daemon/src/config.rs @@ -9,7 +9,11 @@ use std::{ use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; -use environment_protocol::{registration::validate_registration_metadata, shared::SecretString}; +use environment_protocol::{ + bytes::SecretString, + metadata::{DisplayName, Metadata}, + routes::{CanonicalOrigin, canonical_origin, gateway_stream_url}, +}; /// Listener address when the daemon is not registering outbound. pub const DEFAULT_LISTEN: &str = "127.0.0.1:19091"; @@ -20,6 +24,13 @@ pub const ENV_PREFIX: &str = "LIGHTSPEED_ENVD_"; /// Internal one-exec handoff for a registration key that configuration has /// already read and scrubbed. Never documented as operator configuration. pub(crate) const REEXEC_REGISTRATION_KEY_ENV: &str = "LIGHTSPEED_ENVD_REEXEC_REGISTRATION_KEY"; +/// Set on the re-executed replacement after an automatic upgrade: the +/// gateway protocol version the upgrade targeted. A later challenge naming +/// that same version is refused instead of upgrading again. +pub(crate) const AUTO_UPGRADE_ATTEMPTED_ENV: &str = "LIGHTSPEED_ENVD_AUTO_UPGRADE_ATTEMPTED"; +/// Name of the state directory below `$XDG_STATE_HOME` (or +/// `$HOME/.local/state`). +pub const STATE_DIR_NAME: &str = "lightspeed-envd"; #[derive(Parser, Debug)] #[command( @@ -39,23 +50,25 @@ pub struct DaemonArgs { #[arg(long, env = "LIGHTSPEED_ENVD_LISTEN")] pub listen: Option, - /// Public environment-gateway connect URL to dial outbound - /// (`wss://host/environment-gateway/connect`). + /// Environment-gateway connect URL to dial outbound + /// (`wss://host[/prefix]/environment-gateway/connect`). Stream sockets + /// are dialed on the same origin and prefix. #[arg(long, env = "LIGHTSPEED_ENVD_GATEWAY_URL", global = true)] pub gateway_url: Option, /// Discovery document used for manual or automatic upgrades. Defaults to - /// `/.well-known/lightspeed-envd` on the configured gateway host. + /// `/.well-known/lightspeed-envd` on the configured gateway origin. #[arg(long, env = "LIGHTSPEED_ENVD_DISCOVERY_URL", global = true)] pub discovery_url: Option, - /// Upgrade automatically when an outbound gateway advertises a different - /// environment protocol version. + /// Upgrade automatically when an outbound gateway challenges with a + /// different environment protocol version. #[arg(long, env = "LIGHTSPEED_ENVD_AUTO_UPGRADE", default_value_t = false)] pub auto_upgrade: bool, /// Registration key admitting this daemon as a new environment. Read - /// once and removed from the process environment. + /// once, removed from the process environment, and sent only until the + /// first admission. #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_KEY", hide_env_values = true)] pub registration_key: Option, @@ -63,15 +76,16 @@ pub struct DaemonArgs { #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE")] pub registration_key_file: Option, - /// Display-name hint recorded on the environment at first registration. + /// Display name recorded on the environment at first registration. #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_NAME")] pub registration_name: Option, - /// Bounded JSON object of string correlation metadata. + /// Bounded JSON object of string correlation metadata, recorded at + /// first registration. #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_METADATA")] pub registration_metadata: Option, - /// Where to write the registration receipt JSON once accepted. + /// Where to write the registration receipt JSON once admitted. #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_RECEIPT")] pub registration_receipt: Option, @@ -85,6 +99,10 @@ pub struct DaemonArgs { #[arg(long, env = "LIGHTSPEED_ENVD_FS_ROOT")] pub fs_root: Option, + /// Daemon key, job records and transfer journal. Defaults to + /// `$XDG_STATE_HOME/lightspeed-envd`, else + /// `$HOME/.local/state/lightspeed-envd`. The data plane never addresses + /// it, wherever it lives. #[arg(long, env = "LIGHTSPEED_ENVD_STATE_DIR")] pub state_dir: Option, @@ -155,6 +173,8 @@ pub struct DaemonConfig { pub listen: Option, pub cwd: PathBuf, pub fs_root: PathBuf, + /// Created `0700` when missing. Never addressable through the data + /// plane. pub state_dir: PathBuf, pub read_only_fs: bool, /// Outbound registration, when a gateway URL is configured. @@ -167,11 +187,15 @@ pub struct DaemonConfig { /// it is registration-key policy, and the daemon behaves the same either way. #[derive(Clone)] pub struct RegistrationConfig { + /// The gateway connect URL; it must end with `CONNECT_PATH`. pub gateway_url: String, - /// Present only until the first successful registration consumes it. + /// Sent with `registration/register` until the first admission, then + /// dropped (never handed to an upgraded replacement after that). pub registration_key: Option, - pub display_name: Option, - pub metadata: BTreeMap, + /// Read by the gateway only at first admission. + pub display_name: Option, + /// Read by the gateway only at first admission. + pub metadata: Metadata, pub receipt_path: Option, pub ca_file: Option, /// Absolute discovery URL reported on mismatch and used by automatic @@ -180,6 +204,9 @@ pub struct RegistrationConfig { /// Present only when mismatch-triggered replacement and re-exec is opted /// in. The argument vector is captured before the daemon starts serving. pub(crate) auto_upgrade: Option, + /// The gateway protocol an automatic upgrade already targeted in this + /// process lineage. + pub(crate) upgrade_attempted_for: Option, } #[derive(Clone)] @@ -203,10 +230,22 @@ impl std::fmt::Debug for RegistrationConfig { .field("ca_file", &self.ca_file) .field("discovery_url", &self.discovery_url) .field("auto_upgrade", &self.auto_upgrade.is_some()) + .field("upgrade_attempted_for", &self.upgrade_attempted_for) .finish() } } +/// The URLs one gateway connect URL determines. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GatewayUrls { + /// The connect URL as configured. + pub connect: String, + /// What the registration signature binds. + pub origin: CanonicalOrigin, + /// Where every reverse-dialed stream goes. + pub stream: String, +} + impl RegistrationConfig { /// Registration that never replaces the running binary: no key, name, /// metadata, receipt, or CA file until the caller sets them. The CLI @@ -217,57 +256,66 @@ impl RegistrationConfig { gateway_url, registration_key: None, display_name: None, - metadata: BTreeMap::new(), + metadata: Metadata::default(), receipt_path: None, ca_file: None, discovery_url, auto_upgrade: None, + upgrade_attempted_for: None, } } - pub fn validate(&self) -> Result<()> { - validate_gateway_url(&self.gateway_url)?; - validate_registration_metadata(self.display_name.as_deref(), &self.metadata) - .map_err(|message| anyhow::anyhow!("registration metadata: {message}"))?; + pub fn validate(&self) -> Result { + let urls = gateway_urls(&self.gateway_url)?; if let Some(key) = &self.registration_key && (key.is_empty() || key.expose().chars().any(char::is_whitespace)) { bail!("registration key must be a single non-empty token"); } crate::upgrade::validate_discovery_url(&self.discovery_url)?; - Ok(()) + Ok(urls) } } -/// Plain `ws://` is accepted only toward loopback; everything else must be -/// `wss://` so the registration key never crosses a network in the clear. -pub fn validate_gateway_url(url: &str) -> Result<()> { - let rest = if let Some(rest) = url.strip_prefix("wss://") { - rest - } else if let Some(rest) = url.strip_prefix("ws://") { - let host = rest - .split(['/', '?', '#']) - .next() - .unwrap_or_default() - .rsplit_once(':') - .map(|(host, _)| host) - .unwrap_or(rest.split(['/', '?', '#']).next().unwrap_or_default()); - let host = host.trim_start_matches('[').trim_end_matches(']'); - let loopback = host == "localhost" - || host - .parse::() - .is_ok_and(|ip| ip.is_loopback()); - if !loopback { - bail!("gateway URL must use wss:// except toward loopback: {url}"); +/// Validate a gateway connect URL and derive what it determines. Plain +/// `ws://` is accepted only toward loopback; everything else must be +/// `wss://`, so neither the registration key nor any stream crosses a +/// network in the clear. The path must end with `CONNECT_PATH` (a prefix +/// before it is kept), with no query or fragment. +pub fn gateway_urls(url: &str) -> Result { + let scheme = url.split_once("://").map(|(scheme, _)| scheme); + match scheme { + Some("wss") => {} + Some("ws") => { + let origin = canonical_origin(url)?; + let host = origin + .as_str() + .trim_start_matches("ws://") + .rsplit_once(':') + .filter(|(_, port)| port.bytes().all(|byte| byte.is_ascii_digit())) + .map_or(origin.as_str().trim_start_matches("ws://"), |(host, _)| { + host + }) + .trim_start_matches('[') + .trim_end_matches(']') + .to_owned(); + let loopback = host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()); + if !loopback { + bail!("gateway URL must use wss:// except toward loopback: {url}"); + } } - rest - } else { - bail!("gateway URL must start with wss:// (or ws:// toward loopback): {url}"); - }; - if rest.is_empty() || rest.starts_with('/') { - bail!("gateway URL has no host: {url}"); + _ => bail!("gateway URL must start with wss:// (or ws:// toward loopback): {url}"), } - Ok(()) + let origin = canonical_origin(url).with_context(|| format!("gateway URL {url}"))?; + let stream = gateway_stream_url(url).with_context(|| format!("gateway URL {url}"))?; + Ok(GatewayUrls { + connect: url.to_owned(), + origin, + stream, + }) } /// Names of every daemon configuration variable present in the process @@ -293,9 +341,13 @@ impl DaemonArgs { install_path: started_executable().context("find the started envd executable")?, target: release_info::TARGET.to_owned(), expected_protocol: None, + other_than: None, }) } + /// Resolve the configuration. Must run before the async runtime starts + /// any thread: it removes secret-bearing variables from the process + /// environment. pub fn into_config(self) -> Result { let cwd = canonical_dir( self.cwd @@ -313,71 +365,20 @@ impl DaemonArgs { fs_root.display() ); } - let state_dir = match self.state_dir { - Some(path) if path.is_absolute() => path, - Some(path) => cwd.join(path), - None => cwd.join(".lightspeed-envd"), - }; + let state_dir = resolve_state_dir(self.state_dir, &cwd, |name| std::env::var_os(name))?; let scrubbed_env = scrubbed_env_names(); let registration = match self.gateway_url { - Some(gateway_url) => { - let discovery_url = crate::upgrade::resolve_discovery_url( - Some(&gateway_url), - self.discovery_url.as_deref(), - )?; - let auto_upgrade = self - .auto_upgrade - .then(|| -> Result { - Ok(RestartInvocation { - executable: started_executable() - .context("find the started envd executable")?, - arg0: std::env::args_os() - .next() - .ok_or_else(|| anyhow::anyhow!("process has no argv[0]"))?, - args: std::env::args_os().skip(1).collect(), - }) - }) - .transpose()?; - let reexec_registration_key = std::env::var(REEXEC_REGISTRATION_KEY_ENV).ok(); - let registration_key = if let Some(key) = reexec_registration_key { - Some(SecretString::new(key)) - } else { - match (self.registration_key, self.registration_key_file) { - (Some(_), Some(_)) => { - bail!("set either the registration key or the key file, not both") - } - (Some(key), None) => Some(SecretString::new(key.trim().to_owned())), - (None, Some(path)) => Some(SecretString::new( - std::fs::read_to_string(&path) - .with_context(|| { - format!("read registration key file {}", path.display()) - })? - .trim() - .to_owned(), - )), - (None, None) => None, - } - }; - let metadata = match self.registration_metadata.as_deref() { - Some(json) if !json.trim().is_empty() => { - serde_json::from_str::>(json) - .context("registration metadata must be a JSON object of strings")? - } - _ => BTreeMap::new(), - }; - let registration = RegistrationConfig { - gateway_url, - registration_key, - display_name: self.registration_name.filter(|name| !name.is_empty()), - metadata, - receipt_path: self.registration_receipt, - ca_file: self.ca_file, - discovery_url, - auto_upgrade, - }; - registration.validate()?; - Some(registration) - } + Some(gateway_url) => Some(registration_config( + gateway_url, + self.discovery_url, + self.auto_upgrade, + self.registration_key, + self.registration_key_file, + self.registration_name, + self.registration_metadata, + self.registration_receipt, + self.ca_file, + )?), None => { if self.registration_key.is_some() || self.registration_key_file.is_some() { bail!("a registration key needs LIGHTSPEED_ENVD_GATEWAY_URL to dial"); @@ -406,6 +407,140 @@ impl DaemonArgs { } } +#[allow(clippy::too_many_arguments)] +fn registration_config( + gateway_url: String, + discovery_url: Option, + auto_upgrade: bool, + registration_key: Option, + registration_key_file: Option, + registration_name: Option, + registration_metadata: Option, + registration_receipt: Option, + ca_file: Option, +) -> Result { + let discovery_url = + crate::upgrade::resolve_discovery_url(Some(&gateway_url), discovery_url.as_deref())?; + let auto_upgrade = auto_upgrade + .then(|| -> Result { + Ok(RestartInvocation { + executable: started_executable().context("find the started envd executable")?, + arg0: std::env::args_os() + .next() + .ok_or_else(|| anyhow::anyhow!("process has no argv[0]"))?, + args: std::env::args_os().skip(1).collect(), + }) + }) + .transpose()?; + let upgrade_attempted_for = match std::env::var(AUTO_UPGRADE_ATTEMPTED_ENV) { + Ok(value) => Some(value.trim().parse::().with_context(|| { + format!("{AUTO_UPGRADE_ATTEMPTED_ENV} must be a protocol version: {value}") + })?), + Err(_) => None, + }; + let reexec_registration_key = std::env::var(REEXEC_REGISTRATION_KEY_ENV).ok(); + let registration_key = if let Some(key) = reexec_registration_key { + Some(SecretString::new(key)) + } else { + match (registration_key, registration_key_file) { + (Some(_), Some(_)) => { + bail!("set either the registration key or the key file, not both") + } + (Some(key), None) => Some(SecretString::new(key.trim().to_owned())), + (None, Some(path)) => Some(SecretString::new( + std::fs::read_to_string(&path) + .with_context(|| format!("read registration key file {}", path.display()))? + .trim() + .to_owned(), + )), + (None, None) => None, + } + }; + let metadata = match registration_metadata.as_deref() { + Some(json) if !json.trim().is_empty() => Metadata::new( + serde_json::from_str::>(json) + .context("registration metadata must be a JSON object of strings")?, + ) + .context("registration metadata")?, + _ => Metadata::default(), + }; + let display_name = registration_name + .filter(|name| !name.is_empty()) + .map(DisplayName::new) + .transpose() + .context("registration name")?; + let registration = RegistrationConfig { + gateway_url, + registration_key, + display_name, + metadata, + receipt_path: registration_receipt, + ca_file, + discovery_url, + auto_upgrade, + upgrade_attempted_for, + }; + registration.validate()?; + Ok(registration) +} + +/// The state directory, created `0700` and canonical. A defaulted +/// directory must not lie inside `cwd`, the agent's working tree; an +/// explicitly configured one may (confinement refuses data-plane access to +/// it wherever it lives). A relative explicit path is taken against `cwd`. +fn resolve_state_dir( + configured: Option, + cwd: &Path, + env: impl Fn(&str) -> Option, +) -> Result { + let (path, defaulted) = match configured { + Some(path) if path.is_absolute() => (path, false), + Some(path) => (cwd.join(path), false), + None => (default_state_dir(env)?, true), + }; + let state_dir = prepare_state_dir(&path)?; + if defaulted && state_dir.starts_with(cwd) { + bail!( + "the default state directory {} lies inside cwd {}: set LIGHTSPEED_ENVD_STATE_DIR", + state_dir.display(), + cwd.display() + ); + } + Ok(state_dir) +} + +/// `$XDG_STATE_HOME/lightspeed-envd` when that variable is an absolute +/// path, else `$HOME/.local/state/lightspeed-envd`. +fn default_state_dir(env: impl Fn(&str) -> Option) -> Result { + if let Some(state_home) = env("XDG_STATE_HOME").map(PathBuf::from) + && state_home.is_absolute() + { + return Ok(state_home.join(STATE_DIR_NAME)); + } + match env("HOME").map(PathBuf::from) { + Some(home) if home.is_absolute() => { + Ok(home.join(".local").join("state").join(STATE_DIR_NAME)) + } + _ => bail!( + "neither XDG_STATE_HOME nor HOME names an absolute directory: set LIGHTSPEED_ENVD_STATE_DIR" + ), + } +} + +/// Create the state directory when missing (`0700`; parents with default +/// permissions), restrict it, and return its canonical path. +pub fn prepare_state_dir(path: &Path) -> Result { + std::fs::create_dir_all(path) + .with_context(|| format!("create state dir {}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) + .with_context(|| format!("restrict state dir {}", path.display()))?; + } + canonical_dir(path.to_path_buf(), "state dir") +} + /// Resolve argv[0] once, before serving anything, so replacement and re-exec /// use the path the operator or service manager launched rather than a later /// PATH lookup. Preserve a symlink at that path: the atomic replacement is of @@ -434,17 +569,19 @@ fn started_executable() -> Result { std::env::current_exe().context("fall back to the running executable path") } -/// Drop the secret-bearing variables from this process's environment as soon -/// as they have been read. Children additionally get every daemon variable -/// removed at spawn; this narrows what an in-process bug could leak. +/// Drop the secret-bearing and one-exec variables from this process's +/// environment as soon as they have been read. Children additionally get +/// every daemon variable removed at spawn; this narrows what an in-process +/// bug could leak and what a later re-exec could inherit. fn forget_secret_env() { for name in [ "LIGHTSPEED_ENVD_REGISTRATION_KEY", "LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE", REEXEC_REGISTRATION_KEY_ENV, + AUTO_UPGRADE_ATTEMPTED_ENV, ] { - // SAFETY: called from configuration parsing on the main thread before - // the daemon spawns any thread that reads the environment. + // SAFETY: called from configuration parsing on the main thread + // before the async runtime or any other thread exists. unsafe { std::env::remove_var(name) }; } } @@ -483,7 +620,7 @@ mod tests { registration_metadata: None, registration_receipt: None, ca_file: None, - cwd: Some(temp.path().to_path_buf()), + cwd: Some(temp.path().join("work")), fs_root: Some(temp.path().to_path_buf()), state_dir: Some(temp.path().join("state")), read_only_fs: false, @@ -491,9 +628,15 @@ mod tests { } } + fn temp() -> tempfile::TempDir { + let temp = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir(temp.path().join("work")).expect("work"); + temp + } + #[test] fn listener_config_needs_no_identity_or_secret() { - let temp = tempfile::tempdir().expect("tempdir"); + let temp = temp(); let config = DaemonArgs { listen: Some("127.0.0.1:0".parse().unwrap()), ..args(&temp) @@ -531,7 +674,7 @@ mod tests { #[test] fn outbound_config_reads_the_key_once_and_disables_the_listener() { - let temp = tempfile::tempdir().expect("tempdir"); + let temp = temp(); let key_file = temp.path().join("key"); std::fs::write(&key_file, "lsrk_secret\n").expect("write key"); let config = DaemonArgs { @@ -552,14 +695,11 @@ mod tests { .map(|key| key.expose()), Some("lsrk_secret") ); - assert_eq!(registration.display_name.as_deref(), Some("worker-1")); assert_eq!( - registration - .metadata - .get("harbor.trialId") - .map(String::as_str), - Some("t-1") + registration.display_name.as_ref().map(DisplayName::as_str), + Some("worker-1") ); + assert_eq!(registration.metadata.get("harbor.trialId"), Some("t-1")); assert!(!format!("{registration:?}").contains("lsrk_secret")); let both = DaemonArgs { @@ -588,14 +728,79 @@ mod tests { } #[test] - fn plain_websocket_gateway_urls_are_loopback_only() { - assert!(validate_gateway_url("wss://gateway.example/environment-gateway/connect").is_ok()); - assert!(validate_gateway_url("ws://127.0.0.1:18080/environment-gateway/connect").is_ok()); - assert!(validate_gateway_url("ws://localhost:18080/x").is_ok()); - assert!(validate_gateway_url("ws://[::1]:18080/x").is_ok()); - assert!(validate_gateway_url("ws://gateway.example/x").is_err()); - assert!(validate_gateway_url("http://gateway.example/x").is_err()); - assert!(validate_gateway_url("wss:///x").is_err()); + fn gateway_urls_are_loopback_only_in_the_clear_and_end_with_the_connect_path() { + let urls = gateway_urls("wss://Gateway.example/ls/environment-gateway/connect").unwrap(); + assert_eq!(urls.origin.as_str(), "wss://gateway.example"); + assert_eq!( + urls.stream, + "wss://gateway.example/ls/environment-gateway/data" + ); + for good in [ + "ws://127.0.0.1:18080/environment-gateway/connect", + "ws://localhost:18080/environment-gateway/connect", + "ws://[::1]:18080/environment-gateway/connect", + ] { + assert!(gateway_urls(good).is_ok(), "{good}"); + } + for bad in [ + "ws://gateway.example/environment-gateway/connect", + "http://gateway.example/environment-gateway/connect", + "wss:///environment-gateway/connect", + "wss://gateway.example/x", + "wss://gateway.example/environment-gateway/connect?x=1", + "wss://user@gateway.example/environment-gateway/connect", + ] { + assert!(gateway_urls(bad).is_err(), "{bad}"); + } + } + + #[test] + fn the_default_state_dir_follows_xdg_and_stays_out_of_cwd() { + let env = |pairs: &'static [(&'static str, &'static str)]| { + move |name: &str| { + pairs + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| OsString::from(value)) + } + }; + assert_eq!( + default_state_dir(env(&[("XDG_STATE_HOME", "/xdg"), ("HOME", "/home/a")])).unwrap(), + PathBuf::from("/xdg/lightspeed-envd") + ); + assert_eq!( + default_state_dir(env(&[("XDG_STATE_HOME", "relative"), ("HOME", "/home/a")])).unwrap(), + PathBuf::from("/home/a/.local/state/lightspeed-envd") + ); + assert!(default_state_dir(env(&[])).is_err()); + + let temp = temp(); + let cwd = temp.path().join("work").canonicalize().unwrap(); + let home = cwd.clone(); + let inside = resolve_state_dir(None, &cwd, |name| { + (name == "HOME").then(|| home.clone().into_os_string()) + }); + assert!( + inside.is_err(), + "a defaulted state dir inside cwd is refused" + ); + + let explicit = resolve_state_dir(Some(PathBuf::from(".envd")), &cwd, |_| None) + .expect("an explicit state dir inside cwd is allowed"); + assert_eq!(explicit, cwd.join(".envd")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + let mode = std::fs::metadata(&explicit).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + let outside_home = temp.path().canonicalize().unwrap().join("home"); + let outside = resolve_state_dir(None, &cwd, |name| { + (name == "HOME").then(|| outside_home.clone().into_os_string()) + }) + .expect("a default outside cwd"); + assert!(outside.ends_with(".local/state/lightspeed-envd")); } #[test] @@ -622,11 +827,9 @@ mod tests { #[test] fn outbound_upgrade_configuration_derives_discovery_and_captures_restart() { - let temp = tempfile::tempdir().expect("tempdir"); + let temp = temp(); let config = DaemonArgs { - gateway_url: Some( - "wss://gateway.example/environment-gateway/connect?ignored=1".to_owned(), - ), + gateway_url: Some("wss://gateway.example/ls/environment-gateway/connect".to_owned()), auto_upgrade: true, ..args(&temp) } diff --git a/crates/environment-daemon/src/dispatch.rs b/crates/environment-daemon/src/dispatch.rs new file mode 100644 index 000000000..f9aca14bd --- /dev/null +++ b/crates/environment-daemon/src/dispatch.rs @@ -0,0 +1,278 @@ +//! Data-plane request dispatch, driven by the protocol's method manifest. +//! +//! The connection loop ([`crate::server`]) owns the handshake and hands +//! every request after `initialize` to [`handle_request`]. Dispatch looks +//! the method up in [`manifest`], refuses it when its capability is absent, +//! touches the idle clock for `use` methods, decodes the params strictly +//! and calls the handler. Handlers receive a [`RequestContext`] whose +//! cancellation token fires on `$/cancelRequest` and when the connection +//! closes. + +use std::{future::Future, sync::Arc}; + +use environment_protocol::{ + error::{Capability, ProtocolError}, + handshake::Plane, + jsonrpc, + manifest::{ + self, Activity, DataInitialize, Direction, EnvGetIdle, FsCreateDirectory, FsGetMetadata, + FsGlobFiles, FsReadDirectory, FsReadFile, FsRemove, FsRename, FsScan, FsSearchText, + FsWriteFile, JobCancel, JobRead, JobStart, MessageKind, ProcessRead, ProcessStart, + ProcessTerminate, ProcessWrite, RequestMethod, TransferAbort, TransferAdvance, + TransferAppend, TransferBegin, TransferCommit, TransferGetStatus, TransferListMissing, + TransferRead, TransferReadInventory, TransferWrite, + }, +}; +use serde::Serialize; +use serde_json::Value; +use tokio_util::sync::CancellationToken; + +use crate::DaemonRuntime; + +/// One data connection. +pub struct ConnectionInfo { + /// Daemon-unique, increasing; for logs and per-connection ownership. + pub connection_id: u64, + /// Cancelled when the connection's reader loop ends (peer close, + /// transport error, daemon shutdown). + pub closed: CancellationToken, +} + +/// Passed to every handler. +#[derive(Clone)] +pub struct RequestContext { + pub connection: Arc, + /// Child of `connection.closed`; also cancelled by `$/cancelRequest` + /// naming this request. Abortable handlers select on + /// `cancel.cancelled()` and return `Err(ProtocolError::cancelled())` + /// (the reply is dropped anyway when the connection closed). + pub cancel: CancellationToken, +} + +impl RequestContext { + /// A context for one request on `connection`. + pub fn new(connection: Arc) -> Self { + let cancel = connection.closed.child_token(); + Self { connection, cancel } + } + + /// A context on a connection of its own, for in-process callers and + /// tests. + pub fn detached() -> Self { + Self::new(Arc::new(ConnectionInfo { + connection_id: 0, + closed: CancellationToken::new(), + })) + } +} + +/// Runs once, after the response frame was written to the socket. +pub type WrittenHook = Box; + +/// A successful answer plus an optional hook the writer task runs only +/// after the response frame was written to the socket. +pub struct Reply { + pub value: T, + pub on_written: Option, +} + +impl Reply { + pub fn new(value: T) -> Self { + Self { + value, + on_written: None, + } + } + + pub fn with_commit(value: T, commit: impl FnOnce() + Send + 'static) -> Self { + Self { + value, + on_written: Some(Box::new(commit)), + } + } + + pub fn map(self, map: impl FnOnce(T) -> U) -> Reply { + Reply { + value: map(self.value), + on_written: self.on_written, + } + } +} + +impl std::fmt::Debug for Reply +where + T: std::fmt::Debug, +{ + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("Reply") + .field("value", &self.value) + .field("on_written", &self.on_written.is_some()) + .finish() + } +} + +/// Dispatch one already-initialized data request: manifest lookup +/// (methodNotFound), capability check (capabilityUnavailable), activity +/// touch for `Activity::Use`, strict params decode with +/// `jsonrpc::decode_params` (invalidParams naming the field), handler call, +/// result encode. Public for in-process tests. +pub async fn handle_request( + runtime: &DaemonRuntime, + ctx: &RequestContext, + method: &str, + params: Value, +) -> Result, ProtocolError> { + let spec = manifest::find(Plane::Data, method) + .filter(|spec| { + spec.kind == MessageKind::Request && spec.direction == Direction::ClientToServer + }) + .ok_or_else(|| ProtocolError::method_not_found(method))?; + if let Some(capability) = spec.capability { + runtime.require_capability(capability)?; + } + if spec.activity == Some(Activity::Use) { + runtime.touch_activity(); + } + let params = ¶ms; + let filesystem = runtime.filesystem(); + let processes = runtime.processes(); + let jobs = runtime.jobs(); + match spec.name { + DataInitialize::NAME => Err(ProtocolError::invalid_request( + "initialize was already answered on this connection", + )), + EnvGetIdle::NAME => { + answer::(params, |_| async { Ok(runtime.idle_report().await) }).await + } + FsReadFile::NAME => { + answer::(params, |p| filesystem.read_file(ctx, p)).await + } + FsWriteFile::NAME => { + answer::(params, |p| filesystem.write_file(ctx, p)).await + } + FsCreateDirectory::NAME => { + answer::(params, |p| filesystem.create_directory(ctx, p)).await + } + FsGetMetadata::NAME => { + answer::(params, |p| filesystem.get_metadata(ctx, p)).await + } + FsReadDirectory::NAME => { + answer::(params, |p| filesystem.read_directory(ctx, p)).await + } + FsRemove::NAME => answer::(params, |p| filesystem.remove(ctx, p)).await, + FsRename::NAME => answer::(params, |p| filesystem.rename(ctx, p)).await, + FsSearchText::NAME => { + answer::(params, |p| filesystem.search_text(ctx, p)).await + } + FsGlobFiles::NAME => { + answer::(params, |p| filesystem.glob_files(ctx, p)).await + } + FsScan::NAME => answer::(params, |p| filesystem.scan(ctx, p)).await, + TransferBegin::NAME => { + answer::(params, |p| filesystem.transfer_begin(ctx, p)).await + } + TransferAdvance::NAME => { + answer::(params, |p| filesystem.transfer_advance(ctx, p)).await + } + TransferGetStatus::NAME => { + answer::(params, |p| filesystem.transfer_get_status(ctx, p)) + .await + } + TransferReadInventory::NAME => { + answer::(params, |p| { + filesystem.transfer_read_inventory(ctx, p) + }) + .await + } + TransferAppend::NAME => { + answer::(params, |p| filesystem.transfer_append(ctx, p)).await + } + TransferListMissing::NAME => { + answer::(params, |p| { + filesystem.transfer_list_missing(ctx, p) + }) + .await + } + TransferRead::NAME => { + answer::(params, |p| filesystem.transfer_read(ctx, p)).await + } + TransferWrite::NAME => { + answer::(params, |p| filesystem.transfer_write(ctx, p)).await + } + TransferCommit::NAME => { + answer::(params, |p| filesystem.transfer_commit(ctx, p)).await + } + TransferAbort::NAME => { + answer::(params, |p| filesystem.transfer_abort(ctx, p)).await + } + ProcessStart::NAME => { + answer::(params, |p| processes.start(ctx, p)).await + } + ProcessRead::NAME => reply::(params, |p| processes.read(ctx, p)).await, + ProcessWrite::NAME => { + answer::(params, |p| processes.write(ctx, p)).await + } + ProcessTerminate::NAME => { + answer::(params, |p| processes.terminate(ctx, p)).await + } + JobStart::NAME => answer::(params, |p| jobs.start(ctx, p)).await, + JobRead::NAME => answer::(params, |p| jobs.read(ctx, p)).await, + JobCancel::NAME => answer::(params, |p| jobs.cancel(ctx, p)).await, + _ => Err(ProtocolError::method_not_found(method)), + } +} + +/// Decode, call a handler that answers a plain response, encode. +async fn answer(params: &Value, handler: F) -> Result, ProtocolError> +where + M: RequestMethod, + F: FnOnce(M::Params) -> Fut, + Fut: Future>, +{ + let params = jsonrpc::decode_params::(params)?; + let response = handler(params).await?; + Ok(Reply::new(encode(&response)?)) +} + +/// Decode, call a handler that answers a [`Reply`], encode. +async fn reply(params: &Value, handler: F) -> Result, ProtocolError> +where + M: RequestMethod, + F: FnOnce(M::Params) -> Fut, + Fut: Future, ProtocolError>>, +{ + let params = jsonrpc::decode_params::(params)?; + let reply = handler(params).await?; + let value = encode(&reply.value)?; + Ok(Reply { + value, + on_written: reply.on_written, + }) +} + +fn encode(value: &T) -> Result { + serde_json::to_value(value) + .map_err(|error| ProtocolError::internal(format!("encode result: {error}"))) +} + +/// Whether the daemon's capabilities include `capability`. +pub(crate) fn has_capability( + capabilities: &environment_protocol::data::handshake::DataCapabilities, + capability: Capability, +) -> bool { + match capability { + Capability::Filesystem => capabilities.filesystem.is_some(), + Capability::FilesystemWrite => capabilities + .filesystem + .as_ref() + .is_some_and(|filesystem| filesystem.write), + Capability::Process => capabilities.process.is_some(), + Capability::ProcessPty => capabilities + .process + .as_ref() + .is_some_and(|process| process.pty), + Capability::Job => capabilities.job.is_some(), + Capability::AdoptTarget | Capability::Ingress => false, + } +} diff --git a/crates/environment-daemon/src/exec.rs b/crates/environment-daemon/src/exec.rs new file mode 100644 index 000000000..4c024f29d --- /dev/null +++ b/crates/environment-daemon/src/exec.rs @@ -0,0 +1,323 @@ +//! The execution primitive processes and jobs share. +//! +//! One [`Execution`] is one root process spawned from a +//! [`CommandSpec`]: it runs as the leader of its own process group, its +//! stdout, stderr or PTY output is redacted and captured into a byte-offset +//! [`OutputSink`], its input is fed by a writer of its own, and its exit is +//! observed independently of its pipes. `process/*` and `job/*` differ only +//! in what they do around it: processes keep a daemon cursor and leave +//! leftovers running; jobs persist their output and sweep their group. +//! +//! Start order: validate, spawn, attach the output readers, the exit +//! watcher and the deadline, and only then start the input writer, so no +//! amount of input can block the start or keep a timeout from firing. + +mod execution; +mod input; +pub(crate) mod job_store; +pub(crate) mod log; +pub(crate) mod outcome; +pub(crate) mod spawn_record; + +use std::{collections::BTreeMap, path::PathBuf, process::Stdio}; + +use environment_protocol::{ + bytes::SecretString, + data::exec::{CommandSpec, ExecInput, PtySize}, + error::ProtocolError, +}; +use portable_pty::{CommandBuilder, native_pty_system}; +use tokio::process::Command; + +use crate::{process_group, redaction::redactions_for_secret_env}; + +pub(crate) use execution::{AfterExit, ExecOptions, Execution, OutputSink}; +use input::{Input, InputSink}; + +/// Checks a command before anything runs. `field` is the wire path of the +/// command, such as `command` or `jobs[2].command`. +pub(crate) fn validate_command(spec: &CommandSpec, field: &str) -> Result<(), ProtocolError> { + if spec.argv.is_empty() { + return Err(ProtocolError::invalid_field( + format!("{field}.argv"), + "argv must name a program", + )); + } + if spec.argv[0].is_empty() { + return Err(ProtocolError::invalid_field( + format!("{field}.argv"), + "argv[0] must not be empty", + )); + } + if spec.argv.iter().any(|arg| arg.contains('\0')) { + return Err(ProtocolError::invalid_field( + format!("{field}.argv"), + "arguments must not contain NUL", + )); + } + for (name, value) in &spec.env { + validate_env_name(name, &format!("{field}.env"))?; + if value.contains('\0') { + return Err(ProtocolError::invalid_field( + format!("{field}.env"), + format!("value of {name} contains NUL"), + )); + } + } + for (name, value) in &spec.secret_env { + validate_env_name(name, &format!("{field}.secretEnv"))?; + if value.expose().contains('\0') { + return Err(ProtocolError::invalid_field( + format!("{field}.secretEnv"), + format!("value of {name} contains NUL"), + )); + } + if spec.env.contains_key(name) { + return Err(ProtocolError::invalid_field( + format!("{field}.secretEnv"), + format!("{name} is also set in env"), + )); + } + } + if spec.timeout_ms == Some(0) { + return Err(ProtocolError::invalid_field( + format!("{field}.timeoutMs"), + "timeoutMs must be positive; omit it for no deadline", + )); + } + Ok(()) +} + +fn validate_env_name(name: &str, field: &str) -> Result<(), ProtocolError> { + if name.is_empty() || name.contains('=') || name.contains('\0') { + return Err(ProtocolError::invalid_field( + field, + format!("invalid variable name {name:?}"), + )); + } + Ok(()) +} + +/// A validated command with its working directory resolved, ready to +/// spawn. +pub(crate) struct PreparedCommand { + pub argv: Vec, + pub cwd: PathBuf, + pub env: BTreeMap, + pub secret_env: BTreeMap, + /// Daemon variables no child inherits. + pub scrubbed_env: Vec, + pub input: ExecInput, + pub pty: Option, +} + +impl PreparedCommand { + pub fn new( + spec: CommandSpec, + cwd: PathBuf, + scrubbed_env: &[String], + pty: Option, + ) -> Self { + Self { + argv: spec.argv, + cwd, + env: spec.env, + secret_env: spec.secret_env, + scrubbed_env: scrubbed_env.to_vec(), + input: spec.input, + pty, + } + } + + /// Spawns the root and starts capture, exit watching and the deadline. + /// The input writer starts with [`Execution::start_input`]. + pub fn spawn( + self, + sink: S, + options: ExecOptions, + input_queue_bytes: u64, + ) -> Result, ProtocolError> { + let redactions = redactions_for_secret_env(&self.secret_env); + if let Some(size) = self.pty { + self.spawn_pty(size, sink, options, redactions, input_queue_bytes) + } else { + self.spawn_pipes(sink, options, redactions, input_queue_bytes) + } + } + + fn spawn_pipes( + self, + sink: S, + options: ExecOptions, + redactions: Vec>, + input_queue_bytes: u64, + ) -> Result, ProtocolError> { + let mut command = Command::new(&self.argv[0]); + command + .args(&self.argv[1..]) + .current_dir(&self.cwd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .stdin(match self.input { + ExecInput::None {} => Stdio::null(), + ExecInput::Bytes { .. } | ExecInput::Pipe {} => Stdio::piped(), + }); + for name in &self.scrubbed_env { + command.env_remove(name); + } + command.envs(&self.env); + for (name, value) in &self.secret_env { + command.env(name, value.expose()); + } + process_group::spawn_in_own_group(&mut command); + let mut child = command.spawn().map_err(|error| { + ProtocolError::process_failed(format!("spawn {:?}: {error}", self.argv[0])) + })?; + let pid = child + .id() + .ok_or_else(|| ProtocolError::process_failed("spawned process has no pid"))?; + let input = match (self.input, child.stdin.take()) { + (ExecInput::Bytes { data }, Some(stdin)) => Input::new( + InputSink::Pipe(stdin), + Some(data.into_inner()), + true, + false, + 0, + ), + (ExecInput::Pipe {}, Some(stdin)) => { + Input::new(InputSink::Pipe(stdin), None, false, true, input_queue_bytes) + } + _ => Input::none(), + }; + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + Ok(Execution::start_pipes( + pid, child, stdout, stderr, input, sink, redactions, options, + )) + } + + fn spawn_pty( + self, + size: PtySize, + sink: S, + options: ExecOptions, + redactions: Vec>, + input_queue_bytes: u64, + ) -> Result, ProtocolError> { + let pair = native_pty_system() + .openpty(portable_pty::PtySize { + rows: u16::try_from(size.rows).unwrap_or(u16::MAX), + cols: u16::try_from(size.cols).unwrap_or(u16::MAX), + pixel_width: 0, + pixel_height: 0, + }) + .map_err(|error| ProtocolError::process_failed(format!("open PTY: {error}")))?; + let mut command = CommandBuilder::new(&self.argv[0]); + command.args(&self.argv[1..]); + command.cwd(&self.cwd); + for name in &self.scrubbed_env { + command.env_remove(name); + } + for (name, value) in &self.env { + command.env(name, value); + } + for (name, value) in &self.secret_env { + command.env(name, value.expose()); + } + // Every fallible step on the master comes before the spawn, so a + // started child is never left unsupervised by a failed start. + let master = pair.master; + let reader = master + .try_clone_reader() + .map_err(|error| ProtocolError::process_failed(format!("PTY reader: {error}")))?; + let writer = master + .take_writer() + .map_err(|error| ProtocolError::process_failed(format!("PTY writer: {error}")))?; + let mut child = pair.slave.spawn_command(command).map_err(|error| { + ProtocolError::process_failed(format!("spawn {:?} on a PTY: {error}", self.argv[0])) + })?; + // The child holds the slave now; the reader sees end of file only + // once every holder closed it. + drop(pair.slave); + // The PTY child is a session leader, so its pid is its group id. + let Some(pid) = child.process_id() else { + let _ = child.kill(); + let _ = child.wait(); + return Err(ProtocolError::process_failed("spawned process has no pid")); + }; + let initial = match self.input { + ExecInput::Bytes { data } => Some(data.into_inner()), + ExecInput::None {} | ExecInput::Pipe {} => None, + }; + let input = Input::new( + InputSink::Pty(writer), + initial, + false, + true, + input_queue_bytes, + ); + Ok(Execution::start_pty( + pid, child, master, reader, input, sink, redactions, options, + )) + } +} + +#[cfg(test)] +mod tests { + use environment_protocol::{ + bytes::ByteChunk, + error::{ErrorData, ProtocolErrorKind}, + }; + + use super::*; + + fn spec(argv: &[&str]) -> CommandSpec { + CommandSpec { + argv: argv.iter().map(|arg| (*arg).to_owned()).collect(), + cwd: None, + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input: ExecInput::Bytes { + data: ByteChunk::from(b"x".as_slice()), + }, + timeout_ms: None, + } + } + + fn field(error: &ProtocolError) -> Option<&str> { + match &error.data { + ErrorData::InvalidParams { field, .. } => field.as_deref(), + _ => None, + } + } + + #[test] + fn commands_are_validated_with_wire_field_paths() { + assert!(validate_command(&spec(&["true"]), "command").is_ok()); + let error = validate_command(&spec(&[]), "jobs[1].command").unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + assert_eq!(field(&error), Some("jobs[1].command.argv")); + + let mut collides = spec(&["true"]); + collides.env.insert("TOKEN".to_owned(), "plain".to_owned()); + collides + .secret_env + .insert("TOKEN".to_owned(), SecretString::new("secret")); + let error = validate_command(&collides, "command").unwrap_err(); + assert_eq!(field(&error), Some("command.secretEnv")); + + let mut bad_name = spec(&["true"]); + bad_name.env.insert("A=B".to_owned(), "x".to_owned()); + assert_eq!( + field(&validate_command(&bad_name, "command").unwrap_err()), + Some("command.env") + ); + + let mut zero = spec(&["true"]); + zero.timeout_ms = Some(0); + assert_eq!( + field(&validate_command(&zero, "command").unwrap_err()), + Some("command.timeoutMs") + ); + } +} diff --git a/crates/environment-daemon/src/exec/execution.rs b/crates/environment-daemon/src/exec/execution.rs new file mode 100644 index 000000000..164790833 --- /dev/null +++ b/crates/environment-daemon/src/exec/execution.rs @@ -0,0 +1,551 @@ +//! One running root process: output capture, exit, deadline and signals. + +use std::{ + io::Read as _, + sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; + +use environment_protocol::{ + data::exec::{ExitStatus, LeftoverDisposition, LeftoverReport, OutputStream, StoppedBy}, + error::ProtocolError, +}; +use portable_pty::MasterPty; +use tokio::{ + io::{AsyncRead, AsyncReadExt as _}, + process::{Child, ChildStderr, ChildStdout}, + sync::watch, + task::JoinHandle, +}; + +use super::{input::Input, outcome}; +use crate::{ + process_group::{self, Group, GroupSignal, OUTPUT_DRAIN_GRACE}, + redaction::redact_bytes, +}; + +const READ_BUFFER_BYTES: usize = 8192; + +/// Where captured, redacted output goes. +pub(crate) trait OutputSink: Send + 'static { + fn append(&mut self, stream: OutputStream, data: &[u8]); +} + +/// What happens to the output readers once the root exits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AfterExit { + /// Processes: readers drain for a grace period, then stay on the pipes + /// discarding, so leftovers holding them never see `EPIPE`. + Keep, + /// Jobs: the owner sweeps the group first, then calls + /// [`Execution::settle`]. + Manual, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ExecOptions { + /// `timeoutMs`: stop with `SIGTERM`, the grace, then `SIGKILL`. + pub timeout: Option, + /// Grace between `SIGTERM` and `SIGKILL` for terminate and timeout. + pub terminate_grace: Duration, + pub after_exit: AfterExit, +} + +/// A spawned root process. Cheap to clone. +pub(crate) struct Execution { + inner: Arc>, +} + +impl Clone for Execution { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +struct Inner { + pid: u32, + /// The root's process group, which it leads. + group: Group, + /// Latched once the group was seen empty (or its id reused) after the + /// root exited: nothing of it is signalled or sampled again. + group_gone: AtomicBool, + state: Mutex>, + /// Bumped on every change: output, exit, settlement, stop. + version: watch::Sender, + /// Pipe reader tasks, aborted by a settle that abandons them. + readers: Mutex>>, + /// The PTY master, for the terminal's foreground group. + pty: Option>>, + input: Input, + terminate_grace: Duration, +} + +/// What an execution has observed so far. +pub(crate) struct State { + pub sink: S, + redactions: Vec>, + readers_open: u32, + discarding: bool, + exited: bool, + exit: Option, + error: Option, + stopped_by: Option, + settled_at: Option, +} + +impl State { + /// The root process exited. + pub fn exited(&self) -> bool { + self.exited + } + + /// The root exited and its output settled: every reader reached end of + /// file or was discarded, so the output's end is final. + pub fn settled(&self) -> bool { + self.settled_at.is_some() + } + + pub fn settled_at(&self) -> Option { + self.settled_at + } + + pub fn exit(&self) -> Option<&ExitStatus> { + self.exit.as_ref() + } + + /// Why output capture or the wait failed, if either did. + pub fn error(&self) -> Option<&ProtocolError> { + self.error.as_ref() + } + + pub fn stopped_by(&self) -> Option { + self.stopped_by + } +} + +impl Execution { + #[allow(clippy::too_many_arguments)] + pub(super) fn start_pipes( + pid: u32, + child: Child, + stdout: Option, + stderr: Option, + input: Input, + sink: S, + redactions: Vec>, + options: ExecOptions, + ) -> Self { + let readers_open = u32::from(stdout.is_some()) + u32::from(stderr.is_some()); + let execution = Self::new(pid, None, input, sink, redactions, readers_open, options); + let inner = &execution.inner; + let mut readers = Vec::new(); + if let Some(stdout) = stdout { + readers.push(tokio::spawn(read_pipe( + inner.clone(), + stdout, + OutputStream::Stdout, + ))); + } + if let Some(stderr) = stderr { + readers.push(tokio::spawn(read_pipe( + inner.clone(), + stderr, + OutputStream::Stderr, + ))); + } + *lock(&inner.readers) = readers; + let watched = execution.clone(); + tokio::spawn(async move { + let status = { + let mut child = child; + child.wait().await + }; + watched.observe_exit(status, options.after_exit).await; + }); + execution.start_deadline(options.timeout); + execution + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn start_pty( + pid: u32, + child: Box, + master: Box, + reader: Box, + input: Input, + sink: S, + redactions: Vec>, + options: ExecOptions, + ) -> Self { + let execution = Self::new(pid, Some(master), input, sink, redactions, 1, options); + // PTY reads and waits block. They run on threads of their own, not + // the runtime's blocking pool, so a PTY that leftovers keep open can + // never hold up the daemon's shutdown. + let inner = execution.inner.clone(); + if let Err(error) = std::thread::Builder::new() + .name("envd-pty-reader".to_owned()) + .spawn(move || read_pty(inner, reader)) + { + execution.inner.capture_failed(OutputStream::Pty, error); + execution.inner.reader_closed(); + } + // portable-pty's own exit status reports every signal death as + // code 1; the std child underneath keeps the signal. + let (exited, status) = tokio::sync::oneshot::channel(); + let waiter = match child.into_any().downcast::() { + Ok(mut child) => std::thread::Builder::new() + .name("envd-pty-wait".to_owned()) + .spawn(move || { + let _ = exited.send(child.wait()); + }) + .map(drop), + Err(_) => { + let _ = exited.send(Err(std::io::Error::other( + "the PTY child cannot be waited on", + ))); + Ok(()) + } + }; + let watched = execution.clone(); + tokio::spawn(async move { + let status = match waiter { + Ok(()) => status + .await + .unwrap_or_else(|_| Err(std::io::Error::other("the PTY wait thread ended"))), + Err(error) => Err(error), + }; + watched.observe_exit(status, options.after_exit).await; + }); + execution.start_deadline(options.timeout); + execution + } + + fn new( + pid: u32, + pty: Option>, + input: Input, + sink: S, + redactions: Vec>, + readers_open: u32, + options: ExecOptions, + ) -> Self { + Self { + inner: Arc::new(Inner { + pid, + group: Group::capture(pid), + group_gone: AtomicBool::new(false), + state: Mutex::new(State { + sink, + redactions, + readers_open, + discarding: false, + exited: false, + exit: None, + error: None, + stopped_by: None, + settled_at: None, + }), + version: watch::Sender::new(0), + readers: Mutex::new(Vec::new()), + pty: pty.map(Mutex::new), + input, + terminate_grace: options.terminate_grace, + }), + } + } + + async fn observe_exit( + &self, + status: std::io::Result, + after_exit: AfterExit, + ) { + { + let mut state = self.state(); + state.exited = true; + match status { + Ok(status) => state.exit = Some(outcome::exit_status(status)), + Err(error) => { + state + .error + .get_or_insert(ProtocolError::process_failed(format!( + "wait for process {}: {error}", + self.inner.pid + ))); + } + } + } + self.inner.input.shut(); + self.bump(); + if after_exit == AfterExit::Keep { + self.settle(false).await; + } + } + + fn start_deadline(&self, timeout: Option) { + let Some(timeout) = timeout else { + return; + }; + let execution = Arc::downgrade(&self.inner); + tokio::spawn(async move { + tokio::time::sleep(timeout).await; + if let Some(inner) = execution.upgrade() { + let execution = Execution { inner }; + if !execution.state().exited { + execution.stop(StoppedBy::Timeout); + } + } + }); + } +} + +impl Execution { + /// OS pid of the root, which is also its process group id. + pub fn pid(&self) -> u32 { + self.inner.pid + } + + /// The root's process group. + pub fn group(&self) -> Group { + self.inner.group + } + + /// The root runs, or its group still has members of its own. After the + /// root exited, an emptied group latches as gone, so a later group that + /// reuses the id is never mistaken for this one. + pub fn group_alive(&self) -> bool { + if !self.state().exited { + return true; + } + if self.inner.group_gone.load(Ordering::Acquire) { + return false; + } + let alive = self.inner.group.alive(); + if !alive { + self.inner.group_gone.store(true, Ordering::Release); + } + alive + } + + /// What is alive in the group now, as a report with `disposition`. + /// Absent when nothing is left or the platform cannot enumerate groups. + pub fn sample_leftovers(&self, disposition: LeftoverDisposition) -> Option { + if !self.group_alive() { + return None; + } + outcome::leftover_report(process_group::group_members(self.inner.pid), disposition) + } + + /// Runs `f` on the current state under the execution's lock. + pub fn with_state(&self, f: impl FnOnce(&mut State) -> R) -> R { + f(&mut self.state()) + } + + /// Notified on every change. + pub fn subscribe(&self) -> watch::Receiver { + self.inner.version.subscribe() + } + + /// Starts the input writer: the last step of a start. + pub fn start_input(&self) { + self.inner.input.start(); + } + + /// Queues input; see [`Input::offer`]. + pub fn offer_input(&self, data: &[u8], close: bool) -> Result { + self.inner.input.offer(data, close) + } + + /// Waits until `done` holds. + pub async fn wait_until(&self, done: impl Fn(&State) -> bool) { + let mut changes = self.subscribe(); + loop { + changes.borrow_and_update(); + if done(&self.state()) { + return; + } + if changes.changed().await.is_err() { + return; + } + } + } + + /// Settles the output after the root exited: waits up to the drain + /// grace for the readers to reach end of file, then discards what the + /// rest still read, and with `abort_readers` also abandons the pipes. + pub async fn settle(&self, abort_readers: bool) { + let mut changes = self.subscribe(); + let deadline = tokio::time::Instant::now() + OUTPUT_DRAIN_GRACE; + loop { + changes.borrow_and_update(); + if self.state().readers_open == 0 { + break; + } + tokio::select! { + changed = changes.changed() => if changed.is_err() { break }, + _ = tokio::time::sleep_until(deadline) => break, + } + } + if abort_readers { + for reader in std::mem::take(&mut *lock(&self.inner.readers)) { + reader.abort(); + } + } + { + let mut state = self.state(); + if state.readers_open > 0 { + state.discarding = true; + } + // Nothing is captured any more, so the secret values go. + state.redactions = Vec::new(); + state.settled_at.get_or_insert_with(Instant::now); + } + self.bump(); + } + + /// Stops the group: `kill` at once, `terminate` and `timeout` with + /// `SIGTERM`, the grace, then `SIGKILL`. Records `stoppedBy` only while + /// the root still runs; after its exit the signal still reaches its + /// leftovers. Returns whether the root was running. + pub fn stop(&self, reason: StoppedBy) -> bool { + let running = { + let mut state = self.state(); + let running = !state.exited; + if running && (state.stopped_by.is_none() || reason == StoppedBy::Kill) { + state.stopped_by = Some(reason); + } + running + }; + if running || self.group_alive() { + let group = self.inner.group; + match reason { + StoppedBy::Kill => { + group.signal(GroupSignal::Kill); + } + StoppedBy::Terminate | StoppedBy::Timeout => { + if group.signal(GroupSignal::Terminate) { + tokio::spawn(group.kill_after_grace(self.inner.terminate_grace)); + } + } + } + } + self.bump(); + running + } + + /// `SIGINT` to the foreground group: the terminal's under a PTY, + /// otherwise the process group. Returns whether the root was running. + pub fn interrupt(&self) -> bool { + let running = !self.state().exited; + if !self.group_alive() { + return false; + } + let foreground = self.inner.pty.as_ref().and_then(|master| { + lock(master) + .process_group_leader() + .and_then(|leader| u32::try_from(leader).ok()) + }); + match foreground { + Some(foreground) => process_group::interrupt_foreground(foreground), + None => self.inner.group.signal(GroupSignal::Interrupt), + }; + running + } + + fn state(&self) -> MutexGuard<'_, State> { + lock(&self.inner.state) + } + + fn bump(&self) { + self.inner + .version + .send_modify(|version| *version = version.wrapping_add(1)); + } +} + +impl Inner { + fn push(&self, stream: OutputStream, data: &[u8]) { + { + let mut state = lock(&self.state); + if state.discarding { + return; + } + if state.redactions.is_empty() { + state.sink.append(stream, data); + } else { + let redacted = redact_bytes(data, &state.redactions); + state.sink.append(stream, &redacted); + } + } + self.bump(); + } + + fn capture_failed(&self, stream: OutputStream, error: std::io::Error) { + let mut state = lock(&self.state); + if !state.exited { + state + .error + .get_or_insert(ProtocolError::process_failed(format!( + "capture {stream:?} output: {error}" + ))); + } + } + + fn reader_closed(&self) { + { + let mut state = lock(&self.state); + state.readers_open = state.readers_open.saturating_sub(1); + } + self.bump(); + } + + fn bump(&self) { + self.version + .send_modify(|version| *version = version.wrapping_add(1)); + } +} + +async fn read_pipe( + inner: Arc>, + mut reader: R, + stream: OutputStream, +) { + let mut buffer = vec![0; READ_BUFFER_BYTES]; + loop { + match reader.read(&mut buffer).await { + Ok(0) => break, + Ok(read) => inner.push(stream, &buffer[..read]), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} + Err(error) => { + inner.capture_failed(stream, error); + break; + } + } + } + inner.reader_closed(); +} + +fn read_pty(inner: Arc>, mut reader: Box) { + let mut buffer = vec![0; READ_BUFFER_BYTES]; + loop { + match reader.read(&mut buffer) { + Ok(0) => break, + Ok(read) => inner.push(OutputStream::Pty, &buffer[..read]), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} + // Linux answers EIO once every slave descriptor is closed: the + // PTY's end of file. + Err(_) => break, + } + } + inner.reader_closed(); +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} diff --git a/crates/environment-daemon/src/exec/input.rs b/crates/environment-daemon/src/exec/input.rs new file mode 100644 index 000000000..6c9f44bb2 --- /dev/null +++ b/crates/environment-daemon/src/exec/input.rs @@ -0,0 +1,310 @@ +//! A command's input: a bounded queue drained by its own writer. +//! +//! The writer runs off the runtime thread: a Tokio task for a +//! non-blocking pipe, a thread of its own for a PTY master, whose writes +//! block when the child stops reading. Callers only enqueue, so a +//! child that never reads can fill the queue but never stall the daemon, +//! and signals still reach it while a write is pending. + +use std::{ + io::Write as _, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, +}; + +use environment_protocol::error::{PreconditionReason, ProtocolError}; +use tokio::{io::AsyncWriteExt as _, process::ChildStdin, sync::mpsc}; + +/// `VEOF` in the default terminal settings (Ctrl-D): end of input for a +/// canonical-mode reader at the start of a line. +const PTY_END_OF_INPUT: u8 = 0x04; + +/// Where queued input goes. +pub(crate) enum InputSink { + Pipe(ChildStdin), + Pty(Box), +} + +enum Message { + Data(Vec), + Close, +} + +#[derive(Default)] +struct Shared { + queued: AtomicU64, + /// The input takes no more bytes: closed by the caller, by the child + /// (broken pipe), or because the process exited. + closed: AtomicBool, +} + +/// The input of one execution. +pub(crate) struct Input { + /// `process/write` is served: a pipe or a PTY. + writable: bool, + capacity: u64, + shared: Arc, + sender: Mutex>>, + /// The sink and the queue's receiver until the writer starts. + pending: Mutex)>>, +} + +impl Input { + /// No input at all: `/dev/null`. + pub fn none() -> Self { + let shared = Arc::new(Shared::default()); + shared.closed.store(true, Ordering::Release); + Self { + writable: false, + capacity: 0, + shared, + sender: Mutex::new(None), + pending: Mutex::new(None), + } + } + + /// Input to `sink`. `initial` is queued first whatever the capacity; + /// `close_after_initial` then ends the input. `writable` inputs accept + /// `process/write` up to `capacity` queued bytes. + pub fn new( + sink: InputSink, + initial: Option>, + close_after_initial: bool, + writable: bool, + capacity: u64, + ) -> Self { + let (sender, receiver) = mpsc::unbounded_channel(); + let shared = Arc::new(Shared::default()); + if let Some(initial) = initial.filter(|initial| !initial.is_empty()) { + shared + .queued + .fetch_add(initial.len() as u64, Ordering::AcqRel); + let _ = sender.send(Message::Data(initial)); + } + let sender = if close_after_initial { + let _ = sender.send(Message::Close); + shared.closed.store(true, Ordering::Release); + None + } else { + Some(sender) + }; + Self { + writable, + capacity, + shared, + sender: Mutex::new(sender), + pending: Mutex::new(Some((sink, receiver))), + } + } + + /// Starts the writer. Called once the output readers, the exit watcher + /// and the deadline are in place, so input can never block them. + pub fn start(&self) { + let Some((sink, receiver)) = self.pending.lock().expect("input poisoned").take() else { + return; + }; + let shared = self.shared.clone(); + match sink { + InputSink::Pipe(stdin) => { + tokio::spawn(write_pipe(stdin, receiver, shared)); + } + InputSink::Pty(writer) => { + // A thread of its own, not the blocking pool: a write the + // child never drains must not hold up the daemon's shutdown. + let closed = shared.clone(); + if std::thread::Builder::new() + .name("envd-pty-writer".to_owned()) + .spawn(move || write_pty(writer, receiver, shared)) + .is_err() + { + closed.closed.store(true, Ordering::Release); + } + } + } + } + + /// Queues a prefix of `data` and answers how many bytes were accepted; + /// `close` ends the input once every byte was accepted. + pub fn offer(&self, data: &[u8], close: bool) -> Result { + let mut sender = self.sender.lock().expect("input poisoned"); + let open = sender + .as_ref() + .filter(|_| self.writable && !self.shared.closed.load(Ordering::Acquire)); + let Some(open) = open else { + return Err(ProtocolError::failed_precondition( + PreconditionReason::InputClosed, + if self.writable { + "the process input is closed" + } else { + "the process was started without writable input" + }, + )); + }; + let queued = self.shared.queued.load(Ordering::Acquire); + let free = self.capacity.saturating_sub(queued); + let accepted = (data.len() as u64).min(free); + if accepted > 0 { + self.shared.queued.fetch_add(accepted, Ordering::AcqRel); + if open + .send(Message::Data(data[..accepted as usize].to_vec())) + .is_err() + { + self.shared.queued.fetch_sub(accepted, Ordering::AcqRel); + self.shared.closed.store(true, Ordering::Release); + return Err(ProtocolError::failed_precondition( + PreconditionReason::InputClosed, + "the process input is closed", + )); + } + } + if close && accepted == data.len() as u64 { + let _ = open.send(Message::Close); + self.shared.closed.store(true, Ordering::Release); + *sender = None; + } + Ok(accepted) + } + + /// Ends the input without a close message, because the process exited. + /// A writer stuck on a full PTY stays on its blocking thread until the + /// write fails; nothing waits for it. + pub fn shut(&self) { + self.shared.closed.store(true, Ordering::Release); + self.sender.lock().expect("input poisoned").take(); + self.pending.lock().expect("input poisoned").take(); + } + + /// Bytes queued and not yet written. + #[cfg(test)] + pub fn queued_bytes(&self) -> u64 { + self.shared.queued.load(Ordering::Acquire) + } +} + +async fn write_pipe( + stdin: ChildStdin, + mut receiver: mpsc::UnboundedReceiver, + shared: Arc, +) { + let mut stdin = Some(stdin); + while let Some(message) = receiver.recv().await { + match message { + Message::Data(data) => { + if let Some(writer) = stdin.as_mut() + && writer.write_all(&data).await.is_err() + { + // Broken pipe: the child closed its input, which is its + // right, not a failure of the command. + stdin = None; + shared.closed.store(true, Ordering::Release); + } + shared.queued.fetch_sub(data.len() as u64, Ordering::AcqRel); + } + Message::Close => { + stdin = None; + shared.closed.store(true, Ordering::Release); + } + } + } +} + +fn write_pty( + writer: Box, + mut receiver: mpsc::UnboundedReceiver, + shared: Arc, +) { + let mut writer = Some(writer); + while let Some(message) = receiver.blocking_recv() { + let (data, close) = match message { + Message::Data(data) => (data, false), + Message::Close => (vec![PTY_END_OF_INPUT], true), + }; + if let Some(pty) = writer.as_mut() + && pty.write_all(&data).and_then(|()| pty.flush()).is_err() + { + writer = None; + shared.closed.store(true, Ordering::Release); + } + if close { + shared.closed.store(true, Ordering::Release); + } else { + shared.queued.fetch_sub(data.len() as u64, Ordering::AcqRel); + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use environment_protocol::error::{ErrorData, ProtocolErrorKind}; + + use super::*; + + fn is_input_closed(error: &ProtocolError) -> bool { + matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::InputClosed + } + ) + } + + /// A writer that never makes progress, like a PTY whose child stopped + /// reading. + struct Stuck(std::sync::mpsc::Receiver<()>); + + impl std::io::Write for Stuck { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + let _ = self.0.recv(); + Err(std::io::ErrorKind::BrokenPipe.into()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[tokio::test(flavor = "current_thread")] + async fn a_stuck_writer_fills_the_queue_without_blocking_the_caller() { + let (release, wait) = std::sync::mpsc::channel(); + let input = Input::new( + InputSink::Pty(Box::new(Stuck(wait))), + None, + false, + true, + 1024, + ); + input.start(); + let started = std::time::Instant::now(); + assert_eq!(input.offer(&[b'x'; 4096], false).unwrap(), 1024); + assert_eq!(input.offer(b"more", false).unwrap(), 0, "queue full"); + assert!(started.elapsed() < Duration::from_secs(1)); + // The runtime thread is free: a timer still fires. + tokio::time::sleep(Duration::from_millis(10)).await; + drop(release); + for _ in 0..200 { + if input.offer(b"x", false).is_err() { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + let error = input.offer(b"x", false).unwrap_err(); + assert!(is_input_closed(&error), "a broken pipe closes the input"); + } + + #[tokio::test(flavor = "current_thread")] + async fn close_applies_only_when_every_byte_was_accepted() { + let (_release, wait) = std::sync::mpsc::channel(); + let input = Input::new(InputSink::Pty(Box::new(Stuck(wait))), None, false, true, 4); + assert_eq!(input.offer(b"abcdef", true).unwrap(), 4); + assert_eq!(input.queued_bytes(), 4); + assert_eq!(input.offer(b"", false).unwrap(), 0, "still open"); + let none = Input::none(); + let error = none.offer(b"x", false).unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::FailedPrecondition); + assert!(is_input_closed(&error)); + } +} diff --git a/crates/environment-daemon/src/exec/job_store.rs b/crates/environment-daemon/src/exec/job_store.rs new file mode 100644 index 000000000..1e52aabb8 --- /dev/null +++ b/crates/environment-daemon/src/exec/job_store.rs @@ -0,0 +1,613 @@ +//! Job records and output on disk, written by one thread of their own. +//! +//! Each job has a directory `//` with `record.json`, its +//! summary (rewritten only on state transitions), and its output as +//! append-only framed logs: `output.head` holds the bytes below the head +//! cap, `output.tail` the bytes after it. The tail log is compacted to the +//! newest tail-cap bytes once it holds twice that, so a verbose job writes +//! O(output) bytes in total and never more than the cap plus slack stays on +//! disk. Writes are buffered and flushed on an interval and when the job +//! finishes; nothing here runs on the runtime thread. +//! +//! Output waits for the store thread in a per-job pending buffer, not in +//! the command queue: of the bytes past the head cap only the newest +//! tail-cap bytes are kept there, so a job that produces faster than the +//! disk takes it holds at most its retained output in memory, and the +//! bytes dropped are ones the tail compaction would drop anyway. + +use std::{ + collections::BTreeMap, + fs::{File, OpenOptions}, + io::{BufWriter, Read as _, Write as _}, + path::{Path, PathBuf}, + sync::{Arc, Mutex, MutexGuard, mpsc}, + time::{Duration, Instant}, +}; + +use environment_protocol::{data::exec::OutputStream, ids::JobId}; +use tokio::sync::oneshot; + +use super::log::OutputLog; + +/// How often buffered output reaches the disk while a job runs. +const FLUSH_INTERVAL: Duration = Duration::from_millis(250); + +const RECORD_FILE: &str = "record.json"; +const HEAD_FILE: &str = "output.head"; +const TAIL_FILE: &str = "output.tail"; + +/// Frame header: offset (u64), stream (u8), length (u32), little endian. +const FRAME_HEADER_BYTES: usize = 13; + +enum Command { + WriteRecord { + job_id: JobId, + record: Vec, + done: Option>, + }, + /// The job has pending output. + Drain { + job_id: JobId, + }, + /// Flushes and closes the job's output, then writes its final record. + Finish { + job_id: JobId, + record: Vec, + done: oneshot::Sender<()>, + }, + Remove { + job_id: JobId, + }, +} + +/// A frame of output: offset, stream, bytes. +type Frame = (u64, OutputStream, Vec); + +/// Output of one job not yet taken by the store thread. +#[derive(Default)] +struct Pending { + /// In offset order; a frame lies wholly below or wholly past the head + /// cap. + frames: Vec, + /// Bytes of `frames` past the head cap. + tail_bytes: u64, + /// A drain command for the job is queued. + queued: bool, +} + +type PendingOutput = Arc>>; + +/// A handle to the store thread. Cheap to clone. +#[derive(Clone)] +pub(crate) struct JobStore { + root: PathBuf, + head_cap: u64, + tail_cap: u64, + pending: PendingOutput, + sender: mpsc::Sender, +} + +impl JobStore { + /// Starts the store thread for job directories under `root`. Output at + /// offsets below `head_cap` goes to the head log, the rest to the tail + /// log, which keeps at least the newest `tail_cap` bytes. + pub fn start(root: PathBuf, head_cap: u64, tail_cap: u64) -> std::io::Result { + let (sender, receiver) = mpsc::channel(); + let pending = PendingOutput::default(); + let writer = Writer { + root: root.clone(), + head_cap, + tail_cap, + pending: pending.clone(), + open: BTreeMap::new(), + }; + std::thread::Builder::new() + .name("envd-job-store".to_owned()) + .spawn(move || writer.run(receiver))?; + Ok(Self { + root, + head_cap, + tail_cap, + pending, + sender, + }) + } + + pub fn root(&self) -> &Path { + &self.root + } + + /// Queues a record write; the receiver answers once it is on disk. + pub fn write_record(&self, job_id: &JobId, record: Vec) -> oneshot::Receiver<()> { + let (done, written) = oneshot::channel(); + self.send(Command::WriteRecord { + job_id: job_id.clone(), + record, + done: Some(done), + }); + written + } + + /// Queues a record write without waiting. + pub fn queue_record(&self, job_id: &JobId, record: Vec) { + self.send(Command::WriteRecord { + job_id: job_id.clone(), + record, + done: None, + }); + } + + /// Adds output at `offset` to the job's pending buffer, keeping its + /// head bytes and its newest tail-cap bytes past the head cap. + pub fn append(&self, job_id: &JobId, offset: u64, stream: OutputStream, data: &[u8]) { + let head_len = self.head_cap.saturating_sub(offset).min(data.len() as u64) as usize; + let (head, tail) = data.split_at(head_len); + let wake = { + let mut pending = lock(&self.pending); + let entry = pending.entry(job_id.clone()).or_default(); + entry.push(offset, stream, head, self.head_cap); + entry.push(offset + head_len as u64, stream, tail, self.head_cap); + entry.tail_bytes += tail.len() as u64; + entry.trim_tail(self.head_cap, self.tail_cap); + !std::mem::replace(&mut entry.queued, true) + }; + if wake { + self.send(Command::Drain { + job_id: job_id.clone(), + }); + } + } + + /// Flushes the job's output and writes its final record; answers once + /// both are on disk. + pub async fn finish(&self, job_id: &JobId, record: Vec) { + let (done, written) = oneshot::channel(); + self.send(Command::Finish { + job_id: job_id.clone(), + record, + done, + }); + let _ = written.await; + } + + pub fn remove(&self, job_id: &JobId) { + self.send(Command::Remove { + job_id: job_id.clone(), + }); + } + + fn send(&self, command: Command) { + if self.sender.send(command).is_err() { + eprintln!("lightspeed-envd: the job store thread has stopped"); + } + } +} + +/// Reads a job's retained output back. `end` is the end recorded with the +/// job's final status; a larger end found in the logs wins (a job that was +/// interrupted before its final record). +pub(crate) fn load_output(dir: &Path, end: u64, cap: u64) -> std::io::Result { + let mut records = read_frames(&dir.join(HEAD_FILE))?; + records.extend(read_frames(&dir.join(TAIL_FILE))?); + records.sort_by_key(|(offset, _, _)| *offset); + let mut log = OutputLog::from_records(records, end); + log.enforce_cap(cap, 0); + Ok(log) +} + +struct OpenOutput { + head: Option>, + tail: Option>, + tail_bytes: u64, +} + +impl Pending { + fn push(&mut self, offset: u64, stream: OutputStream, data: &[u8], head_cap: u64) { + if data.is_empty() { + return; + } + // Contiguous output of one stream on one side of the head cap + // becomes one frame. + if let Some((last_offset, last_stream, last_data)) = self.frames.last_mut() + && *last_stream == stream + && *last_offset + last_data.len() as u64 == offset + && (*last_offset < head_cap) == (offset < head_cap) + { + last_data.extend_from_slice(data); + return; + } + self.frames.push((offset, stream, data.to_vec())); + } + + /// Drops the oldest bytes past the head cap beyond the newest + /// `tail_cap`. + fn trim_tail(&mut self, head_cap: u64, tail_cap: u64) { + while self.tail_bytes > tail_cap { + let Some(first) = self + .frames + .iter() + .position(|(offset, _, _)| *offset >= head_cap) + else { + return; + }; + let excess = self.tail_bytes - tail_cap; + let (offset, _, data) = &mut self.frames[first]; + if data.len() as u64 <= excess { + self.tail_bytes -= data.len() as u64; + self.frames.remove(first); + } else { + data.drain(..excess as usize); + *offset += excess; + self.tail_bytes -= excess; + } + } + } +} + +struct Writer { + root: PathBuf, + head_cap: u64, + tail_cap: u64, + pending: PendingOutput, + open: BTreeMap, +} + +impl Writer { + fn run(mut self, receiver: mpsc::Receiver) { + let mut flushed = Instant::now(); + loop { + // Flushes on the interval even while commands keep arriving. + let wait = FLUSH_INTERVAL.saturating_sub(flushed.elapsed()); + match receiver.recv_timeout(wait) { + Ok(command) => self.handle(command), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + let jobs: Vec = lock(&self.pending).keys().cloned().collect(); + for job_id in jobs { + self.drain(&job_id); + } + self.flush_all(); + return; + } + } + if flushed.elapsed() >= FLUSH_INTERVAL { + self.flush_all(); + flushed = Instant::now(); + } + } + } + + /// Writes the job's pending output. + fn drain(&mut self, job_id: &JobId) { + let frames = { + let mut pending = lock(&self.pending); + let Some(entry) = pending.get_mut(job_id) else { + return; + }; + entry.queued = false; + entry.tail_bytes = 0; + std::mem::take(&mut entry.frames) + }; + for (offset, stream, data) in frames { + if let Err(error) = self.append(job_id, offset, stream, &data) { + eprintln!("lightspeed-envd: could not store output of job {job_id}: {error}"); + return; + } + } + } + + fn handle(&mut self, command: Command) { + match command { + Command::WriteRecord { + job_id, + record, + done, + } => { + self.write_record(&job_id, &record); + if let Some(done) = done { + let _ = done.send(()); + } + } + Command::Drain { job_id } => self.drain(&job_id), + Command::Finish { + job_id, + record, + done, + } => { + self.drain(&job_id); + lock(&self.pending).remove(&job_id); + if let Some(mut output) = self.open.remove(&job_id) { + for writer in [output.head.as_mut(), output.tail.as_mut()] + .into_iter() + .flatten() + { + if let Err(error) = writer.flush() { + eprintln!( + "lightspeed-envd: could not flush output of job {job_id}: {error}" + ); + } + } + } + self.write_record(&job_id, &record); + let _ = done.send(()); + } + Command::Remove { job_id } => { + lock(&self.pending).remove(&job_id); + self.open.remove(&job_id); + let dir = self.root.join(job_id.as_str()); + if let Err(error) = std::fs::remove_dir_all(&dir) + && error.kind() != std::io::ErrorKind::NotFound + { + eprintln!("lightspeed-envd: could not remove job {job_id}: {error}"); + } + } + } + } + + fn write_record(&self, job_id: &JobId, record: &[u8]) { + let dir = self.root.join(job_id.as_str()); + let result = std::fs::create_dir_all(&dir).and_then(|()| { + let path = dir.join(RECORD_FILE); + let temp = dir.join(format!("{RECORD_FILE}.tmp")); + std::fs::write(&temp, record)?; + std::fs::rename(&temp, &path) + }); + if let Err(error) = result { + eprintln!("lightspeed-envd: could not persist job {job_id}: {error}"); + } + } + + fn append( + &mut self, + job_id: &JobId, + offset: u64, + stream: OutputStream, + data: &[u8], + ) -> std::io::Result<()> { + let head_cap = self.head_cap; + let head_len = head_cap.saturating_sub(offset).min(data.len() as u64) as usize; + let (head, tail) = data.split_at(head_len); + let dir = self.root.join(job_id.as_str()); + if !self.open.contains_key(job_id) { + std::fs::create_dir_all(&dir)?; + self.open.insert( + job_id.clone(), + OpenOutput { + head: None, + tail: None, + tail_bytes: file_len(&dir.join(TAIL_FILE)), + }, + ); + } + let output = self.open.get_mut(job_id).expect("opened above"); + if !head.is_empty() { + if output.head.is_none() { + output.head = Some(BufWriter::new(append_to(&dir.join(HEAD_FILE))?)); + } + let writer = output.head.as_mut().expect("opened above"); + write_frame(writer, offset, stream, head)?; + } + if !tail.is_empty() { + if output.tail.is_none() { + output.tail = Some(BufWriter::new(append_to(&dir.join(TAIL_FILE))?)); + } + let writer = output.tail.as_mut().expect("opened above"); + write_frame(writer, offset + head_len as u64, stream, tail)?; + output.tail_bytes += (FRAME_HEADER_BYTES + tail.len()) as u64; + if output.tail_bytes > 2 * self.tail_cap + 64 * 1024 { + let tail_cap = self.tail_cap; + compact_tail(&dir, output, tail_cap)?; + } + } + Ok(()) + } + + fn flush_all(&mut self) { + for (job_id, output) in &mut self.open { + for writer in [output.head.as_mut(), output.tail.as_mut()] + .into_iter() + .flatten() + { + if let Err(error) = writer.flush() { + eprintln!("lightspeed-envd: could not flush output of job {job_id}: {error}"); + } + } + } + } +} + +/// Rewrites the tail log with only its newest `tail_cap` bytes. +fn compact_tail(dir: &Path, output: &mut OpenOutput, tail_cap: u64) -> std::io::Result<()> { + if let Some(mut writer) = output.tail.take() { + writer.flush()?; + } + let path = dir.join(TAIL_FILE); + let mut records = read_frames(&path)?; + let mut kept = 0u64; + let mut first = records.len(); + while first > 0 && kept < tail_cap { + first -= 1; + kept += records[first].2.len() as u64; + } + let mut records = records.split_off(first); + if let Some((offset, _, data)) = records.first_mut() + && kept > tail_cap + { + let excess = (kept - tail_cap) as usize; + data.drain(..excess); + *offset += excess as u64; + } + let temp = dir.join(format!("{TAIL_FILE}.tmp")); + { + let mut writer = BufWriter::new(File::create(&temp)?); + for (offset, stream, data) in &records { + write_frame(&mut writer, *offset, *stream, data)?; + } + writer.flush()?; + } + std::fs::rename(&temp, &path)?; + output.tail_bytes = file_len(&path); + output.tail = Some(BufWriter::new(append_to(&path)?)); + Ok(()) +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn append_to(path: &Path) -> std::io::Result { + OpenOptions::new().create(true).append(true).open(path) +} + +fn file_len(path: &Path) -> u64 { + std::fs::metadata(path).map_or(0, |metadata| metadata.len()) +} + +fn write_frame( + writer: &mut impl std::io::Write, + offset: u64, + stream: OutputStream, + data: &[u8], +) -> std::io::Result<()> { + let stream = match stream { + OutputStream::Stdout => 0u8, + OutputStream::Stderr => 1, + OutputStream::Pty => 2, + }; + let len = u32::try_from(data.len()).map_err(std::io::Error::other)?; + writer.write_all(&offset.to_le_bytes())?; + writer.write_all(&[stream])?; + writer.write_all(&len.to_le_bytes())?; + writer.write_all(data) +} + +/// Every whole frame of a log; a frame cut short by a crash ends it. +fn read_frames(path: &Path) -> std::io::Result)>> { + let mut bytes = Vec::new(); + match File::open(path) { + Ok(mut file) => { + file.read_to_end(&mut bytes)?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + } + let mut frames = Vec::new(); + let mut rest = bytes.as_slice(); + while rest.len() >= FRAME_HEADER_BYTES { + let offset = u64::from_le_bytes(rest[..8].try_into().expect("8 bytes")); + let stream = match rest[8] { + 0 => OutputStream::Stdout, + 1 => OutputStream::Stderr, + 2 => OutputStream::Pty, + _ => break, + }; + let len = u32::from_le_bytes(rest[9..13].try_into().expect("4 bytes")) as usize; + let Some(data) = rest.get(FRAME_HEADER_BYTES..FRAME_HEADER_BYTES + len) else { + break; + }; + frames.push((offset, stream, data.to_vec())); + rest = &rest[FRAME_HEADER_BYTES + len..]; + } + Ok(frames) +} + +#[cfg(test)] +mod tests { + use environment_protocol::data::exec::GapReason; + + use super::*; + + #[tokio::test(flavor = "current_thread")] + async fn output_round_trips_with_head_tail_and_bounded_disk_use() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = JobStore::start(temp.path().to_owned(), 1000, 1000).expect("store"); + let job_id = JobId::new("verbose").unwrap(); + // 200 KiB through 8 KiB appends against a 1000-byte tail. + let mut offset = 0u64; + for i in 0..25u8 { + let data = vec![i; 8192]; + store.append(&job_id, offset, OutputStream::Stdout, &data); + offset += data.len() as u64; + } + store.finish(&job_id, b"{}".to_vec()).await; + let dir = temp.path().join("verbose"); + assert!( + file_len(&dir.join(TAIL_FILE)) + <= 2 * 1000 + 64 * 1024 + 8192 + 2 * FRAME_HEADER_BYTES as u64, + "the tail log is compacted" + ); + assert_eq!( + file_len(&dir.join(HEAD_FILE)), + 1000 + FRAME_HEADER_BYTES as u64 + ); + assert_eq!(std::fs::read(dir.join(RECORD_FILE)).unwrap(), b"{}"); + + let log = load_output(&dir, offset, 2000).expect("load"); + assert_eq!(log.end(), offset); + let page = log.page(0, 10_000, 4096); + assert_eq!(page.gaps.len(), 1); + assert_eq!(page.gaps[0].reason, GapReason::Overflow); + assert_eq!(page.gaps[0].offset, 1000); + assert_eq!(page.gaps[0].omitted_bytes, offset - 2000); + assert_eq!(page.bytes, 2000); + assert_eq!( + page.chunks.last().unwrap().data.as_slice().last(), + Some(&24) + ); + + store.remove(&job_id); + let _ = store + .write_record(&JobId::new("other").unwrap(), b"{}".to_vec()) + .await; + assert!(!dir.exists(), "commands run in order"); + } + + #[test] + fn pending_output_keeps_the_head_and_the_newest_tail_only() { + let mut pending = Pending::default(); + let (head_cap, tail_cap) = (100u64, 300u64); + let mut offset = 0u64; + for i in 0..50u8 { + let data = [i; 64]; + let head_len = head_cap.saturating_sub(offset).min(64) as usize; + let (head, tail) = data.split_at(head_len); + pending.push(offset, OutputStream::Stdout, head, head_cap); + pending.push( + offset + head_len as u64, + OutputStream::Stdout, + tail, + head_cap, + ); + pending.tail_bytes += tail.len() as u64; + pending.trim_tail(head_cap, tail_cap); + offset += 64; + let held: u64 = pending + .frames + .iter() + .map(|(_, _, data)| data.len() as u64) + .sum(); + assert!(held <= head_cap + tail_cap, "{held} bytes pending"); + } + let frames = &pending.frames; + assert_eq!(frames.len(), 2, "contiguous output coalesces per side"); + assert_eq!((frames[0].0, frames[0].2.len()), (0, 100)); + assert_eq!(frames[1].0, offset - tail_cap); + assert_eq!(frames[1].2.len() as u64, tail_cap); + assert_eq!(frames[1].2.last(), Some(&49)); + } + + #[test] + fn a_frame_cut_short_ends_the_log() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("log"); + let mut bytes = Vec::new(); + write_frame(&mut bytes, 0, OutputStream::Stderr, b"whole").unwrap(); + write_frame(&mut bytes, 5, OutputStream::Stdout, b"cut").unwrap(); + bytes.truncate(bytes.len() - 1); + std::fs::write(&path, bytes).unwrap(); + let frames = read_frames(&path).unwrap(); + assert_eq!(frames, vec![(0, OutputStream::Stderr, b"whole".to_vec())]); + } +} diff --git a/crates/environment-daemon/src/exec/log.rs b/crates/environment-daemon/src/exec/log.rs new file mode 100644 index 000000000..8e711c446 --- /dev/null +++ b/crates/environment-daemon/src/exec/log.rs @@ -0,0 +1,422 @@ +//! Byte-offset output logs. +//! +//! A log holds a handle's combined output as segments addressed by byte +//! offset. `end` counts every byte ever appended; a retention cap drops +//! bytes but never renumbers them, so a page cut at any byte loses nothing +//! and every range the log no longer holds is reported as a typed gap. + +use std::collections::VecDeque; + +use environment_protocol::{ + bytes::ByteChunk, + data::exec::{GapReason, OutputChunk, OutputGap, OutputStream}, +}; + +/// Consecutive appends from one stream merge into one segment up to this +/// size, which keeps the chunk count of a page low. +const MERGE_LIMIT_BYTES: usize = 64 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Segment { + offset: u64, + stream: OutputStream, + data: Vec, +} + +impl Segment { + fn end(&self) -> u64 { + self.offset + self.data.len() as u64 + } +} + +/// One page cut from a log. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Page { + pub chunks: Vec, + pub gaps: Vec, + /// Offset after the last returned byte or gap; the start when the page + /// is empty. + pub next: u64, + /// Bytes of `data` returned. + pub bytes: u64, +} + +impl Page { + pub fn items(&self) -> u32 { + u32::try_from(self.chunks.len() + self.gaps.len()).unwrap_or(u32::MAX) + } +} + +/// Retained output of one handle. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct OutputLog { + /// Sorted by offset, disjoint. + segments: VecDeque, + end: u64, + retained: u64, + /// Bytes below this offset were delivered and released. + released_until: u64, + /// Ranges `[start, end)` dropped from the middle, sorted, disjoint and + /// never adjacent. Ranges below `released_until` are forgotten and then + /// read as released. + overflow: Vec<(u64, u64)>, +} + +impl OutputLog { + /// A log rebuilt from persisted records, in offset order. Holes between + /// records, and a missing tail up to `end`, read as overflow. + pub fn from_records( + records: impl IntoIterator)>, + end: u64, + ) -> Self { + let mut log = Self::default(); + for (offset, stream, data) in records { + if data.is_empty() { + continue; + } + // Overlapping records (a tail that was compacted after a crash + // left a stale copy) keep the first copy of every byte. + let record_end = offset + data.len() as u64; + if record_end <= log.end { + continue; + } + let skip = log.end.saturating_sub(offset) as usize; + let offset = offset.max(log.end); + if offset > log.end { + log.add_overflow(log.end, offset); + log.end = offset; + } + log.append(stream, &data[skip..]); + } + if end > log.end { + log.add_overflow(log.end, end); + log.end = end; + } + log + } + + /// Bytes produced so far. + pub fn end(&self) -> u64 { + self.end + } + + /// Bytes held. + pub fn retained_bytes(&self) -> u64 { + self.retained + } + + pub fn append(&mut self, stream: OutputStream, data: &[u8]) { + if data.is_empty() { + return; + } + let offset = self.end; + match self.segments.back_mut() { + Some(last) + if last.stream == stream + && last.end() == offset + && last.data.len() + data.len() <= MERGE_LIMIT_BYTES => + { + last.data.extend_from_slice(data); + } + _ => self.segments.push_back(Segment { + offset, + stream, + data: data.to_vec(), + }), + } + self.end += data.len() as u64; + self.retained += data.len() as u64; + } + + /// Keeps at most `cap` bytes. First releases delivered bytes (below + /// `cursor`), oldest first; if still over, drops from the middle of the + /// undelivered region, keeping its head (the first half of `cap` from + /// `cursor`) and the newest bytes. Every dropped range becomes an + /// overflow gap; consecutive drops extend one gap. + pub fn enforce_cap(&mut self, cap: u64, cursor: u64) { + while self.retained > cap { + let Some(front) = self.segments.front_mut() else { + break; + }; + if front.offset >= cursor { + break; + } + let releasable = cursor.min(front.end()) - front.offset; + let release = releasable.min(self.retained - cap) as usize; + front.data.drain(..release); + front.offset += release as u64; + self.retained -= release as u64; + self.released_until = self.released_until.max(front.offset); + if front.data.is_empty() { + self.segments.pop_front(); + } + } + let released_until = self.released_until; + self.overflow.retain(|(_, end)| *end > released_until); + + if self.retained <= cap { + return; + } + let head_end = cursor.max(self.released_until).saturating_add(cap / 2); + let mut excess = self.retained - cap; + let mut index = self + .segments + .partition_point(|segment| segment.end() <= head_end); + while excess > 0 && index < self.segments.len() { + let segment = &mut self.segments[index]; + if segment.offset < head_end { + // The segment straddles the head: keep its head part. + let keep = (head_end - segment.offset) as usize; + let rest = segment.data.split_off(keep); + let tail = Segment { + offset: head_end, + stream: segment.stream, + data: rest, + }; + self.segments.insert(index + 1, tail); + index += 1; + continue; + } + let drop = excess.min(segment.data.len() as u64); + let start = segment.offset; + segment.data.drain(..drop as usize); + segment.offset += drop; + let emptied = segment.data.is_empty(); + self.retained -= drop; + excess -= drop; + self.add_overflow(start, start + drop); + if emptied { + self.segments.remove(index); + } + } + } + + /// A page from `start`: at most `max_bytes` of data, split at exactly + /// that byte, and at most `max_items` chunks and gaps together. An empty + /// budget returns an empty page at `start`. + pub fn page(&self, start: u64, max_bytes: u64, max_items: u32) -> Page { + let mut page = Page { + next: start, + ..Page::default() + }; + let mut pos = start; + let mut budget = max_bytes; + while pos < self.end && budget > 0 && page.items() < max_items { + if let Some(&(_, end)) = self + .overflow + .iter() + .find(|(range_start, end)| *range_start <= pos && pos < *end) + { + push_gap(&mut page, pos, end, GapReason::Overflow, max_items); + pos = end; + } else if pos < self.released_until { + let end = self + .overflow + .iter() + .map(|(range_start, _)| *range_start) + .find(|range_start| *range_start > pos) + .map_or(self.released_until, |range_start| { + range_start.min(self.released_until) + }); + push_gap(&mut page, pos, end, GapReason::Expired, max_items); + pos = end; + } else { + let index = self + .segments + .partition_point(|segment| segment.end() <= pos); + match self.segments.get(index) { + Some(segment) if segment.offset <= pos => { + let from = (pos - segment.offset) as usize; + let take = (segment.data.len() - from).min(budget as usize); + page.chunks.push(OutputChunk { + offset: pos, + stream: segment.stream, + data: ByteChunk::from(&segment.data[from..from + take]), + }); + pos += take as u64; + budget -= take as u64; + page.bytes += take as u64; + } + next => { + // A hole no rule explains reads as overflow. + let end = next.map_or(self.end, |segment| segment.offset); + push_gap(&mut page, pos, end, GapReason::Overflow, max_items); + pos = end; + } + } + } + page.next = pos; + } + page + } + + fn add_overflow(&mut self, start: u64, end: u64) { + if start >= end { + return; + } + let index = self + .overflow + .partition_point(|(_, range_end)| *range_end < start); + let mut merged = (start, end); + while let Some(&(range_start, range_end)) = self.overflow.get(index) { + if range_start > merged.1 { + break; + } + merged = (merged.0.min(range_start), merged.1.max(range_end)); + self.overflow.remove(index); + } + self.overflow.insert(index, merged); + } +} + +fn push_gap(page: &mut Page, start: u64, end: u64, reason: GapReason, max_items: u32) { + if let Some(last) = page.gaps.last_mut() + && last.reason == reason + && last.offset + last.omitted_bytes == start + && page + .chunks + .last() + .is_none_or(|chunk| chunk.offset < last.offset) + { + last.omitted_bytes += end - start; + return; + } + debug_assert!(page.items() < max_items); + page.gaps.push(OutputGap { + offset: start, + omitted_bytes: end - start, + reason, + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bytes(page: &Page) -> Vec { + page.chunks + .iter() + .flat_map(|chunk| chunk.data.as_slice().to_vec()) + .collect() + } + + fn log_of(parts: &[&[u8]]) -> OutputLog { + let mut log = OutputLog::default(); + for (index, part) in parts.iter().enumerate() { + let stream = if index % 2 == 0 { + OutputStream::Stdout + } else { + OutputStream::Stderr + }; + log.append(stream, part); + } + log + } + + #[test] + fn pages_cut_at_max_bytes_and_continue_without_loss() { + // Job paging used to skip the remainder of a cut chunk: every byte + // of 20 000 must arrive through 100-byte pages. + let data: Vec = (0..20_000u32).map(|i| (i % 251) as u8).collect(); + let mut log = OutputLog::default(); + for chunk in data.chunks(8192) { + log.append(OutputStream::Stdout, chunk); + } + let mut collected = Vec::new(); + let mut next = 0; + while next < log.end() { + let page = log.page(next, 100, 4096); + assert!(page.bytes <= 100); + assert!(page.gaps.is_empty()); + assert_eq!(page.chunks[0].offset, next); + collected.extend(bytes(&page)); + next = page.next; + } + assert_eq!(collected, data); + } + + #[test] + fn zero_budget_leaves_the_position_in_place() { + let log = log_of(&[b"abc"]); + let page = log.page(1, 0, 4096); + assert!(page.chunks.is_empty() && page.gaps.is_empty()); + assert_eq!(page.next, 1); + } + + #[test] + fn chunks_keep_stream_and_offset() { + let log = log_of(&[b"out", b"err"]); + let page = log.page(2, 100, 4096); + assert_eq!(page.chunks.len(), 2); + assert_eq!(page.chunks[0].offset, 2); + assert_eq!(page.chunks[0].stream, OutputStream::Stdout); + assert_eq!(page.chunks[1].offset, 3); + assert_eq!(page.chunks[1].stream, OutputStream::Stderr); + assert_eq!(bytes(&page), b"terr"); + assert_eq!(page.next, 6); + } + + #[test] + fn cap_drops_one_middle_range_and_keeps_head_and_newest() { + let mut log = OutputLog::default(); + for i in 0..64u8 { + log.append(OutputStream::Stdout, &[i; 100]); + log.enforce_cap(1000, 0); + } + assert_eq!(log.end(), 6400); + assert_eq!(log.retained_bytes(), 1000); + let page = log.page(0, 10_000, 4096); + assert_eq!(page.gaps.len(), 1, "one contiguous gap: {:?}", page.gaps); + assert_eq!(page.gaps[0].reason, GapReason::Overflow); + assert_eq!(page.gaps[0].offset, 500); + assert_eq!(page.gaps[0].omitted_bytes, 6400 - 1000); + let data = bytes(&page); + assert_eq!( + &data[..500], + &[[0u8; 100], [1; 100], [2; 100], [3; 100], [4; 100]].concat()[..] + ); + assert_eq!(data[999], 63, "the newest byte survives"); + assert_eq!(page.next, 6400); + } + + #[test] + fn delivered_bytes_are_released_first_and_read_as_expired() { + let mut log = log_of(&[b"0123456789"]); + // Everything up to 8 was delivered through the cursor. + log.append(OutputStream::Stdout, b"abcdef"); + log.enforce_cap(10, 8); + assert_eq!(log.retained_bytes(), 10); + let page = log.page(0, 100, 4096); + assert_eq!(page.gaps[0].reason, GapReason::Expired); + assert_eq!(page.gaps[0].offset, 0); + assert_eq!(page.gaps[0].omitted_bytes, 6); + assert_eq!(bytes(&page), b"6789abcdef"); + } + + #[test] + fn page_item_limit_ends_the_page_early_without_loss() { + let log = log_of(&[b"a", b"b", b"c", b"d"]); + let page = log.page(0, 100, 2); + assert_eq!(bytes(&page), b"ab"); + assert_eq!(page.next, 2); + let rest = log.page(page.next, 100, 2); + assert_eq!(bytes(&rest), b"cd"); + } + + #[test] + fn records_rebuild_head_tail_and_the_gap_between() { + let log = OutputLog::from_records( + [ + (0, OutputStream::Stdout, b"head".to_vec()), + (100, OutputStream::Stderr, b"tail".to_vec()), + ], + 110, + ); + assert_eq!(log.end(), 110); + let page = log.page(0, 100, 4096); + assert_eq!(bytes(&page), b"headtail"); + assert_eq!(page.gaps.len(), 2); + assert_eq!((page.gaps[0].offset, page.gaps[0].omitted_bytes), (4, 96)); + assert_eq!((page.gaps[1].offset, page.gaps[1].omitted_bytes), (104, 6)); + } +} diff --git a/crates/environment-daemon/src/exec/outcome.rs b/crates/environment-daemon/src/exec/outcome.rs new file mode 100644 index 000000000..05d1d01a0 --- /dev/null +++ b/crates/environment-daemon/src/exec/outcome.rs @@ -0,0 +1,179 @@ +//! How a root process ended, and what its group left behind. + +use environment_protocol::{ + data::exec::{ExitStatus, LeftoverDisposition, LeftoverProcess, LeftoverReport, SignalName}, + limits::{MAX_LEFTOVER_COMMAND_BYTES, MAX_LEFTOVER_PROCESSES}, +}; + +/// The wire exit of a reaped child: its status code, or the signal that +/// killed it by name. +pub(crate) fn exit_status(status: std::process::ExitStatus) -> ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + if let Some(signal) = status.signal() { + return ExitStatus::Signal { + signal: signal_name(signal), + core_dumped: status.core_dumped(), + }; + } + } + ExitStatus::Code { + code: status.code().unwrap_or(-1), + } +} + +/// `exit 3` or `SIGSEGV (core dumped)`, for error messages. +pub(crate) fn describe_exit(exit: &ExitStatus) -> String { + match exit { + ExitStatus::Code { code } => format!("exited with status {code}"), + ExitStatus::Signal { + signal, + core_dumped: true, + } => format!("terminated by signal {signal} (core dumped)"), + ExitStatus::Signal { signal, .. } => format!("terminated by signal {signal}"), + } +} + +/// The POSIX name of signal number `raw` on this platform. A number +/// without a portable name is spelled `SIG`. +pub(crate) fn signal_name(raw: i32) -> SignalName { + #[cfg(unix)] + { + use rustix::process::Signal; + let known: &[(Signal, &str)] = &[ + (Signal::HUP, "SIGHUP"), + (Signal::INT, "SIGINT"), + (Signal::QUIT, "SIGQUIT"), + (Signal::ILL, "SIGILL"), + (Signal::TRAP, "SIGTRAP"), + (Signal::ABORT, "SIGABRT"), + (Signal::BUS, "SIGBUS"), + (Signal::FPE, "SIGFPE"), + (Signal::KILL, "SIGKILL"), + (Signal::USR1, "SIGUSR1"), + (Signal::SEGV, "SIGSEGV"), + (Signal::USR2, "SIGUSR2"), + (Signal::PIPE, "SIGPIPE"), + (Signal::ALARM, "SIGALRM"), + (Signal::TERM, "SIGTERM"), + (Signal::CHILD, "SIGCHLD"), + (Signal::CONT, "SIGCONT"), + (Signal::STOP, "SIGSTOP"), + (Signal::TSTP, "SIGTSTP"), + (Signal::TTIN, "SIGTTIN"), + (Signal::TTOU, "SIGTTOU"), + (Signal::URG, "SIGURG"), + (Signal::XCPU, "SIGXCPU"), + (Signal::XFSZ, "SIGXFSZ"), + (Signal::VTALARM, "SIGVTALRM"), + (Signal::PROF, "SIGPROF"), + (Signal::WINCH, "SIGWINCH"), + (Signal::IO, "SIGIO"), + (Signal::SYS, "SIGSYS"), + #[cfg(target_os = "linux")] + (Signal::STKFLT, "SIGSTKFLT"), + #[cfg(target_os = "linux")] + (Signal::POWER, "SIGPWR"), + #[cfg(target_os = "macos")] + (Signal::EMT, "SIGEMT"), + #[cfg(target_os = "macos")] + (Signal::INFO, "SIGINFO"), + ]; + if let Some((_, name)) = known.iter().find(|(signal, _)| signal.as_raw() == raw) { + return SignalName::new(*name).expect("known signal names are valid"); + } + } + SignalName::new(format!("SIG{}", raw.unsigned_abs())) + .expect("SIG followed by at most 10 digits is a valid signal name") +} + +/// A report from enumerated members, lowest pid first: at most +/// `MAX_LEFTOVER_PROCESSES` listed, commands cut to +/// `MAX_LEFTOVER_COMMAND_BYTES` on a character boundary. +pub(crate) fn leftover_report( + mut members: Vec<(u32, Option)>, + disposition: LeftoverDisposition, +) -> Option { + if members.is_empty() { + return None; + } + members.sort_by_key(|(pid, _)| *pid); + let listed = MAX_LEFTOVER_PROCESSES as usize; + let omitted_process_count = + u32::try_from(members.len().saturating_sub(listed)).unwrap_or(u32::MAX); + members.truncate(listed); + let processes = members + .into_iter() + .map(|(pid, command)| LeftoverProcess { + pid, + command: command + .map(|command| cut(command, MAX_LEFTOVER_COMMAND_BYTES as usize)) + .filter(|command| !command.is_empty()), + }) + .collect(); + Some(LeftoverReport { + disposition, + processes, + omitted_process_count, + }) +} + +fn cut(mut text: String, max: usize) -> String { + if text.len() > max { + let mut end = max; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + } + text +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(unix)] + #[test] + fn signal_deaths_are_named_and_codes_kept() { + use std::os::unix::process::ExitStatusExt as _; + // A raw wait status: low 7 bits the signal, 0x80 the core flag. + let segv = + std::process::ExitStatus::from_raw(rustix::process::Signal::SEGV.as_raw() | 0x80); + assert_eq!( + exit_status(segv), + ExitStatus::Signal { + signal: SignalName::new("SIGSEGV").unwrap(), + core_dumped: true, + } + ); + let term = std::process::ExitStatus::from_raw(rustix::process::Signal::TERM.as_raw()); + assert_eq!( + describe_exit(&exit_status(term)), + "terminated by signal SIGTERM" + ); + let code = std::process::ExitStatus::from_raw(3 << 8); + assert_eq!(exit_status(code), ExitStatus::Code { code: 3 }); + assert_eq!(signal_name(200).as_str(), "SIG200"); + } + + #[test] + fn leftover_reports_are_capped_sorted_and_cut() { + let members = (0..100u32) + .rev() + .map(|pid| (pid + 10, Some("é".repeat(1000)))) + .collect(); + let report = leftover_report(members, LeftoverDisposition::Kept).expect("report"); + assert_eq!(report.processes.len(), MAX_LEFTOVER_PROCESSES as usize); + assert_eq!(report.omitted_process_count, 100 - MAX_LEFTOVER_PROCESSES); + assert_eq!(report.processes[0].pid, 10); + let command = report.processes[0].command.as_deref().unwrap(); + assert!(command.len() <= MAX_LEFTOVER_COMMAND_BYTES as usize); + assert!(command.len() >= MAX_LEFTOVER_COMMAND_BYTES as usize - 1); + assert!(leftover_report(Vec::new(), LeftoverDisposition::Swept).is_none()); + let unnamed = leftover_report(vec![(5, Some(String::new()))], LeftoverDisposition::Swept) + .expect("report"); + assert_eq!(unnamed.processes[0].command, None); + } +} diff --git a/crates/environment-daemon/src/exec/spawn_record.rs b/crates/environment-daemon/src/exec/spawn_record.rs new file mode 100644 index 000000000..1e3c6b4e2 --- /dev/null +++ b/crates/environment-daemon/src/exec/spawn_record.rs @@ -0,0 +1,101 @@ +//! Persisted spawn records: which process groups an earlier daemon +//! instance owned. +//! +//! A record names a group by its id, its leader's start time and the boot +//! it ran in. After a restart in the same boot, a group whose record still +//! matches is the earlier instance's: jobs kill it, processes adopt it as +//! leftovers. A group id cannot be reused while the group has members, so a +//! live group whose leader is gone is still the recorded one; a live leader +//! must also match its recorded start time. + +use std::path::Path; + +use environment_protocol::ids::DaemonInstanceId; +use serde::{Deserialize, Serialize}; + +use crate::process_group::{self, Group}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct SpawnRecord { + pub pgid: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub leader_start_time: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub boot_id: Option, + pub daemon_instance_id: DaemonInstanceId, +} + +impl SpawnRecord { + /// A record of `group`, spawned by this daemon instance. + pub fn new(group: Group, instance_id: &DaemonInstanceId) -> Self { + Self { + pgid: group.pgid, + leader_start_time: group.leader_start_time, + boot_id: process_group::boot_id(), + daemon_instance_id: instance_id.clone(), + } + } + + /// The recorded group. + pub fn group(&self) -> Group { + Group { + pgid: self.pgid, + leader_start_time: self.leader_start_time, + } + } + + /// The record was written in `boot`, the current boot. + pub fn same_boot(&self, boot: Option<&str>) -> bool { + boot.is_some() && self.boot_id.as_deref() == boot + } + + /// The recorded group still has members, and they are the recorded + /// group's. Only meaningful for a record of the current boot. + pub fn group_survives(&self) -> bool { + self.group().alive() + } +} + +/// Writes `value` as JSON to `path` through a sibling temporary file, so a +/// crash leaves either the old or the new content. +pub(crate) fn write_json_atomic(path: &Path, value: &T) -> std::io::Result<()> { + let bytes = serde_json::to_vec(value).map_err(std::io::Error::other)?; + let mut temp = path.as_os_str().to_owned(); + temp.push(".tmp"); + std::fs::write(&temp, bytes)?; + std::fs::rename(&temp, path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[tokio::test(flavor = "current_thread")] + async fn a_live_group_survives_and_a_reused_leader_does_not() { + let mut command = tokio::process::Command::new("/bin/sh"); + command.args(["-c", "sleep 30"]); + process_group::spawn_in_own_group(&mut command); + let mut child = command.spawn().expect("spawn"); + let pgid = child.id().expect("pid"); + let instance = DaemonInstanceId::new("envd_test").unwrap(); + let record = SpawnRecord::new(Group::capture(pgid), &instance); + assert!(record.same_boot(process_group::boot_id().as_deref())); + assert!(!record.same_boot(None)); + assert!(record.group_survives()); + + let reused = SpawnRecord { + leader_start_time: record.leader_start_time.map(|started| started + 1), + ..record.clone() + }; + assert!( + !reused.group_survives(), + "another process with the same pid" + ); + + process_group::signal_group(pgid, process_group::GroupSignal::Kill); + child.wait().await.expect("wait"); + assert!(!record.group_survives()); + } +} diff --git a/crates/environment-daemon/src/filesystem.rs b/crates/environment-daemon/src/filesystem.rs index ed49142e7..af69d711f 100644 --- a/crates/environment-daemon/src/filesystem.rs +++ b/crates/environment-daemon/src/filesystem.rs @@ -1,1119 +1,1407 @@ -#[cfg(any(target_os = "linux", target_os = "macos"))] -mod backend; -#[cfg(any(target_os = "linux", target_os = "macos"))] +//! envd's filesystem: the `fs/*` and `transfer/*` handlers. +//! +//! Every path goes through one [`Confinement`]: resolved beneath the root's +//! directory descriptor, following a symlink only while its target stays +//! inside the root, and never addressing the daemon state directory. Every +//! later operation names the resolved entry by its parent descriptor and +//! name, so nothing follows a link swapped in after resolution. Handlers do +//! their I/O on blocking threads; walks poll the request's cancellation +//! token between entries. + +mod confinement; +mod os; mod scan; -use environment_protocol::data::transfer::*; +mod search; mod transfer; +mod walk; + use std::{ - io, - path::{Component, Path, PathBuf}, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + ffi::OsString, + fmt::Display, + io::{self, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + sync::Arc, }; +pub use confinement::Confinement; +use confinement::Lookup; use environment_protocol::{ - data::fs::{ - CopyParams, CopyResponse, CreateDirectoryParams, CreateDirectoryResponse, - GetMetadataParams, GetMetadataResponse, GlobFilesParams, GlobFilesResponse, GlobFilesStop, - ReadDirectoryEntry, ReadDirectoryParams, ReadDirectoryResponse, ReadFileParams, - ReadFileResponse, RemoveParams, RemoveResponse, SearchTextMatch, SearchTextParams, - SearchTextResponse, SearchTextStop, WriteFileParams, WriteFileResponse, + data::{ + fs::{ + CreateDirectoryParams, CreateDirectoryResponse, DirectoryEntry, FileKind, FileMetadata, + FileVersion, GetMetadataParams, GetMetadataResponse, GlobFilesParams, + GlobFilesResponse, ReadDirectoryParams, ReadDirectoryResponse, ReadFileParams, + ReadFileResponse, RemoveParams, RemoveResponse, RenameParams, RenameResponse, + SearchTextParams, SearchTextResponse, WriteCondition, WriteFileParams, + WriteFileResponse, + }, + handshake::FilesystemLimits, + scan::{ScanParams, ScanResponse}, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, ListTransferMissingParams, + ListTransferMissingResponse, ReadTransferContentParams, ReadTransferContentResponse, + ReadTransferInventoryParams, ReadTransferInventoryResponse, TransferOperationParams, + TransferStatus, WriteTransferContentParams, WriteTransferContentResponse, + }, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::{ByteChunk, EnvironmentPath}, + error::{Capability, PermissionReason, PreconditionReason, ProtocolError}, + ids::DirectoryCursor, + paths::{EntryName, EnvironmentPath}, }; -use grep_searcher::{BinaryDetection, SearcherBuilder, sinks::UTF8}; -use tokio::fs; +use os::{Directory, EntryStat, RenameMode}; +use rustix::io::Errno; +use sha2::{Digest, Sha256}; + +use crate::dispatch::RequestContext; + +/// Prefix of `fs/writeFile` staging files next to their target. +const WRITE_STAGE_PREFIX: &str = ".lightspeed-write-"; + +/// What the filesystem handlers need from the daemon. +pub struct FilesystemConfig { + /// The root and state directory; transfers journal in + /// `/transfers`. + pub confinement: Arc, + /// Canonical working directory, inside the root. + pub cwd: PathBuf, + pub writable: bool, + pub limits: FilesystemLimits, +} +/// The `fs/*` and `transfer/*` handlers. Cheap to clone. #[derive(Clone)] pub struct LocalFileSystem { - root: PathBuf, + inner: Arc, +} + +struct Inner { + confinement: Arc, cwd: PathBuf, writable: bool, - #[cfg(any(target_os = "linux", target_os = "macos"))] - transfers: std::sync::Arc>, + limits: FilesystemLimits, + transfers: transfer::Transfers, } impl LocalFileSystem { - pub fn new(root: PathBuf, cwd: PathBuf, writable: bool) -> Self { - Self { - root: normalize_path(root), - cwd: normalize_path(cwd), - writable, - #[cfg(any(target_os = "linux", target_os = "macos"))] - transfers: Default::default(), - } - } - - pub fn with_transfer_state_dir( - mut self, - path: PathBuf, - ) -> Result { - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - self.transfers = std::sync::Arc::new(std::sync::Mutex::new( - transfer::session::TransferManager::with_journal(path)?, - )); - let weak = std::sync::Arc::downgrade(&self.transfers); - std::thread::Builder::new() - .name("transfer-expiry".into()) - .spawn(move || { - loop { - std::thread::sleep(Duration::from_secs(30)); - let Some(manager) = weak.upgrade() else { - break; - }; - let directory = - manager.lock().ok().and_then(|mut manager| manager.expire()); - drop(manager); - if let Some(directory) = directory { - transfer::session::TransferManager::cleanup_expired(&directory); - } - } - }) - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - e.to_string(), - ) - })?; - } - Ok(self) - } - - pub async fn transfer( - &self, - mut request: environment_protocol::data::transfer_session::TransferRequest, - ) -> Result< - environment_protocol::data::transfer_session::TransferResponse, - EnvironmentProtocolError, - > { - use environment_protocol::data::transfer_session::{TransferRequest, TransferSelection}; - if let TransferRequest::Begin { selection, .. } = &mut request { - let path = match selection { - TransferSelection::Capture { source } => source, - TransferSelection::Materialize { destination, .. } - | TransferSelection::WriteFile { destination } => { - self.ensure_writable()?; - destination - } - }; - *path = EnvironmentPath::new(self.resolve_scoped_path(path)?.to_string_lossy()) - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - e.to_string(), - ) - })?; - } - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - let transfers = self.transfers.clone(); - let root = self.root.clone(); - tokio::task::spawn_blocking(move || { - transfers - .lock() - .map_err(|_| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - "transfer lock poisoned", - ) - })? - .execute(&root, request) - }) - .await - .map_err(|e| { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::Internal, e.to_string()) - })? - } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - { - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Unsupported, - "filesystem transfer backend unavailable", - )) - } + pub fn new(config: FilesystemConfig) -> Result { + let transfers = transfer::Transfers::new( + config.confinement.clone(), + config.limits.transfer, + config.writable, + )?; + Ok(Self { + inner: Arc::new(Inner { + confinement: config.confinement, + cwd: config.cwd, + writable: config.writable, + limits: config.limits, + transfers, + }), + }) } - pub async fn scan( - &self, - mut params: environment_protocol::data::inventory::ScanParams, - ) -> Result { - for path in &mut params.roots { - *path = EnvironmentPath::new(self.resolve_scoped_path(path)?.to_string_lossy()) - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - e.to_string(), - ) - })?; - } - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - let root = self.root.clone(); - tokio::task::spawn_blocking(move || scan::scan(&root, params)) - .await - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - e.to_string(), - ) - })? - } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - { - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Unsupported, - "filesystem scan backend unavailable", - )) - } + pub fn confinement(&self) -> &Arc { + &self.inner.confinement } - pub async fn capture( - &self, - params: CaptureParams, - ) -> Result { - transfer::capture(self, params).await + pub fn cwd(&self) -> &Path { + &self.inner.cwd } - pub async fn materialize( + async fn blocking( &self, - params: MaterializeParams, - ) -> Result { - self.ensure_writable()?; - transfer::materialize(self, params).await + work: impl FnOnce(&Inner) -> Result + Send + 'static, + ) -> Result { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || work(&inner)) + .await + .map_err(|error| ProtocolError::internal(format!("filesystem worker: {error}")))? } pub async fn read_file( &self, + _: &RequestContext, params: ReadFileParams, - ) -> Result { - let path = self.resolve(¶ms.path)?; - let metadata = fs::metadata(&path) - .await - .map_err(|error| io_error(error, &path))?; - if !metadata.is_file() { - // Preserve the exact outcome a plain read produces for non-file - // paths (directories fail with the authentic io error). - return match fs::read(&path).await { - Ok(data) => Ok(ReadFileResponse { - file_size: Some(data.len() as u64), - data: ByteChunk::from(data), - truncated: false, - }), - Err(error) => Err(io_error(error, &path)), - }; - } - let file_size = metadata.len(); - let offset = params.offset.unwrap_or(0).min(file_size); - let remaining = file_size - offset; - let take = params.max_bytes.map_or(remaining, |max| max.min(remaining)); - - let data = if offset == 0 && take == file_size { - fs::read(&path) - .await - .map_err(|error| io_error(error, &path))? - } else { - use tokio::io::{AsyncReadExt, AsyncSeekExt}; - let mut file = fs::File::open(&path) - .await - .map_err(|error| io_error(error, &path))?; - file.seek(std::io::SeekFrom::Start(offset)) - .await - .map_err(|error| io_error(error, &path))?; - let mut data = Vec::with_capacity(take.min(64 * 1024 * 1024) as usize); - file.take(take) - .read_to_end(&mut data) - .await - .map_err(|error| io_error(error, &path))?; - data - }; - let truncated = offset.saturating_add(data.len() as u64) < file_size; - Ok(ReadFileResponse { - data: ByteChunk::from(data), - file_size: Some(file_size), - truncated, - }) + ) -> Result { + self.blocking(move |inner| inner.read_file(params)).await } pub async fn write_file( &self, + _: &RequestContext, params: WriteFileParams, - ) -> Result { - self.ensure_writable()?; - let path = self.resolve(¶ms.path)?; - fs::write(&path, params.data.into_inner()) - .await - .map_err(|error| io_error(error, &path))?; - Ok(WriteFileResponse {}) + ) -> Result { + self.blocking(move |inner| inner.write_file(params)).await } pub async fn create_directory( &self, + _: &RequestContext, params: CreateDirectoryParams, - ) -> Result { - self.ensure_writable()?; - let path = self.resolve(¶ms.path)?; - if params.recursive.unwrap_or(false) { - fs::create_dir_all(&path) - .await - .map_err(|error| io_error(error, &path))?; - } else { - fs::create_dir(&path) - .await - .map_err(|error| io_error(error, &path))?; - } - Ok(CreateDirectoryResponse {}) + ) -> Result { + self.blocking(move |inner| inner.create_directory(params)) + .await } pub async fn get_metadata( &self, + _: &RequestContext, params: GetMetadataParams, - ) -> Result { - let path = self.resolve(¶ms.path)?; - let metadata = fs::symlink_metadata(&path) - .await - .map_err(|error| io_error(error, &path))?; - let file_type = metadata.file_type(); - let modified = metadata.modified().unwrap_or(UNIX_EPOCH); - let created = metadata.created().unwrap_or(modified); - Ok(GetMetadataResponse { - is_directory: file_type.is_dir(), - is_file: file_type.is_file(), - is_symlink: file_type.is_symlink(), - created_at_ms: system_time_ms(created), - modified_at_ms: system_time_ms(modified), - }) + ) -> Result { + self.blocking(move |inner| inner.get_metadata(params)).await } pub async fn read_directory( &self, + _: &RequestContext, params: ReadDirectoryParams, - ) -> Result { - let path = self.resolve(¶ms.path)?; - let mut directory = fs::read_dir(&path) - .await - .map_err(|error| io_error(error, &path))?; - let mut entries = Vec::new(); - while let Some(entry) = directory - .next_entry() + ) -> Result { + self.blocking(move |inner| inner.read_directory(params)) .await - .map_err(|error| io_error(error, &path))? - { - let metadata = entry - .metadata() - .await - .map_err(|error| io_error(error, &entry.path()))?; - entries.push(ReadDirectoryEntry { - file_name: entry.file_name().to_string_lossy().into_owned(), - is_directory: metadata.is_dir(), - is_file: metadata.is_file(), - }); - } - entries.sort_by(|left, right| left.file_name.cmp(&right.file_name)); - Ok(ReadDirectoryResponse { entries }) } pub async fn remove( &self, + _: &RequestContext, params: RemoveParams, - ) -> Result { - self.ensure_writable()?; - let path = self.resolve(¶ms.path)?; - let metadata = match fs::symlink_metadata(&path).await { - Ok(metadata) => metadata, - Err(error) - if params.force.unwrap_or(false) && error.kind() == io::ErrorKind::NotFound => - { - return Ok(RemoveResponse {}); - } - Err(error) => return Err(io_error(error, &path)), - }; - if metadata.is_dir() { - if params.recursive.unwrap_or(false) { - fs::remove_dir_all(&path) - .await - .map_err(|error| io_error(error, &path))?; - } else { - fs::remove_dir(&path) - .await - .map_err(|error| io_error(error, &path))?; - } - } else { - fs::remove_file(&path) - .await - .map_err(|error| io_error(error, &path))?; - } - Ok(RemoveResponse {}) + ) -> Result { + self.blocking(move |inner| inner.remove(params)).await } - pub async fn copy(&self, params: CopyParams) -> Result { - self.ensure_writable()?; - let source = self.resolve(¶ms.source_path)?; - let destination = self.resolve(¶ms.destination_path)?; - tokio::task::spawn_blocking(move || copy_path(&source, &destination, params.recursive)) - .await - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - error.to_string(), - ) - })??; - Ok(CopyResponse {}) + pub async fn rename( + &self, + _: &RequestContext, + params: RenameParams, + ) -> Result { + self.blocking(move |inner| inner.rename(params)).await } - /// Bounded recursive text search executed in-process with the ripgrep - /// engine crates. No external `rg` binary is involved, and the regex - /// dialect matches the caller-side generic fallback. + /// Bounded recursive regex search with the ripgrep engine crates. pub async fn search_text( &self, + context: &RequestContext, params: SearchTextParams, - ) -> Result { - let root = self.resolve(¶ms.root)?; - tokio::task::spawn_blocking(move || search_text_blocking(root, params)) + ) -> Result { + search::check_search(¶ms, &self.inner.limits.search)?; + let cancel = context.cancel.clone(); + self.blocking(move |inner| search::search_text(&inner.confinement, params, &cancel)) .await - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - error.to_string(), - ) - })? } - /// Bounded recursive file enumeration executed locally, so a broad glob - /// does not become one `fs/readDirectory` round trip per directory. + /// Bounded recursive enumeration of entries matching `filter.include`. pub async fn glob_files( &self, + context: &RequestContext, params: GlobFilesParams, - ) -> Result { - let root = self.resolve(¶ms.root)?; - tokio::task::spawn_blocking(move || glob_files_blocking(root, params)) + ) -> Result { + search::check_glob(¶ms, &self.inner.limits.glob)?; + let cancel = context.cancel.clone(); + self.blocking(move |inner| search::glob_files(&inner.confinement, params, &cancel)) .await - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - error.to_string(), - ) - })? } - fn resolve_scoped_path( + pub async fn scan( &self, - path: &EnvironmentPath, - ) -> Result { - if Path::new(path.as_str()).is_absolute() { - return self.resolve(path); - } - // Cwd and root are administrator-owned configuration. Resolve their aliases, - // then retain the configured root spelling for scope checks and returned paths. - // User-selected components are still opened without following symlinks. - let root = self - .root - .canonicalize() - .map_err(|e| io_error(e, &self.root))?; - let cwd = self - .cwd - .canonicalize() - .map_err(|e| io_error(e, &self.cwd))?; - let relative = cwd.strip_prefix(&root).map_err(|_| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Forbidden, - "configured cwd is outside filesystem root", - ) - })?; - let path = EnvironmentPath::new( - self.root - .join(relative) - .join(path.as_str()) - .to_string_lossy(), - ) - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - e.to_string(), - ) - })?; - self.resolve(&path) + context: &RequestContext, + params: ScanParams, + ) -> Result { + scan::check_scan(¶ms, &self.inner.limits.scan)?; + let cancel = context.cancel.clone(); + self.blocking(move |inner| scan::scan(&inner.confinement, params, &cancel)) + .await } - fn resolve(&self, path: &EnvironmentPath) -> Result { - let candidate = if path.is_absolute() { - PathBuf::from(path.as_str()) - } else if path.as_str() == "." { - self.cwd.clone() - } else { - self.cwd.join(path.as_str()) - }; - let normalized = normalize_path(candidate); - if !normalized.starts_with(&self.root) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Forbidden, - format!( - "path is outside bridge fs root: {} (root {})", - normalized.display(), - self.root.display() - ), - )); - } - Ok(normalized) + pub async fn transfer_begin( + &self, + _: &RequestContext, + params: BeginTransferParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.begin(params)) + .await + } + + pub async fn transfer_advance( + &self, + _: &RequestContext, + params: TransferOperationParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.advance(params)) + .await + } + + pub async fn transfer_get_status( + &self, + _: &RequestContext, + params: TransferOperationParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.get_status(params)) + .await } - fn ensure_writable(&self) -> Result<(), EnvironmentProtocolError> { + pub async fn transfer_read_inventory( + &self, + _: &RequestContext, + params: ReadTransferInventoryParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.read_inventory(params)) + .await + } + + pub async fn transfer_append( + &self, + _: &RequestContext, + params: AppendTransferInventoryParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.append(params)) + .await + } + + pub async fn transfer_list_missing( + &self, + _: &RequestContext, + params: ListTransferMissingParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.list_missing(params)) + .await + } + + pub async fn transfer_read( + &self, + _: &RequestContext, + params: ReadTransferContentParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.read(params)) + .await + } + + pub async fn transfer_write( + &self, + _: &RequestContext, + params: WriteTransferContentParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.write(params)) + .await + } + + pub async fn transfer_commit( + &self, + _: &RequestContext, + params: TransferOperationParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.commit(params)) + .await + } + + pub async fn transfer_abort( + &self, + _: &RequestContext, + params: TransferOperationParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.abort(params)) + .await + } +} + +impl Inner { + fn ensure_writable(&self) -> Result<(), ProtocolError> { if self.writable { Ok(()) } else { - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::CapabilityUnavailable, - "bridge filesystem is read-only", + Err(ProtocolError::capability_unavailable( + Capability::FilesystemWrite, + "the filesystem is read-only", )) } } -} -fn search_text_blocking( - root: PathBuf, - params: SearchTextParams, -) -> Result { - if params.pattern.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "search pattern must not be empty", - )); + /// Open non-blocking, `fstat` the descriptor and refuse anything but a + /// regular file before reading; read at most `maxBytes` (plus one byte + /// to learn whether more follow). + fn read_file(&self, params: ReadFileParams) -> Result { + let ceiling = self.limits.max_read_file_bytes; + if params.max_bytes > ceiling { + return Err(ProtocolError::above_ceiling( + "maxBytes", + params.max_bytes, + ceiling, + )); + } + let path = ¶ms.path; + let resolved = self.confinement.lookup(path, Lookup::FOLLOW)?; + let stat = resolved.stat.expect("a followed lookup reports its target"); + require_regular(path, &stat)?; + let mut file = resolved + .parent + .open_read(&resolved.name) + .map_err(|error| io_error(error, path))?; + let opened = EntryStat::of_file(&file).map_err(|error| io_error(error, path))?; + require_regular(path, &opened)?; + if params.offset > 0 { + file.seek(SeekFrom::Start(params.offset)) + .map_err(|error| io_error(error, path))?; + } + let mut data = os::read_up_to(&mut file, params.max_bytes.saturating_add(1)) + .map_err(|error| io_error(error, path))?; + let read = data.len() as u64; + let more = read > params.max_bytes; + data.truncate(params.max_bytes as usize); + // Files such as procfs entries report size 0; their size is what + // could be read. Every other file's size is what `fstat` reported, + // also for an `offset` past its end. + let file_size = if opened.size == 0 && read > 0 { + params.offset.saturating_add(read) + } else { + opened.size + }; + let truncated = more || params.offset.saturating_add(data.len() as u64) < file_size; + Ok(ReadFileResponse { + data: data.into(), + file_size, + truncated, + version: opened.version(), + }) } - let limits = params.limits; - if limits.max_matches == 0 - || limits.max_files == 0 - || limits.max_bytes == 0 - || limits.max_duration_ms == 0 - { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "search limits must all be at least 1", + + /// Stage next to the final target, fsync, set the mode and rename over + /// it. + fn write_file(&self, params: WriteFileParams) -> Result { + self.ensure_writable()?; + let ceiling = self.limits.max_write_file_bytes; + let size = params.data.len() as u64; + if size > ceiling { + return Err(ProtocolError::above_ceiling("data", size, ceiling)); + } + let path = ¶ms.path; + let resolved = self + .confinement + .lookup(path, Lookup::FOLLOW.allowing_missing())?; + write_preconditions( + path, + resolved.stat.as_ref(), + params.create, + params.if_match.as_ref(), + )?; + let base = resolved + .stat + .map_or(os::DEFAULT_FILE_MODE, |stat| stat.mode); + let mode = if params.executable { + os::with_execute_bits(base) + } else { + base + }; + let rename = if params.create == WriteCondition::IfAbsent { + RenameMode::NoReplace + } else { + RenameMode::Replace + }; + let stage = OsString::from(format!( + "{WRITE_STAGE_PREFIX}{:032x}", + rand::random::() )); + let parent = &resolved.parent; + let io = |error| io_error(error, path); + let mut file = parent + .create_file(&stage, os::PRIVATE_FILE_MODE) + .map_err(io)?; + let staged = file + .write_all(params.data.as_slice()) + .and_then(|()| os::set_mode(&file, mode)) + .and_then(|()| file.sync_all()) + .and_then(|()| parent.rename(&stage, parent, &resolved.name, rename)); + if let Err(error) = staged { + let _ = parent.unlink(&stage); + if rename == RenameMode::NoReplace && error.kind() == io::ErrorKind::AlreadyExists { + return Err(create_condition(path, true)); + } + return Err(io(error)); + } + parent.sync().map_err(io)?; + let version = EntryStat::of_file(&file).map_err(io)?.version(); + Ok(WriteFileResponse { version }) } - let matcher = grep_regex::RegexMatcherBuilder::new() - .case_insensitive(!params.case_sensitive) - .build(¶ms.pattern) - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("invalid search regex: {error}"), - ) - })?; - let include = params - .include - .as_deref() - .map(glob::Pattern::new) - .transpose() - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("invalid search include glob: {error}"), - ) - })?; - - let started = Instant::now(); - let deadline = Duration::from_millis(limits.max_duration_ms); - let mut searcher = SearcherBuilder::new() - .line_number(true) - .binary_detection(BinaryDetection::quit(0)) - .build(); - let mut walker = ignore::WalkBuilder::new(&root); - // Mirror the generic traversal: no ignore-file or hidden filtering, no - // symlink following, deterministic order. The generic walker's max_depth - // counts directory descents below the root, so files at walker depth - // `max_depth + 1` are still in range. - walker - .standard_filters(false) - .follow_links(false) - .sort_by_file_path(|left, right| left.cmp(right)); - if let Some(max_depth) = params.max_depth { - walker.max_depth(Some((max_depth as usize).saturating_add(1))); + + fn create_directory( + &self, + params: CreateDirectoryParams, + ) -> Result { + self.ensure_writable()?; + let path = ¶ms.path; + let lookup = if params.recursive { + Lookup::FOLLOW.allowing_missing().creating_parents() + } else { + Lookup::FOLLOW.allowing_missing() + }; + let resolved = self.confinement.lookup(path, lookup)?; + match resolved.stat { + Some(stat) if params.recursive && stat.is_dir() => {} + Some(_) => { + return Err(ProtocolError::already_exists(format!("{path} exists"))); + } + None => match resolved + .parent + .mkdir(&resolved.name, os::DEFAULT_DIRECTORY_MODE) + { + Ok(()) => resolved + .parent + .sync() + .map_err(|error| io_error(error, path))?, + Err(error) + if params.recursive + && error.kind() == io::ErrorKind::AlreadyExists + && resolved + .parent + .stat(&resolved.name) + .is_ok_and(|stat| stat.is_dir()) => {} + Err(error) => return Err(io_error(error, path)), + }, + } + Ok(CreateDirectoryResponse {}) } - let mut matches = Vec::new(); - let mut files_searched = 0u64; - let mut bytes_searched = 0u64; - let mut stopped = None; + fn get_metadata( + &self, + params: GetMetadataParams, + ) -> Result { + let path = ¶ms.path; + let lookup = if params.follow_symlinks { + Lookup::FOLLOW + } else { + Lookup::NO_FOLLOW + }; + let resolved = self.confinement.lookup(path, lookup)?; + let stat = resolved + .parent + .stat_with_birth(&resolved.name) + .map_err(|error| io_error(error, path))?; + Ok(GetMetadataResponse { + metadata: metadata(&resolved.parent, &resolved.name, &stat), + }) + } - 'walk: for entry in walker.build() { - // Unreadable entries are skipped rather than failing the search. - let Ok(entry) = entry else { continue }; - if !entry.file_type().is_some_and(|kind| kind.is_file()) { - continue; - } - let path = entry.path(); - if let Some(include) = &include - && !local_path_matches_include(include, path, &root) - { - continue; + fn read_directory( + &self, + params: ReadDirectoryParams, + ) -> Result { + let ceiling = self.limits.max_directory_entries; + if params.limit == 0 { + return Err(ProtocolError::invalid_field( + "limit", + "limit must be at least 1", + )); } - if started.elapsed() >= deadline { - stopped = Some(SearchTextStop::TimeLimit); - break; + if params.limit > ceiling { + return Err(ProtocolError::above_ceiling( + "limit", + params.limit.into(), + ceiling.into(), + )); } - if files_searched >= limits.max_files { - stopped = Some(SearchTextStop::FileLimit); - break; + let position = params.cursor.as_ref().map(cursor::decode).transpose()?; + let path = ¶ms.path; + let resolved = self.confinement.lookup(path, Lookup::FOLLOW)?; + if !resolved.stat.is_some_and(|stat| stat.is_dir()) { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{path} is not a directory"), + )); } - let file_len = entry.metadata().map(|metadata| metadata.len()).unwrap_or(0); - if bytes_searched.saturating_add(file_len) > limits.max_bytes { - stopped = Some(SearchTextStop::ByteLimit); - break; + let directory = resolved + .open_directory() + .map_err(|error| io_error(error, path))?; + let mut names = directory.names().map_err(|error| io_error(error, path))?; + if resolved + .stat + .is_some_and(|stat| self.confinement.holds_state(&stat)) + { + names.retain(|name| { + directory + .stat(name) + .map_or(true, |stat| !self.confinement.is_state(&stat)) + }); } - files_searched += 1; - bytes_searched = bytes_searched.saturating_add(file_len); - - let match_path = response_path(¶ms.root, path, &root)?; - let mut hit_match_limit = false; - let sink = UTF8(|line_number, line| { - if matches.len() as u64 >= limits.max_matches { - hit_match_limit = true; - return Ok(false); - } - matches.push(SearchTextMatch { - path: match_path.clone(), - line_number, - line: line.trim_end_matches(['\r', '\n']).to_string(), + names.sort_by(|left, right| left.as_encoded_bytes().cmp(right.as_encoded_bytes())); + let start = position.map_or(0, |position| position.start(&names)); + let end = start.saturating_add(params.limit as usize).min(names.len()); + let page = &names[start.min(end)..end]; + let mut response = ReadDirectoryResponse { + entries: Vec::with_capacity(page.len()), + next_cursor: (end < names.len()) + .then(|| { + page.last() + .map(|name| cursor::encode(name.as_encoded_bytes())) + }) + .flatten(), + skipped_entry_count: 0, + }; + for name in page { + let Some(entry_name) = os::utf8_name(name).and_then(|name| EntryName::new(name).ok()) + else { + response.skipped_entry_count += 1; + continue; + }; + let stat = match directory.stat_with_birth(name) { + Ok(stat) => stat, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(io_error(error, format!("{path}/{entry_name}"))), + }; + response.entries.push(DirectoryEntry { + name: entry_name, + metadata: metadata(&directory, name, &stat), }); - Ok(true) - }); - // Per-file search errors (e.g. the file vanished mid-walk) skip the - // file rather than failing the whole search. - let _ = searcher.search_path(&matcher, path, sink); - // Like the generic fallback, the match limit reports truncation only - // when a match beyond the cap actually arrived. - if hit_match_limit { - stopped = Some(SearchTextStop::MatchLimit); - break 'walk; } + Ok(response) } - Ok(SearchTextResponse { - matches, - files_searched, - bytes_searched, - elapsed_ms: started.elapsed().as_millis() as u64, - stopped, - }) -} - -fn glob_files_blocking( - root: PathBuf, - params: GlobFilesParams, -) -> Result { - if params.pattern.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "glob pattern must not be empty", - )); - } - let limits = params.limits; - if limits.max_matches == 0 || limits.max_entries == 0 || limits.max_duration_ms == 0 { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "glob limits must all be at least 1", - )); - } - let pattern = glob::Pattern::new(¶ms.pattern).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("invalid glob pattern: {error}"), - ) - })?; - - let started = Instant::now(); - let deadline = Duration::from_millis(limits.max_duration_ms); - let mut walker = ignore::WalkBuilder::new(&root); - walker - .standard_filters(false) - .follow_links(false) - .sort_by_file_path(|left, right| left.cmp(right)); - if let Some(max_depth) = params.max_depth { - walker.max_depth(Some((max_depth as usize).saturating_add(1))); - } - - let mut matches = Vec::new(); - let mut entries_visited = 0u64; - let mut stopped = None; - - for entry in walker.build() { - let Ok(entry) = entry else { continue }; - if entry.depth() == 0 { - continue; + fn remove(&self, params: RemoveParams) -> Result { + self.ensure_writable()?; + let path = ¶ms.path; + let resolved = match self + .confinement + .lookup(path, Lookup::NO_FOLLOW.allowing_missing()) + { + Ok(resolved) => resolved, + Err(error) + if params.force + && matches!( + error.data, + environment_protocol::error::ErrorData::NotFound { .. } + ) => + { + return Ok(RemoveResponse {}); + } + Err(error) => return Err(error), + }; + if resolved.is_root() { + return Err(filesystem_root()); } - if started.elapsed() >= deadline { - stopped = Some(GlobFilesStop::TimeLimit); - break; + self.confinement.refuse_containing_state(&resolved, path)?; + let io = |error| io_error(error, path); + match resolved.stat { + None if params.force => return Ok(RemoveResponse {}), + None => return Err(ProtocolError::not_found(format!("{path} does not exist"))), + Some(stat) if stat.is_dir() => { + if params.recursive { + resolved.parent.remove_tree(&resolved.name).map_err(io)?; + } else { + resolved.parent.rmdir(&resolved.name).map_err(io)?; + } + } + Some(_) => resolved.parent.unlink(&resolved.name).map_err(io)?, } - if entries_visited >= limits.max_entries { - stopped = Some(GlobFilesStop::EntryLimit); - break; + resolved.parent.sync().map_err(io)?; + Ok(RemoveResponse {}) + } + + fn rename(&self, params: RenameParams) -> Result { + self.ensure_writable()?; + let source = self.confinement.lookup(¶ms.source, Lookup::NO_FOLLOW)?; + let destination = self + .confinement + .lookup(¶ms.destination, Lookup::NO_FOLLOW.allowing_missing())?; + if source.is_root() || destination.is_root() { + return Err(filesystem_root()); } - entries_visited += 1; - if !entry.file_type().is_some_and(|kind| kind.is_file()) { - continue; + self.confinement + .refuse_containing_state(&source, ¶ms.source)?; + self.confinement + .refuse_containing_state(&destination, ¶ms.destination)?; + let source_stat = source.stat.expect("an existing source has a stat"); + if let Some(expected) = ¶ms.if_match + && source_stat.version() != *expected + { + return Err(ProtocolError::failed_precondition( + PreconditionReason::IfMatchFailed, + format!("{} changed since that version", params.source), + )); } - let path = entry.path(); - let caller_path = response_path(¶ms.root, path, &root)?; - if !glob_pattern_matches(&pattern, ¶ms.pattern, &caller_path, path, &root) { - continue; + if destination.segments.starts_with(&source.segments) + && destination.segments != source.segments + { + return Err(ProtocolError::invalid_field( + "destination", + "a directory cannot move into itself", + )); } - // Like the generic fallback, the match limit reports truncation only - // when a match beyond the cap actually arrives. - if matches.len() as u64 >= limits.max_matches { - stopped = Some(GlobFilesStop::MatchLimit); - break; + if let Some(stat) = &destination.stat { + if stat.is_dir() { + return Err(ProtocolError::failed_precondition( + PreconditionReason::IsDirectory, + format!( + "{} is a directory and is never replaced", + params.destination + ), + )); + } + if !params.overwrite { + return Err(ProtocolError::already_exists(format!( + "{} exists", + params.destination + ))); + } } - matches.push(caller_path); + let mode = if params.overwrite { + RenameMode::Replace + } else { + RenameMode::NoReplace + }; + source + .parent + .rename(&source.name, &destination.parent, &destination.name, mode) + .map_err(|error| io_error(error, ¶ms.destination))?; + source + .parent + .sync() + .and_then(|()| destination.parent.sync()) + .map_err(|error| io_error(error, ¶ms.destination))?; + Ok(RenameResponse {}) } +} - Ok(GlobFilesResponse { - matches, - entries_visited, - elapsed_ms: started.elapsed().as_millis() as u64, - stopped, - }) +fn require_regular(path: &EnvironmentPath, stat: &EntryStat) -> Result<(), ProtocolError> { + match stat.kind { + FileKind::File => Ok(()), + FileKind::Directory => Err(ProtocolError::failed_precondition( + PreconditionReason::IsDirectory, + format!("{path} is a directory"), + )), + FileKind::Symlink | FileKind::Other => Err(ProtocolError::failed_precondition( + PreconditionReason::NotRegularFile, + format!("{path} is not a regular file"), + )), + } } -/// Glob pattern semantics mirror the caller-side generic tool: an absolute -/// pattern matches the caller-space path; otherwise the root-relative path -/// matches, and a pattern without `/` also matches the bare file name. -fn glob_pattern_matches( - pattern: &glob::Pattern, - pattern_text: &str, - caller_path: &EnvironmentPath, - path: &Path, - root: &Path, -) -> bool { - if pattern_text.starts_with('/') { - return pattern.matches(caller_path.as_str()); +/// The checks of `fs/writeFile` and the transfer `writeFile` selection on +/// the final target as it is now. +pub(crate) fn write_preconditions( + path: &EnvironmentPath, + current: Option<&EntryStat>, + create: WriteCondition, + if_match: Option<&FileVersion>, +) -> Result<(), ProtocolError> { + if let Some(stat) = current { + require_regular(path, stat)?; } - let relative = path - .strip_prefix(root) - .map(|relative| { - relative - .to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/") - }) - .unwrap_or_else(|_| path.to_string_lossy().into_owned()); - pattern.matches(&relative) - || (!pattern_text.contains('/') - && path - .file_name() - .is_some_and(|file_name| pattern.matches(&file_name.to_string_lossy()))) + match (current, create) { + (Some(_), WriteCondition::IfAbsent) => return Err(create_condition(path, true)), + (None, WriteCondition::IfPresent) => return Err(create_condition(path, false)), + _ => {} + } + if let Some(expected) = if_match + && current.is_none_or(|stat| stat.version() != *expected) + { + return Err(ProtocolError::failed_precondition( + PreconditionReason::IfMatchFailed, + format!("{path} is not at the expected version"), + )); + } + Ok(()) } -/// Include globs match the root-relative path or the bare file name, exactly -/// like the caller-side generic fallback. -fn local_path_matches_include(pattern: &glob::Pattern, path: &Path, root: &Path) -> bool { - let relative = path - .strip_prefix(root) - .map(|relative| { - relative - .to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/") - }) - .unwrap_or_else(|_| path.to_string_lossy().into_owned()); - pattern.matches(&relative) - || path - .file_name() - .is_some_and(|file_name| pattern.matches(&file_name.to_string_lossy())) +fn create_condition(path: &EnvironmentPath, exists: bool) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::CreateCondition, + if exists { + format!("{path} exists") + } else { + format!("{path} does not exist") + }, + ) } -/// Map a matched local path back into the caller's path space by joining its -/// root-relative suffix onto the requested root. -fn response_path( - requested_root: &EnvironmentPath, - path: &Path, - resolved_root: &Path, -) -> Result { - let relative = path.strip_prefix(resolved_root).map_err(|_| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - format!("search match escaped its root: {}", path.display()), - ) - })?; - let relative = relative - .to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/"); - if relative.is_empty() { - return Ok(requested_root.clone()); +fn filesystem_root() -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::FilesystemRoot, + "the filesystem root cannot be removed, renamed or replaced", + ) +} + +fn metadata(parent: &Directory, name: &std::ffi::OsStr, stat: &EntryStat) -> FileMetadata { + let symlink_target = (stat.kind == FileKind::Symlink) + .then(|| parent.read_link(name).ok()) + .flatten() + .and_then(|target| target.into_string().ok()); + FileMetadata { + kind: stat.kind, + size_bytes: stat.size, + executable: stat.kind == FileKind::File && stat.executable(), + modified_at_ms: stat.modified_ms(), + created_at_ms: stat.birth_ms, + symlink_target, + version: stat.version(), } - let joined = format!( - "{}/{}", - requested_root.as_str().trim_end_matches('/'), - relative - ); - EnvironmentPath::new(&joined).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - format!("search match produced an invalid environment path {joined}: {error}"), - ) - }) } -fn copy_path( - source: &Path, - destination: &Path, - recursive: bool, -) -> Result<(), EnvironmentProtocolError> { - let metadata = std::fs::symlink_metadata(source).map_err(|error| io_error(error, source))?; - if metadata.is_dir() { - if !recursive { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "copy requires recursive=true when source is a directory", - )); +/// Bytes `value` takes as a JSON string, quotes included: `"` and `\` +/// and the short control escapes take two bytes, every other control +/// character six (`\u00XX`). What response budgets charge, so an answer +/// within its budget also encodes within it. +pub(crate) fn json_string_len(value: &str) -> u64 { + let escapes: u64 = value + .bytes() + .map(|byte| match byte { + b'"' | b'\\' | b'\n' | b'\r' | b'\t' | 0x08 | 0x0c => 1, + 0x00..=0x1f => 5, + _ => 0, + }) + .sum(); + value.len() as u64 + escapes + 2 +} + +/// envd's one mapping of OS errors to protocol error kinds. +pub(crate) fn io_error(error: io::Error, subject: impl Display) -> ProtocolError { + let message = format!("{subject}: {error}"); + let precondition = |reason| ProtocolError::failed_precondition(reason, message.clone()); + match Errno::from_io_error(&error) { + Some(Errno::LOOP) => return precondition(PreconditionReason::SymlinkLoop), + Some(Errno::NOTDIR) => return precondition(PreconditionReason::NotDirectory), + Some(Errno::ISDIR) => return precondition(PreconditionReason::IsDirectory), + Some(Errno::NOTEMPTY) => return precondition(PreconditionReason::NotEmpty), + Some(Errno::NOSPC | Errno::DQUOT) => return precondition(PreconditionReason::NoSpace), + Some(Errno::XDEV) => return precondition(PreconditionReason::CrossDevice), + Some(Errno::ROFS) => { + return ProtocolError::capability_unavailable(Capability::FilesystemWrite, message); + } + _ => {} + } + match error.kind() { + io::ErrorKind::NotFound => ProtocolError::not_found(message), + io::ErrorKind::AlreadyExists => ProtocolError::already_exists(message), + io::ErrorKind::PermissionDenied => { + ProtocolError::permission_denied(Some(PermissionReason::FilePermission), message) } - copy_directory(source, destination) - } else { - if let Some(parent) = destination.parent() { - std::fs::create_dir_all(parent).map_err(|error| io_error(error, parent))?; + io::ErrorKind::Interrupted | io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut => { + ProtocolError::unavailable(message) } - std::fs::copy(source, destination) - .map(|_| ()) - .map_err(|error| io_error(error, destination)) + _ => ProtocolError::internal(message), } } -fn copy_directory(source: &Path, destination: &Path) -> Result<(), EnvironmentProtocolError> { - std::fs::create_dir_all(destination).map_err(|error| io_error(error, destination))?; - for entry in std::fs::read_dir(source).map_err(|error| io_error(error, source))? { - let entry = entry.map_err(|error| io_error(error, source))?; - let source_child = entry.path(); - let destination_child = destination.join(entry.file_name()); - let metadata = std::fs::symlink_metadata(&source_child) - .map_err(|error| io_error(error, &source_child))?; - if metadata.is_dir() { - copy_directory(&source_child, &destination_child)?; +/// `fs/readDirectory` cursors: the last name returned, as hex. Names too +/// long for the 255-byte cursor carry a prefix and the name's SHA-256. +mod cursor { + use super::*; + + /// Longest name encoded in full (`x` and two hex digits per byte). + const MAX_EXACT_NAME_BYTES: usize = 127; + const PREFIX_BYTES: usize = 64; + + pub(super) enum Position { + After(Vec), + AfterHashed { prefix: Vec, sha256: Vec }, + } + + pub(super) fn encode(name: &[u8]) -> DirectoryCursor { + let value = if name.len() <= MAX_EXACT_NAME_BYTES { + format!("x{}", hex::encode(name)) } else { - std::fs::copy(&source_child, &destination_child) - .map(|_| ()) - .map_err(|error| io_error(error, &destination_child))?; - } + format!( + "h{}.{}", + hex::encode(&name[..PREFIX_BYTES]), + hex::encode(Sha256::digest(name)) + ) + }; + DirectoryCursor::new(value).expect("hex cursors are short and printable") } - Ok(()) -} -fn normalize_path(path: PathBuf) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(std::path::MAIN_SEPARATOR.to_string()), - Component::CurDir => {} - Component::ParentDir => { - normalized.pop(); - } - Component::Normal(segment) => normalized.push(segment), + pub(super) fn decode(cursor: &DirectoryCursor) -> Result { + let malformed = + || ProtocolError::invalid_field("cursor", "not a cursor this daemon minted"); + let value = cursor.as_str(); + if let Some(exact) = value.strip_prefix('x') { + return hex::decode(exact) + .map(Position::After) + .map_err(|_| malformed()); + } + let (prefix, sha256) = value + .strip_prefix('h') + .and_then(|hashed| hashed.split_once('.')) + .ok_or_else(malformed)?; + let prefix = hex::decode(prefix).map_err(|_| malformed())?; + let sha256 = hex::decode(sha256).map_err(|_| malformed())?; + if prefix.len() != PREFIX_BYTES || sha256.len() != 32 { + return Err(malformed()); } + Ok(Position::AfterHashed { prefix, sha256 }) } - normalized -} -fn io_error(error: io::Error, path: &Path) -> EnvironmentProtocolError { - let code = match error.kind() { - io::ErrorKind::NotFound => EnvironmentProtocolErrorCode::NotFound, - io::ErrorKind::PermissionDenied => EnvironmentProtocolErrorCode::Forbidden, - io::ErrorKind::AlreadyExists => EnvironmentProtocolErrorCode::Conflict, - io::ErrorKind::InvalidInput | io::ErrorKind::InvalidData => { - EnvironmentProtocolErrorCode::InvalidRequest + impl Position { + /// Index of the first name after this position in sorted `names`. + pub(super) fn start(&self, names: &[OsString]) -> usize { + match self { + Self::After(last) => { + names.partition_point(|name| name.as_encoded_bytes() <= last.as_slice()) + } + Self::AfterHashed { prefix, sha256 } => names + .iter() + .position(|name| { + let bytes = name.as_encoded_bytes(); + bytes.starts_with(prefix) && Sha256::digest(bytes).as_slice() == sha256 + }) + .map(|index| index + 1) + // The last name vanished: continue after its prefix. + .unwrap_or_else(|| { + names.partition_point(|name| name.as_encoded_bytes() <= prefix.as_slice()) + }), + } } - _ => EnvironmentProtocolErrorCode::Internal, - }; - EnvironmentProtocolError::new(code, format!("{}: {}", path.display(), error)) -} - -fn system_time_ms(value: SystemTime) -> i64 { - value - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_millis().min(i64::MAX as u128) as i64) - .unwrap_or_default() + } } #[cfg(test)] mod tests { + use std::os::unix::fs::{PermissionsExt, symlink}; + + use environment_protocol::{ + data::handshake::{GlobCeilings, ScanCeilings, SearchCeilings, TransferCeilings}, + error::{ErrorData, ProtocolErrorKind}, + }; + use super::*; - #[tokio::test(flavor = "current_thread")] - async fn filesystem_reads_and_writes_under_root() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let fs = LocalFileSystem::new(root.clone(), root.clone(), true); - let path = EnvironmentPath::new(root.join("file.txt").to_string_lossy()) - .expect("environment path"); - - fs.write_file(WriteFileParams { - path: path.clone(), - data: ByteChunk::from(b"hello".as_slice()), - }) - .await - .expect("write"); - let read = fs - .read_file(ReadFileParams { - path, - offset: None, - max_bytes: None, - }) - .await - .expect("read") - .data - .into_inner(); + pub(crate) fn limits() -> FilesystemLimits { + FilesystemLimits { + max_read_file_bytes: 8 * 1024 * 1024, + max_write_file_bytes: 8 * 1024 * 1024, + max_directory_entries: 1000, + search: SearchCeilings { + max_matches: 10_000, + max_files: 100_000, + max_bytes: 1 << 30, + max_duration_ms: 60_000, + max_line_bytes: 4096, + }, + glob: GlobCeilings { + max_matches: 100_000, + max_files: 1_000_000, + max_duration_ms: 60_000, + }, + scan: ScanCeilings { + max_roots: 32, + max_patterns: 32, + max_entries: 10_000, + max_file_bytes: 256 * 1024, + max_total_bytes: 4 * 1024 * 1024, + max_duration_ms: 30_000, + }, + transfer: TransferCeilings { + max_active_operations: 8, + max_content_chunk_bytes: 4 * 1024 * 1024, + max_inventory_page_entries: 1000, + receipt_retention_ms: 86_400_000, + max_entries: 100_000, + max_depth: 64, + max_file_bytes: 1 << 40, + max_total_bytes: 1 << 40, + max_manifest_bytes: 32 * 1024 * 1024, + max_duration_ms: 86_400_000, + }, + } + } - assert_eq!(read, b"hello"); + pub(crate) fn context() -> RequestContext { + RequestContext::detached() } - #[tokio::test(flavor = "current_thread")] - async fn filesystem_rejects_root_escape() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().join("root"); - std::fs::create_dir(&root).expect("root"); - let root = root.canonicalize().expect("canonical root"); - let fs = LocalFileSystem::new(root, temp.path().to_path_buf(), true); - let outside = - EnvironmentPath::new(temp.path().join("outside.txt").to_string_lossy()).unwrap(); + /// A filesystem rooted at `root` with its state directory inside. + pub(crate) fn filesystem(root: &Path, writable: bool) -> LocalFileSystem { + let confinement = + Arc::new(Confinement::new(root.to_path_buf(), root.join(".state")).unwrap()); + LocalFileSystem::new(FilesystemConfig { + confinement, + cwd: root.to_path_buf(), + writable, + limits: limits(), + }) + .unwrap() + } - let error = fs - .read_file(ReadFileParams { - path: outside, - offset: None, - max_bytes: None, - }) - .await - .expect_err("escape should fail"); + fn path(root: &Path, relative: &str) -> EnvironmentPath { + EnvironmentPath::new(format!("{}/{relative}", root.display())).unwrap() + } - assert_eq!(error.code, EnvironmentProtocolErrorCode::Forbidden); + fn reason(error: &ProtocolError) -> Option { + match error.data { + ErrorData::FailedPrecondition { reason } => Some(reason), + _ => None, + } } - fn search_fixture() -> (tempfile::TempDir, LocalFileSystem, EnvironmentPath) { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - std::fs::create_dir_all(root.join("src/deep")).expect("dirs"); - std::fs::write( - root.join("src/lib.rs"), - "pub fn target() {}\nfn other() {}\n", - ) - .expect("lib"); - std::fs::write(root.join("src/deep/nested.rs"), "target nested\n").expect("nested"); - std::fs::write(root.join("readme.md"), "target readme\n").expect("readme"); - std::fs::write(root.join("binary.bin"), b"tar\x00get\n").expect("binary"); - let fs = LocalFileSystem::new(root.clone(), root.clone(), false); - let environment_root = - EnvironmentPath::new(root.to_string_lossy()).expect("environment root"); - (temp, fs, environment_root) + fn read(path: EnvironmentPath, max_bytes: u64) -> ReadFileParams { + ReadFileParams { + path, + offset: 0, + max_bytes, + } } - fn limits() -> environment_protocol::data::fs::SearchTextLimits { - environment_protocol::data::fs::SearchTextLimits { - max_matches: 100, - max_files: 100, - max_bytes: 1024 * 1024, - max_duration_ms: 10_000, + fn write(path: EnvironmentPath, data: &[u8]) -> WriteFileParams { + WriteFileParams { + path, + data: data.into(), + create: WriteCondition::Always, + if_match: None, + executable: false, } } - fn search_params(root: &EnvironmentPath) -> SearchTextParams { - SearchTextParams { - root: root.clone(), - pattern: "target".to_owned(), - include: None, - case_sensitive: true, - max_depth: None, + #[tokio::test(flavor = "current_thread")] + async fn special_files_are_refused_before_any_read() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + // The root is `/` so a workspace link to a device stays inside it. + let confinement = Arc::new(Confinement::new("/".into(), base.join("state")).unwrap()); + let fs = LocalFileSystem::new(FilesystemConfig { + confinement, + cwd: base.clone(), + writable: false, limits: limits(), + }) + .unwrap(); + symlink("/dev/zero", base.join("zero")).unwrap(); + let status = std::process::Command::new("mkfifo") + .arg(base.join("fifo")) + .status() + .unwrap(); + assert!(status.success()); + for subject in [ + EnvironmentPath::new("/dev/zero").unwrap(), + path(&base, "zero"), + path(&base, "fifo"), + ] { + let error = fs + .read_file(&context(), read(subject.clone(), 1024)) + .await + .unwrap_err(); + assert_eq!( + reason(&error), + Some(PreconditionReason::NotRegularFile), + "{subject}" + ); } + let error = fs + .read_file( + &context(), + read(EnvironmentPath::new(base.to_str().unwrap()).unwrap(), 1), + ) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::IsDirectory)); + } + + #[cfg(target_os = "linux")] + #[tokio::test(flavor = "current_thread")] + async fn zero_size_proc_files_report_the_bytes_read() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let confinement = Arc::new(Confinement::new("/".into(), base.join("state")).unwrap()); + let fs = LocalFileSystem::new(FilesystemConfig { + confinement, + cwd: base, + writable: false, + limits: limits(), + }) + .unwrap(); + let response = fs + .read_file( + &context(), + read(EnvironmentPath::new("/proc/self/status").unwrap(), 1 << 20), + ) + .await + .unwrap(); + assert!(!response.data.is_empty()); + assert_eq!(response.file_size, response.data.len() as u64); + assert!(!response.truncated); } #[tokio::test(flavor = "current_thread")] - async fn search_finds_matches_with_include_and_depth_semantics() { - let (_temp, fs, root) = search_fixture(); - - let mut params = search_params(&root); - params.include = Some("*.rs".to_owned()); - let result = fs.search_text(params).await.expect("search"); - let mut paths = result - .matches - .iter() - .map(|item| item.path.as_str().to_owned()) - .collect::>(); - paths.sort(); + async fn ranged_reads_bound_every_read_and_report_truncation() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + std::fs::write(root.join("file"), b"0123456789").unwrap(); + let ranged = fs + .read_file( + &context(), + ReadFileParams { + path: path(&root, "file"), + offset: 2, + max_bytes: 3, + }, + ) + .await + .unwrap(); + assert_eq!(ranged.data.as_slice(), b"234"); + assert_eq!(ranged.file_size, 10); + assert!(ranged.truncated); + let past_end = fs + .read_file( + &context(), + ReadFileParams { + path: path(&root, "file"), + offset: 100, + max_bytes: 3, + }, + ) + .await + .unwrap(); + assert!(past_end.data.as_slice().is_empty()); assert_eq!( - paths, - vec![ - format!("{}/src/deep/nested.rs", root.as_str()), - format!("{}/src/lib.rs", root.as_str()), - ] - ); - assert_eq!(result.matches[0].line_number, 1); - assert!(result.stopped.is_none()); - // NUL-containing files are treated as binary and skipped, so the - // binary fixture never matches. - assert!( - result - .matches - .iter() - .all(|item| !item.path.as_str().ends_with(".bin")) + past_end.file_size, 10, + "the size of the file, not the offset" ); - - // max_depth 0 mirrors the generic traversal: files directly under - // the root only. - let mut params = search_params(&root); - params.max_depth = Some(0); - let shallow = fs.search_text(params).await.expect("shallow search"); - assert_eq!(shallow.matches.len(), 1); - assert!(shallow.matches[0].path.as_str().ends_with("readme.md")); + assert!(!past_end.truncated); + let whole = fs + .read_file(&context(), read(path(&root, "file"), 10)) + .await + .unwrap(); + assert!(!whole.truncated); + let error = fs + .read_file(&context(), read(path(&root, "file"), u64::MAX)) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::InvalidParams { + ceiling: Some(_), + .. + } + )); } #[tokio::test(flavor = "current_thread")] - async fn search_stops_at_each_limit_with_a_typed_reason() { - let (_temp, fs, root) = search_fixture(); - - let mut params = search_params(&root); - params.limits.max_matches = 1; - let matches = fs.search_text(params).await.expect("match limit"); - assert_eq!(matches.matches.len(), 1); - assert_eq!(matches.stopped, Some(SearchTextStop::MatchLimit)); - - let mut params = search_params(&root); - params.limits.max_files = 1; - let files = fs.search_text(params).await.expect("file limit"); - assert_eq!(files.stopped, Some(SearchTextStop::FileLimit)); - assert_eq!(files.files_searched, 1); - - let mut params = search_params(&root); - params.limits.max_bytes = 1; - let bytes = fs.search_text(params).await.expect("byte limit"); - assert_eq!(bytes.stopped, Some(SearchTextStop::ByteLimit)); - assert!(bytes.matches.is_empty()); + async fn writes_are_atomic_conditional_and_keep_links_and_modes() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + std::fs::create_dir(root.join("src")).unwrap(); + std::fs::write(root.join("src/tool.sh"), b"old").unwrap(); + std::fs::set_permissions( + root.join("src/tool.sh"), + std::fs::Permissions::from_mode(0o700), + ) + .unwrap(); + symlink("src/tool.sh", root.join("link")).unwrap(); + + let current = fs + .read_file(&context(), read(path(&root, "link"), 16)) + .await + .unwrap() + .version; + let written = fs + .write_file( + &context(), + WriteFileParams { + if_match: Some(current.clone()), + ..write(path(&root, "link"), b"new") + }, + ) + .await + .unwrap(); + assert!( + std::fs::symlink_metadata(root.join("link")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(std::fs::read(root.join("src/tool.sh")).unwrap(), b"new"); + let mode = std::fs::metadata(root.join("src/tool.sh")) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o700); + assert_ne!(written.version, current); + + let stale = fs + .write_file( + &context(), + WriteFileParams { + if_match: Some(current), + ..write(path(&root, "src/tool.sh"), b"lost") + }, + ) + .await + .unwrap_err(); + assert_eq!(reason(&stale), Some(PreconditionReason::IfMatchFailed)); + let exists = fs + .write_file( + &context(), + WriteFileParams { + create: WriteCondition::IfAbsent, + ..write(path(&root, "src/tool.sh"), b"x") + }, + ) + .await + .unwrap_err(); + assert_eq!(reason(&exists), Some(PreconditionReason::CreateCondition)); + let missing = fs + .write_file( + &context(), + WriteFileParams { + create: WriteCondition::IfPresent, + ..write(path(&root, "src/new"), b"x") + }, + ) + .await + .unwrap_err(); + assert_eq!(reason(&missing), Some(PreconditionReason::CreateCondition)); + fs.write_file( + &context(), + WriteFileParams { + executable: true, + ..write(path(&root, "src/run"), b"#!/bin/sh\n") + }, + ) + .await + .unwrap(); + let mode = std::fs::metadata(root.join("src/run")) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o755); + let error = fs + .write_file(&context(), write(path(&root, "src"), b"x")) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::IsDirectory)); + let error = fs + .write_file(&context(), write(path(&root, "absent/file"), b"x")) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + // No staging file is left behind. + let leftovers: Vec<_> = std::fs::read_dir(root.join("src")) + .unwrap() + .flatten() + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(WRITE_STAGE_PREFIX) + }) + .collect(); + assert!(leftovers.is_empty()); } #[tokio::test(flavor = "current_thread")] - async fn search_rejects_invalid_requests_and_root_escape() { - let (temp, fs, root) = search_fixture(); - - let mut params = search_params(&root); - params.pattern = "(unclosed".to_owned(); - let error = fs.search_text(params).await.expect_err("invalid regex"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::InvalidRequest); - - let mut params = search_params(&root); - params.limits.max_matches = 0; - let error = fs.search_text(params).await.expect_err("zero limit"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::InvalidRequest); - - let nested_root = temp.path().join("root"); - std::fs::create_dir(&nested_root).expect("nested root"); - let nested_root = nested_root.canonicalize().expect("canonical nested root"); - let confined = LocalFileSystem::new(nested_root, temp.path().to_path_buf(), false); - let error = confined - .search_text(search_params(&root)) + async fn rename_never_replaces_directories_or_unasked_files() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + std::fs::create_dir(root.join("dir")).unwrap(); + std::fs::write(root.join("a"), b"a").unwrap(); + std::fs::write(root.join("b"), b"b").unwrap(); + let rename = |source: &str, destination: &str, overwrite: bool| RenameParams { + source: path(&root, source), + destination: path(&root, destination), + overwrite, + if_match: None, + }; + let error = fs + .rename(&context(), rename("a", "b", false)) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::AlreadyExists); + let error = fs + .rename(&context(), rename("a", "dir", true)) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::IsDirectory)); + fs.rename(&context(), rename("a", "b", true)).await.unwrap(); + assert_eq!(std::fs::read(root.join("b")).unwrap(), b"a"); + fs.rename(&context(), rename("b", "dir/c", false)) + .await + .unwrap(); + assert!(root.join("dir/c").exists()); + let error = fs + .rename(&context(), rename(".state", "moved", false)) .await - .expect_err("escape should fail"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::Forbidden); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::PermissionDenied); } - fn glob_params(root: &EnvironmentPath, pattern: &str) -> GlobFilesParams { - GlobFilesParams { - root: root.clone(), - pattern: pattern.to_owned(), - max_depth: None, - limits: environment_protocol::data::fs::GlobFilesLimits { - max_matches: 100, - max_entries: 100, - max_duration_ms: 10_000, + #[tokio::test(flavor = "current_thread")] + async fn removal_refuses_the_root_and_the_state_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + let remove = |path: EnvironmentPath| RemoveParams { + path, + recursive: true, + force: false, + }; + let error = fs + .remove( + &context(), + remove(EnvironmentPath::new(root.to_str().unwrap()).unwrap()), + ) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::FilesystemRoot)); + let error = fs + .remove(&context(), remove(path(&root, ".state"))) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::DaemonState) + } + )); + fs.remove( + &context(), + RemoveParams { + force: true, + ..remove(path(&root, "missing/deeper")) }, - } + ) + .await + .unwrap(); + let read_only = filesystem(&root, false); + let error = read_only + .remove(&context(), remove(path(&root, "anything"))) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::CapabilityUnavailable); } #[tokio::test(flavor = "current_thread")] - async fn glob_enumerates_with_generic_pattern_semantics_and_limits() { - let (_temp, fs, root) = search_fixture(); - - let result = fs - .glob_files(glob_params(&root, "**/*.rs")) - .await - .expect("glob"); - let mut paths = result - .matches - .iter() - .map(|path| path.as_str().to_owned()) - .collect::>(); - paths.sort(); - assert_eq!( - paths, - vec![ - format!("{}/src/deep/nested.rs", root.as_str()), - format!("{}/src/lib.rs", root.as_str()), - ] - ); - assert!(result.stopped.is_none()); - assert!(result.entries_visited >= 4); - - // A pattern without '/' also matches bare file names anywhere. - let by_name = fs - .glob_files(glob_params(&root, "*.md")) + async fn directories_page_by_cursor_and_hide_the_state_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + for name in ["c", "a", "b\\slash", "d"] { + std::fs::write(root.join(name), name).unwrap(); + } + symlink("a", root.join("link")).unwrap(); + let long = "n".repeat(200); + std::fs::write(root.join(&long), "long").unwrap(); + let mut names = Vec::new(); + let mut cursor = None; + loop { + let page = fs + .read_directory( + &context(), + ReadDirectoryParams { + path: EnvironmentPath::new(root.to_str().unwrap()).unwrap(), + limit: 2, + cursor: cursor.take(), + }, + ) + .await + .unwrap(); + names.extend( + page.entries + .iter() + .map(|entry| (entry.name.as_str().to_owned(), entry.metadata.kind)), + ); + match page.next_cursor { + Some(next) => cursor = Some(next), + None => break, + } + } + let expected: Vec<(String, FileKind)> = [ + ("a", FileKind::File), + ("b\\slash", FileKind::File), + ("c", FileKind::File), + ("d", FileKind::File), + ("link", FileKind::Symlink), + (long.as_str(), FileKind::File), + ] + .into_iter() + .map(|(name, kind)| (name.to_owned(), kind)) + .collect(); + assert_eq!(names, expected); + let error = fs + .read_directory( + &context(), + ReadDirectoryParams { + path: EnvironmentPath::new(root.to_str().unwrap()).unwrap(), + limit: 1, + cursor: Some(DirectoryCursor::new("garbage").unwrap()), + }, + ) .await - .expect("glob by name"); - assert_eq!(by_name.matches.len(), 1); - assert!(by_name.matches[0].as_str().ends_with("readme.md")); - - let mut params = glob_params(&root, "**/*"); - params.limits.max_matches = 1; - let matches = fs.glob_files(params).await.expect("match limit"); - assert_eq!(matches.matches.len(), 1); - assert_eq!(matches.stopped, Some(GlobFilesStop::MatchLimit)); - - let mut params = glob_params(&root, "**/*"); - params.limits.max_entries = 2; - let entries = fs.glob_files(params).await.expect("entry limit"); - assert_eq!(entries.stopped, Some(GlobFilesStop::EntryLimit)); - assert_eq!(entries.entries_visited, 2); - - let mut params = glob_params(&root, "**/*"); - params.pattern = String::new(); - let error = fs.glob_files(params).await.expect_err("empty pattern"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::InvalidRequest); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); } #[tokio::test(flavor = "current_thread")] - async fn ranged_read_truncates_at_the_source_and_reports_true_size() { - let (_temp, fs, root) = search_fixture(); - let path = EnvironmentPath::new(format!("{}/readme.md", root.as_str())).expect("path"); - let full = std::fs::read(std::path::Path::new(root.as_str()).join("readme.md")) - .expect("full contents"); - - let ranged = fs - .read_file(ReadFileParams { - path: path.clone(), - offset: Some(0), - max_bytes: Some(6), - }) + async fn metadata_describes_links_and_follows_on_request() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + std::fs::create_dir(root.join("dir")).unwrap(); + symlink("dir", root.join("link")).unwrap(); + let link = fs + .get_metadata( + &context(), + GetMetadataParams { + path: path(&root, "link"), + follow_symlinks: false, + }, + ) .await - .expect("ranged read"); - assert_eq!(ranged.data.as_slice(), &full[..6]); - assert_eq!(ranged.file_size, Some(full.len() as u64)); - assert!(ranged.truncated); - - let tail = fs - .read_file(ReadFileParams { - path: path.clone(), - offset: Some(7), - max_bytes: None, - }) + .unwrap() + .metadata; + assert_eq!(link.kind, FileKind::Symlink); + assert_eq!(link.symlink_target.as_deref(), Some("dir")); + let followed = fs + .get_metadata( + &context(), + GetMetadataParams { + path: path(&root, "link"), + follow_symlinks: true, + }, + ) .await - .expect("tail read"); - assert_eq!(tail.data.as_slice(), &full[7..]); - assert!(!tail.truncated); + .unwrap() + .metadata; + assert_eq!(followed.kind, FileKind::Directory); + } - let whole = fs - .read_file(ReadFileParams { - path, - offset: None, - max_bytes: Some(u64::MAX), - }) - .await - .expect("whole read"); - assert_eq!(whole.data.as_slice(), full.as_slice()); - assert!(!whole.truncated); - assert_eq!(whole.file_size, Some(full.len() as u64)); + #[test] + fn io_errors_map_to_protocol_kinds() { + let error = |errno: Errno| io_error(io::Error::from(errno), "subject"); + assert_eq!( + reason(&error(Errno::LOOP)), + Some(PreconditionReason::SymlinkLoop) + ); + assert_eq!( + reason(&error(Errno::NOTEMPTY)), + Some(PreconditionReason::NotEmpty) + ); + assert_eq!( + reason(&error(Errno::XDEV)), + Some(PreconditionReason::CrossDevice) + ); + assert_eq!(error(Errno::NOENT).kind(), ProtocolErrorKind::NotFound); + assert_eq!(error(Errno::EXIST).kind(), ProtocolErrorKind::AlreadyExists); + assert!(matches!( + error(Errno::ACCESS).data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::FilePermission) + } + )); + assert_eq!( + error(Errno::ROFS).kind(), + ProtocolErrorKind::CapabilityUnavailable + ); + } + + #[test] + fn json_string_lengths_count_escapes() { + for value in [ + "", + "plain", + "a\"b\\c", + "line\n\ttab", + "\u{1b}[31mred\u{1b}[0m", + "\0", + "ünï", + ] { + let encoded = serde_json::to_string(value).unwrap(); + assert_eq!(json_string_len(value), encoded.len() as u64, "{value:?}"); + } } } diff --git a/crates/environment-daemon/src/filesystem/backend/mod.rs b/crates/environment-daemon/src/filesystem/backend/mod.rs deleted file mode 100644 index d07fa92bc..000000000 --- a/crates/environment-daemon/src/filesystem/backend/mod.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! OS boundary for confined traversal and atomic publication. Wire paths and inventories -//! contain no descriptors, inode numbers, or Unix permission bits. A Windows backend can -//! implement this interface with directory handles, reparse-point checks and native rename. -#[cfg(any(target_os = "linux", target_os = "macos"))] -mod unix; -#[cfg(any(target_os = "linux", target_os = "macos"))] -pub use unix::*; - -use environment_protocol::{ - data::inventory::MAX_INVENTORY_PATH_BYTES, - error::{EnvironmentProtocolError as Error, EnvironmentProtocolErrorCode as Code}, -}; -use sha2::{Digest, Sha256}; -use std::path::Path; -pub(crate) type Result = std::result::Result; -pub(crate) fn error(code: Code, message: impl Into) -> Error { - Error::new(code, message) -} -pub(crate) fn invalid(message: &str) -> Error { - error(Code::InvalidRequest, message) -} -pub(crate) fn io(e: std::io::Error) -> Error { - use std::io::ErrorKind::*; - if is_path_violation(&e) { - return error(Code::Forbidden, e.to_string()); - } - error( - match e.kind() { - NotFound => Code::NotFound, - AlreadyExists => Code::Conflict, - PermissionDenied => Code::Forbidden, - Unsupported => Code::Unsupported, - InvalidInput => Code::InvalidRequest, - _ => Code::Internal, - }, - e.to_string(), - ) -} -pub(crate) fn conflict() -> Error { - error( - Code::Conflict, - "filesystem observation changed; retry the filesystem operation", - ) -} -pub(crate) fn valid_path(path: &str) -> bool { - path.len() <= MAX_INVENTORY_PATH_BYTES - && (path.is_empty() - || (!path.contains(['\\', '\0']) - && path - .split('/') - .all(|p| !p.is_empty() && p != "." && p != ".."))) -} -pub(crate) fn valid_digest(value: &str) -> bool { - value.strip_prefix("sha256:").is_some_and(|s| { - s.len() == 64 - && s.bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) - }) -} -pub(crate) fn digest(hash: Sha256) -> String { - format!("sha256:{}", hex::encode(hash.finalize())) -} -pub(crate) fn relative( - root: &Path, - path: &environment_protocol::shared::EnvironmentPath, -) -> Result { - let path = Path::new(path.as_str()) - .strip_prefix(root) - .map_err(|_| error(Code::Forbidden, "selection outside filesystem root"))?; - let value = path - .components() - .map(|component| match component { - std::path::Component::Normal(name) => name - .to_str() - .map(str::to_owned) - .ok_or_else(|| invalid("non-UTF-8 selection")), - _ => Err(invalid("invalid selection component")), - }) - .collect::>>()? - .join("/"); - if !valid_path(&value) { - return Err(invalid("invalid selection")); - } - Ok(value) -} diff --git a/crates/environment-daemon/src/filesystem/backend/unix.rs b/crates/environment-daemon/src/filesystem/backend/unix.rs deleted file mode 100644 index 7be100dd7..000000000 --- a/crates/environment-daemon/src/filesystem/backend/unix.rs +++ /dev/null @@ -1,504 +0,0 @@ -use std::{ - ffi::{CStr, CString}, - fs::{File, Metadata}, - io, - os::unix::fs::{MetadataExt, PermissionsExt}, - path::{Component, Path}, -}; - -use rustix::fs::{self, AtFlags, FileType, Mode, OFlags}; - -pub struct Directory(File); -#[derive(Clone, Debug)] -pub struct Observation(Metadata); -impl Observation { - pub fn is_dir(&self) -> bool { - self.0.is_dir() - } - pub fn size(&self) -> u64 { - self.0.len() - } - pub fn executable(&self) -> bool { - self.0.mode() & 0o111 != 0 - } - pub fn matches(&self, other: &Self) -> bool { - let (a, b) = (&self.0, &other.0); - a.dev() == b.dev() - && a.ino() == b.ino() - && a.mode() == b.mode() - && a.len() == b.len() - && a.mtime() == b.mtime() - && a.mtime_nsec() == b.mtime_nsec() - && a.ctime() == b.ctime() - && a.ctime_nsec() == b.ctime_nsec() - } -} -fn c(name: &str) -> io::Result { - CString::new(name).map_err(|_| io::Error::from(io::ErrorKind::InvalidInput)) -} -fn check(meta: Metadata) -> io::Result { - if meta.is_file() || meta.is_dir() { - Ok(Observation(meta)) - } else { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "scoped filesystem access does not support symlinks or special files", - )) - } -} -pub fn observe(file: &File) -> io::Result { - check(file.metadata()?) -} -pub fn set_executable(file: &File, executable: bool) -> io::Result<()> { - file.set_permissions(std::fs::Permissions::from_mode(if executable { - 0o755 - } else { - 0o644 - })) -} -/// Streaming directory enumeration; cleanup never collects an entire retired tree. -struct Entries(fs::Dir); -impl Iterator for Entries { - type Item = io::Result; - fn next(&mut self) -> Option { - self.0.find_map(|entry| match entry { - Ok(entry) => { - let name = entry.file_name(); - if name.to_bytes() == b"." || name.to_bytes() == b".." { - None - } else { - Some(Ok(name.to_owned())) - } - } - Err(error) => Some(Err(error.into())), - }) - } -} -fn entries(file: &File) -> io::Result { - // Open a separate description so enumeration never shares the caller's offset. - let fd = fs::openat( - file, - ".", - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, - Mode::empty(), - )?; - Ok(Entries(fs::Dir::new(fd)?)) -} - -pub fn is_path_violation(error: &io::Error) -> bool { - matches!( - rustix::io::Errno::from_io_error(error), - Some(rustix::io::Errno::LOOP | rustix::io::Errno::NOTDIR) - ) -} -pub fn create_private_directory(path: &Path) -> io::Result<()> { - use std::os::unix::fs::DirBuilderExt; - std::fs::DirBuilder::new() - .recursive(true) - .mode(0o700) - .create(path) -} -pub fn sync_directory(path: &Path) -> io::Result<()> { - Directory::anchor(path)?.sync() -} - -impl Directory { - pub fn anchor(root: &Path) -> io::Result { - // Configured roots may contain platform aliases (/var on macOS). Resolve this - // administrator-owned anchor once; user selections are always opened relative to it. - let root = root.canonicalize()?; - let mut dir = Self( - fs::open( - "/", - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::empty(), - )? - .into(), - ); - for part in root.components() { - if let Component::Normal(name) = part { - let name = name - .to_str() - .ok_or_else(|| io::Error::from(io::ErrorKind::InvalidInput))?; - dir = dir.child(name)?; - } - } - Ok(dir) - } - pub fn sync(&self) -> io::Result<()> { - self.0.sync_all() - } - pub fn clone_dir(&self) -> io::Result { - Ok(Self(self.0.try_clone()?)) - } - pub fn child(&self, name: &str) -> io::Result { - self.child_name(name) - } - fn child_name(&self, name: impl rustix::path::Arg) -> io::Result { - Ok(Self( - fs::openat( - &self.0, - name, - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::empty(), - )? - .into(), - )) - } - pub fn parent(&self, path: &str) -> io::Result<(Self, String)> { - self.parent_with_creation(path, false) - } - /// Prepare a destination without following symlinks in existing or raced-in parents. - pub fn ensure_parent(&self, path: &str) -> io::Result<(Self, String)> { - self.parent_with_creation(path, true) - } - fn parent_with_creation(&self, path: &str, create_missing: bool) -> io::Result<(Self, String)> { - let mut parts = path.split('/').collect::>(); - let name = parts.pop().unwrap_or("").to_owned(); - let mut dir = self.clone_dir()?; - for part in parts { - dir = match dir.child(part) { - Ok(child) => child, - Err(error) if create_missing && error.kind() == io::ErrorKind::NotFound => { - let child = match dir.mkdir(part, false) { - Ok(child) => child, - // Another creator may win the mkdir race. Reopen with the same - // directory-only, no-follow checks used for existing parents. - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { - dir.child(part)? - } - Err(error) => return Err(error), - }; - dir.sync()?; - child - } - Err(error) => return Err(error), - }; - } - Ok((dir, if name.is_empty() { ".".into() } else { name })) - } - pub fn open(&self, path: &str) -> io::Result { - self.open_kind(path, false) - } - pub fn metadata(&self, path: &str) -> io::Result { - self.open_kind(path, true) - } - /// Check the same existing-file write permission as a normal write, without - /// truncation, and without following filesystem aliases outside the scope. - pub fn writable_file(&self, path: &str) -> io::Result { - let (dir, name) = self.parent(path)?; - let file: File = fs::openat( - &dir.0, - &name, - OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC, - Mode::empty(), - )? - .into(); - if !file.metadata()?.is_file() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "write requires a regular file", - )); - } - Ok(file) - } - fn open_kind(&self, path: &str, metadata_only: bool) -> io::Result { - let (dir, name) = self.parent(path)?; - #[cfg(target_os = "linux")] - let access = if metadata_only { - OFlags::PATH - } else { - OFlags::RDONLY - }; - #[cfg(target_os = "macos")] - let access = if metadata_only { - // Rustix does not name the macOS metadata-only flag. - OFlags::from_bits_retain(libc::O_EVTONLY as u32) - } else { - OFlags::RDONLY - }; - let file = fs::openat( - &dir.0, - &name, - access | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC, - Mode::empty(), - )? - .into(); - observe(&file)?; - Ok(file) - } - /// Inspect replacement targets without requiring read permission on their content. - pub fn target_exists(&self, name: &str) -> io::Result { - let stat = match fs::statat(&self.0, name, AtFlags::SYMLINK_NOFOLLOW) { - Ok(stat) => stat, - Err(rustix::io::Errno::NOENT) => return Ok(false), - Err(error) => return Err(error.into()), - }; - if !matches!( - FileType::from_raw_mode(stat.st_mode), - FileType::Directory | FileType::RegularFile - ) { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "target is a symlink or special file", - )); - } - Ok(true) - } - pub fn names(file: &File, limit: usize) -> io::Result> { - let mut names = Vec::new(); - for name in entries(file)? { - if names.len() >= limit { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "directory entry limit exceeded", - )); - } - names.push( - name?.into_string().map_err(|_| { - io::Error::new(io::ErrorKind::Unsupported, "non-UTF-8 filename") - })?, - ); - } - names.sort(); - Ok(names) - } - pub fn mkdir(&self, name: &str, private: bool) -> io::Result { - fs::mkdirat( - &self.0, - name, - Mode::from_raw_mode(if private { 0o700 } else { 0o755 }), - )?; - self.child(name) - } - pub fn create(&self, path: &str) -> io::Result { - let (dir, name) = self.parent(path)?; - Ok(fs::openat( - &dir.0, - &name, - OFlags::RDWR | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::RUSR | Mode::WUSR, - )? - .into()) - } - pub fn publish(&self, stage: &Self, target: &str, replace: bool) -> io::Result { - // Rustix handles flagged renames on both Linux libc variants and macOS. - let rename = |exchange: bool| { - fs::renameat_with( - &stage.0, - "tree", - &self.0, - target, - if exchange { - fs::RenameFlags::EXCHANGE - } else { - fs::RenameFlags::NOREPLACE - }, - ) - .map_err(io::Error::from) - }; - match rename(false) { - Ok(()) => Ok(false), - Err(error) if replace && error.kind() == io::ErrorKind::AlreadyExists => { - rename(true)?; - Ok(true) - } - Err(error) => Err(error), - } - } - - /// A normal rename atomically refuses to replace a directory with a file, - /// including a directory created after the caller's destination check. - pub fn publish_file(&self, stage: &Self, target: &str) -> io::Result<()> { - fs::renameat(&stage.0, "tree", &self.0, target).map_err(io::Error::from) - } - pub fn remove_tree(&self, name: &str) -> io::Result<()> { - struct Frame { - name: CString, - directory: Directory, - entries: Entries, - } - let name = c(name)?; - let unlink = |parent: &Directory, name: &CStr, flags: AtFlags| -> io::Result<()> { - Ok(fs::unlinkat(&parent.0, name, flags)?) - }; - let directory = match self.child_name(&name) { - Ok(dir) => dir, - Err(_) => return unlink(self, &name, AtFlags::empty()), - }; - let mut stack = vec![Frame { - name, - entries: entries(&directory.0)?, - directory, - }]; - while let Some(frame) = stack.last_mut() { - if let Some(name) = frame.entries.next() { - let name = name?; - match frame.directory.child_name(&name) { - Ok(directory) => { - if stack.len() >= 256 { - return Err(io::Error::new( - io::ErrorKind::Unsupported, - "retirement cleanup depth limit exceeded", - )); - } - stack.push(Frame { - name, - entries: entries(&directory.0)?, - directory, - }); - } - Err(_) => unlink(&frame.directory, &name, AtFlags::empty())?, - } - } else { - let frame = stack.pop().unwrap(); - let parent = stack.last().map_or(self, |frame| &frame.directory); - unlink(parent, &frame.name, AtFlags::REMOVEDIR)?; - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn file_publication_cannot_replace_a_directory() { - let root = tempfile::tempdir().unwrap(); - let parent = Directory::anchor(root.path()).unwrap(); - let stage = parent.mkdir("staging", true).unwrap(); - std::fs::write(root.path().join("staging/tree"), b"new bytes").unwrap(); - // The file was valid when checked, but another actor replaced it by a - // nonempty directory before publication. - std::fs::write(root.path().join("target"), b"old bytes").unwrap(); - parent.writable_file("target").unwrap(); - std::fs::remove_file(root.path().join("target")).unwrap(); - std::fs::create_dir(root.path().join("target")).unwrap(); - std::fs::write(root.path().join("target/keep"), b"keep").unwrap(); - assert!(parent.publish_file(&stage, "target").is_err()); - assert_eq!( - std::fs::read(root.path().join("target/keep")).unwrap(), - b"keep" - ); - assert_eq!( - std::fs::read(root.path().join("staging/tree")).unwrap(), - b"new bytes" - ); - std::fs::remove_dir_all(root.path().join("target")).unwrap(); - std::fs::write(root.path().join("target"), b"old bytes").unwrap(); - parent.publish_file(&stage, "target").unwrap(); - assert_eq!( - std::fs::read(root.path().join("target")).unwrap(), - b"new bytes" - ); - } - - #[test] - fn directory_enumerations_have_independent_offsets_and_enforce_limits() { - let root = tempfile::tempdir().unwrap(); - for name in ["a", "b", "c"] { - std::fs::write(root.path().join(name), name).unwrap(); - } - let dir = Directory::anchor(root.path()).unwrap(); - let mut first = entries(&dir.0).unwrap(); - let mut observed = vec![first.next().unwrap().unwrap().into_string().unwrap()]; - assert_eq!(Directory::names(&dir.0, 3).unwrap(), ["a", "b", "c"]); - observed.extend(first.map(|name| name.unwrap().into_string().unwrap())); - observed.sort(); - assert_eq!(observed, ["a", "b", "c"]); - assert_eq!( - Directory::names(&dir.0, 2).unwrap_err().kind(), - io::ErrorKind::InvalidInput - ); - assert_eq!(Directory::names(&dir.0, 3).unwrap(), ["a", "b", "c"]); - } - - #[test] - fn cleanup_removes_nested_trees_without_following_symlinks() { - use std::os::unix::fs::symlink; - - let root = tempfile::tempdir().unwrap(); - let outside = tempfile::tempdir().unwrap(); - std::fs::write(outside.path().join("keep"), "untouched").unwrap(); - let dir = Directory::anchor(root.path()).unwrap(); - let retired = dir.mkdir("retired", true).unwrap(); - retired.mkdir("nested", true).unwrap(); - std::fs::write(root.path().join("retired/nested/file"), "content").unwrap(); - symlink(outside.path(), root.path().join("retired/link")).unwrap(); - symlink("missing", root.path().join("retired/dangling")).unwrap(); - dir.remove_tree("retired").unwrap(); - assert!(!root.path().join("retired").exists()); - assert_eq!( - std::fs::read_to_string(outside.path().join("keep")).unwrap(), - "untouched" - ); - } - - // macOS filesystems reject invalid UTF-8 at creation time. - #[cfg(target_os = "linux")] - #[test] - fn cleanup_handles_non_utf8_names() { - use std::os::unix::ffi::OsStrExt; - - let root = tempfile::tempdir().unwrap(); - let dir = Directory::anchor(root.path()).unwrap(); - dir.mkdir("retired", true).unwrap(); - let raw_path = root - .path() - .join("retired") - .join(std::ffi::OsStr::from_bytes(b"raw-\xff")); - std::fs::create_dir(&raw_path).unwrap(); - std::fs::write( - raw_path.join(std::ffi::OsStr::from_bytes(b"file-\xfe")), - "data", - ) - .unwrap(); - dir.remove_tree("retired").unwrap(); - assert!(!root.path().join("retired").exists()); - } - - #[test] - fn publication_refuses_overwrite_and_exchanges_replaced_files_and_trees() { - for directory in [false, true] { - let root = tempfile::tempdir().unwrap(); - let parent = Directory::anchor(root.path()).unwrap(); - let stage = parent.mkdir("stage", true).unwrap(); - let staged = root.path().join("stage/tree"); - let target = root.path().join("target"); - let write = |path: &Path, bytes: &[u8]| { - if directory { - std::fs::create_dir(path).unwrap(); - std::fs::write(path.join("content"), bytes).unwrap(); - } else { - std::fs::write(path, bytes).unwrap(); - } - }; - let read = |path: &Path| { - std::fs::read(if directory { - path.join("content") - } else { - path.to_owned() - }) - .unwrap() - }; - - write(&staged, b"original"); - assert!(!parent.publish(&stage, "target", false).unwrap()); - assert_eq!(read(&target), b"original"); - assert!(!staged.exists()); - - write(&staged, b"replacement"); - let error = parent.publish(&stage, "target", false).unwrap_err(); - assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); - assert_eq!(read(&target), b"original"); - assert_eq!(read(&staged), b"replacement"); - - assert!(parent.publish(&stage, "target", true).unwrap()); - assert_eq!(read(&target), b"replacement"); - // Exchange retains the old target in staging for later cleanup. - assert_eq!(read(&staged), b"original"); - } - } -} diff --git a/crates/environment-daemon/src/filesystem/confinement.rs b/crates/environment-daemon/src/filesystem/confinement.rs new file mode 100644 index 000000000..b4fe081c9 --- /dev/null +++ b/crates/environment-daemon/src/filesystem/confinement.rs @@ -0,0 +1,721 @@ +//! The one confinement rule of every data-plane path, a command's `cwd` +//! included. +//! +//! A wire path that is neither the filesystem root nor below it is +//! `permissionDenied` (`outsideRoot`). Otherwise the root prefix is stripped +//! and the remaining segments are resolved one at a time beneath the root's +//! directory descriptor. A symlink is followed only while its target stays +//! inside the root; a link that leaves it is refused the same way. A path +//! that names the daemon state directory, or passes through or below it, is +//! `permissionDenied` (`daemonState`). +//! +//! The state directory is recognized by its identity (device and inode), +//! never by name: every entry a lookup meets is compared with it, so another +//! spelling on a case-insensitive filesystem, a bind mount or a link cannot +//! reach it. + +use std::{ + collections::VecDeque, + ffi::{OsStr, OsString}, + os::unix::ffi::OsStrExt, + path::{Path, PathBuf}, +}; + +use environment_protocol::{ + error::{PermissionReason, PreconditionReason, ProtocolError}, + paths::EnvironmentPath, +}; + +use super::{ + io_error, + os::{DEFAULT_DIRECTORY_MODE, Directory, EntryStat, PRIVATE_DIRECTORY_MODE, SELF}, +}; + +/// Most symlinks one resolution follows before it is a loop. +const MAX_SYMLINK_FOLLOWS: u32 = 40; + +/// The filesystem root, its directory descriptor and the daemon state +/// directory. Built once per daemon and shared by every manager. +#[derive(Debug)] +pub struct Confinement { + root: PathBuf, + root_path: EnvironmentPath, + root_directory: Directory, + state_dir: PathBuf, + /// Device and inode of the state directory. + state_identity: Identity, + /// Device and inode of each of its ancestors, its parent first. + state_ancestors: Vec, +} + +/// Device and inode of an entry. +pub(crate) type Identity = (u64, u64); + +/// How one lookup treats the final entry and missing entries. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Lookup { + /// Follow a symlink in the final position (inside the root only). + pub follow_final: bool, + /// A missing final entry resolves with `stat: None` instead of + /// `notFound`. + pub allow_missing: bool, + /// Create missing intermediate directories. + pub create_parents: bool, + /// Refuse every symlink, final or not: re-opening a path recorded as + /// canonical, where a link means the tree changed. + pub no_symlinks: bool, +} + +impl Lookup { + pub const FOLLOW: Self = Self { + follow_final: true, + allow_missing: false, + create_parents: false, + no_symlinks: false, + }; + pub const NO_FOLLOW: Self = Self { + follow_final: false, + allow_missing: false, + create_parents: false, + no_symlinks: false, + }; + + pub fn allowing_missing(mut self) -> Self { + self.allow_missing = true; + self + } + + pub fn creating_parents(mut self) -> Self { + self.create_parents = true; + self + } +} + +/// A resolved entry: the directory that holds it and its name there. Every +/// later operation names the entry through `parent` and `name`, so a link +/// swapped in after resolution is never followed. +#[derive(Debug)] +pub(crate) struct Resolved { + pub parent: Directory, + /// `.` for the filesystem root itself. + pub name: OsString, + /// Canonical segments below the root; empty for the root. + pub segments: Vec, + /// `lstat` of the final entry; `None` when it is missing and the lookup + /// allowed that. + pub stat: Option, + /// Canonical segments of the directories this lookup created + /// (`creating_parents`), outermost first. + pub created: Vec>, +} + +impl Resolved { + pub fn is_root(&self) -> bool { + self.segments.is_empty() + } + + /// Open the resolved directory. + pub fn open_directory(&self) -> std::io::Result { + if self.is_root() { + self.parent.try_clone() + } else { + self.parent.child(&self.name) + } + } +} + +impl Confinement { + /// `root` is canonicalized and opened. `state_dir` is created (private) + /// when missing and canonicalized; it must not be the root itself. + pub fn new(root: PathBuf, state_dir: PathBuf) -> Result { + let root = root + .canonicalize() + .map_err(|error| io_error(error, root.display()))?; + let root_path = root + .to_str() + .and_then(|root| EnvironmentPath::new(root).ok()) + .ok_or_else(|| { + ProtocolError::internal(format!( + "filesystem root {} is not a valid environment path", + root.display() + )) + })?; + let root_directory = + Directory::open_host(&root).map_err(|error| io_error(error, root.display()))?; + if !state_dir.exists() { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new() + .recursive(true) + .mode(PRIVATE_DIRECTORY_MODE) + .create(&state_dir) + .map_err(|error| io_error(error, state_dir.display()))?; + } + let state_dir = state_dir + .canonicalize() + .map_err(|error| io_error(error, state_dir.display()))?; + let identity_of = |path: &Path| { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(path) + .map(|metadata| (metadata.dev(), metadata.ino())) + .map_err(|error| io_error(error, path.display())) + }; + let state_identity = identity_of(&state_dir)?; + let state_ancestors = state_dir + .ancestors() + .skip(1) + .map(identity_of) + .collect::, _>>()?; + let root_identity = EntryStat::of_directory(&root_directory) + .map_err(|error| io_error(error, root.display()))? + .identity(); + if root_identity == state_identity { + return Err(ProtocolError::internal( + "the daemon state directory must not be the filesystem root", + )); + } + Ok(Self { + root, + root_path, + root_directory, + state_dir, + state_identity, + state_ancestors, + }) + } + + pub fn root(&self) -> &Path { + &self.root + } + + pub fn state_dir(&self) -> &Path { + &self.state_dir + } + + /// The root as a wire path. + pub fn root_path(&self) -> &EnvironmentPath { + &self.root_path + } + + /// The confinement rule: the resolved host path of the final target, + /// following in-root symlinks. Missing entries are `notFound`. + pub fn resolve(&self, path: &EnvironmentPath) -> Result { + let resolved = self.lookup(path, Lookup::FOLLOW)?; + Ok(self.host_path(&resolved.segments)) + } + + /// [`Self::resolve`], then require a directory + /// (`failedPrecondition` `notDirectory` otherwise). + pub fn resolve_directory(&self, path: &EnvironmentPath) -> Result { + let resolved = self.lookup(path, Lookup::FOLLOW)?; + if !resolved.stat.is_some_and(|stat| stat.is_dir()) { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{path} is not a directory"), + )); + } + Ok(self.host_path(&resolved.segments)) + } + + /// Resolve a wire path under the rule. + pub(crate) fn lookup( + &self, + path: &EnvironmentPath, + lookup: Lookup, + ) -> Result { + let Some(relative) = path.relative_to(&self.root_path) else { + return Err(outside_root(path)); + }; + let components = relative.segments().map(OsString::from).collect(); + self.walk(components, lookup, path) + } + + /// Resolve canonical segments below the root plus a trailing name, + /// following a symlink in final position: how a walker follows a link + /// it met at `segments/name`. + pub(crate) fn lookup_link( + &self, + segments: &[OsString], + name: &OsStr, + subject: &dyn std::fmt::Display, + ) -> Result { + let mut components = segments.to_vec(); + components.push(name.to_owned()); + self.walk(components.into(), Lookup::FOLLOW, subject) + } + + /// Re-open canonical segments recorded earlier. Any symlink on the way + /// means the tree changed: it fails with `ELOOP`'s mapping. + pub(crate) fn reopen( + &self, + segments: &[OsString], + subject: &dyn std::fmt::Display, + ) -> Result { + let lookup = Lookup { + no_symlinks: true, + ..Lookup::NO_FOLLOW + }; + self.walk(segments.iter().cloned().collect(), lookup, subject) + } + + /// The host path of canonical segments. + pub(crate) fn host_path(&self, segments: &[OsString]) -> PathBuf { + let mut path = self.root.clone(); + path.extend(segments); + path + } + + /// The wire path of canonical segments, when they are UTF-8. + pub(crate) fn wire_path(&self, segments: &[OsString]) -> Option { + let mut path = self.root_path.as_str().trim_end_matches('/').to_owned(); + for segment in segments { + path.push('/'); + path.push_str(segment.to_str()?); + } + if path.is_empty() { + path.push('/'); + } + EnvironmentPath::new(path).ok() + } + + /// The entry is the state directory. + pub(crate) fn is_state(&self, stat: &EntryStat) -> bool { + stat.identity() == self.state_identity + } + + /// The entry is the state directory or one of its ancestors: removing, + /// replacing or capturing it would address the state directory. + pub(crate) fn contains_state(&self, stat: &EntryStat) -> bool { + self.is_state(stat) || self.state_ancestors.contains(&stat.identity()) + } + + /// The directory is the state directory's parent, so one of its + /// entries may be the state directory. + pub(crate) fn holds_state(&self, directory: &EntryStat) -> bool { + self.state_ancestors.first() == Some(&directory.identity()) + } + + /// Refuse an operation whose subject contains the state directory. + pub(crate) fn refuse_containing_state( + &self, + resolved: &Resolved, + subject: &dyn std::fmt::Display, + ) -> Result<(), ProtocolError> { + if resolved.stat.is_some_and(|stat| self.contains_state(&stat)) { + return Err(daemon_state(subject)); + } + Ok(()) + } + + /// Remove directories a lookup created (see [`Resolved::created`]) + /// that are still empty, innermost first. Best effort. + pub(crate) fn remove_created(&self, created: &[Vec]) { + for segments in created.iter().rev() { + let Some((name, parent)) = segments.split_last() else { + continue; + }; + let Ok(resolved) = self.reopen(parent, &"created directory") else { + return; + }; + let Ok(directory) = resolved.open_directory() else { + return; + }; + if directory.rmdir(name).is_err() { + return; + } + } + } + + fn walk( + &self, + mut pending: VecDeque, + lookup: Lookup, + subject: &dyn std::fmt::Display, + ) -> Result { + let io = |error| io_error(error, subject); + let mut stack = vec![self.root_directory.try_clone().map_err(io)?]; + let mut segments: Vec = Vec::new(); + let mut created = Vec::new(); + let mut follows = 0; + while let Some(name) = pending.pop_front() { + if name.is_empty() || name == SELF { + continue; + } + if name == ".." { + if segments.pop().is_none() { + return Err(outside_root(subject)); + } + stack.pop(); + continue; + } + let last = pending.is_empty(); + let mut candidate = segments.clone(); + candidate.push(name.clone()); + let directory = stack.last().expect("the root stays on the stack"); + let stat = match directory.stat(&name) { + Ok(stat) => stat, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if last && lookup.allow_missing { + return Ok(Resolved { + parent: directory.try_clone().map_err(io)?, + name, + segments: candidate, + stat: None, + created, + }); + } + if !last && lookup.create_parents { + match directory.mkdir(&name, DEFAULT_DIRECTORY_MODE) { + Ok(()) => created.push(candidate), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(io(error)), + } + directory.sync().map_err(io)?; + pending.push_front(name); + continue; + } + return Err(io(error)); + } + Err(error) => return Err(io(error)), + }; + if self.is_state(&stat) { + return Err(daemon_state(subject)); + } + if stat.kind == environment_protocol::data::fs::FileKind::Symlink + && (!last || lookup.follow_final) + { + if lookup.no_symlinks { + return Err(ProtocolError::failed_precondition( + PreconditionReason::SymlinkLoop, + format!("{subject}: a symlink replaced a recorded entry"), + )); + } + follows += 1; + if follows > MAX_SYMLINK_FOLLOWS { + return Err(ProtocolError::failed_precondition( + PreconditionReason::SymlinkLoop, + format!("{subject}: too many levels of symbolic links"), + )); + } + let target = directory.read_link(&name).map_err(io)?; + let target = Path::new(&target); + let components: Vec = if target.is_absolute() { + let Ok(rest) = target.strip_prefix(&self.root) else { + return Err(outside_root(subject)); + }; + stack.truncate(1); + segments.clear(); + raw_components(rest) + } else { + raw_components(target) + }; + for component in components.into_iter().rev() { + pending.push_front(component); + } + continue; + } + if last { + return Ok(Resolved { + parent: directory.try_clone().map_err(io)?, + name, + segments: candidate, + stat: Some(stat), + created, + }); + } + if !stat.is_dir() { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{subject}: a path component is not a directory"), + )); + } + let child = directory.child(&name).map_err(io)?; + stack.push(child); + segments.push(name); + } + // The path ended on a directory already open: the root, or a + // directory reached through `..` or a link ending in one. + let current = stack.pop().expect("the root stays on the stack"); + let stat = EntryStat::of_directory(¤t).map_err(io)?; + if self.is_state(&stat) { + return Err(daemon_state(subject)); + } + match (stack.pop(), segments.last()) { + (Some(parent), Some(name)) => Ok(Resolved { + parent, + name: name.clone(), + segments, + stat: Some(stat), + created, + }), + _ => Ok(Resolved { + parent: current, + name: OsString::from(SELF), + segments, + stat: Some(stat), + created, + }), + } + } +} + +/// Components of a link target, keeping `.` and `..` for the walk to apply +/// against the open directories. +fn raw_components(path: &Path) -> Vec { + path.as_os_str() + .as_bytes() + .split(|byte| *byte == b'/') + .filter(|segment| !segment.is_empty()) + .map(|segment| OsStr::from_bytes(segment).to_owned()) + .collect() +} + +pub(crate) fn outside_root(subject: &dyn std::fmt::Display) -> ProtocolError { + ProtocolError::permission_denied( + Some(PermissionReason::OutsideRoot), + format!("{subject} is outside the filesystem root"), + ) +} + +pub(crate) fn daemon_state(subject: &dyn std::fmt::Display) -> ProtocolError { + ProtocolError::permission_denied( + Some(PermissionReason::DaemonState), + format!("{subject} addresses the daemon state directory"), + ) +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::symlink; + + use environment_protocol::error::{ErrorData, ProtocolErrorKind}; + + use super::*; + + struct Fixture { + _temp: tempfile::TempDir, + outside: PathBuf, + root: PathBuf, + confinement: Confinement, + } + + fn fixture() -> Fixture { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("root"); + let outside = base.join("outside"); + std::fs::create_dir_all(root.join("src/deep")).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(root.join("src/lib.rs"), "lib").unwrap(); + std::fs::write(outside.join("secret"), "secret").unwrap(); + let confinement = Confinement::new(root.clone(), root.join(".state")).unwrap(); + Fixture { + _temp: temp, + outside, + root, + confinement, + } + } + + fn path(root: &Path, relative: &str) -> EnvironmentPath { + EnvironmentPath::new(format!("{}/{relative}", root.display())).unwrap() + } + + fn permission_reason(error: &ProtocolError) -> Option { + match error.data { + ErrorData::PermissionDenied { reason } => reason, + _ => None, + } + } + + #[test] + fn paths_outside_the_root_are_refused_before_any_lookup() { + let fixture = fixture(); + let error = fixture + .confinement + .resolve( + &EnvironmentPath::new(fixture.outside.join("secret").to_str().unwrap()).unwrap(), + ) + .unwrap_err(); + assert_eq!( + permission_reason(&error), + Some(PermissionReason::OutsideRoot) + ); + } + + #[test] + fn in_root_links_are_followed_and_escaping_links_refused() { + let fixture = fixture(); + symlink("src", fixture.root.join("inside")).unwrap(); + // Relative to the link's directory: up to the root, then back down. + symlink("../../src/lib.rs", fixture.root.join("src/deep/up")).unwrap(); + symlink(&fixture.outside, fixture.root.join("absolute-escape")).unwrap(); + symlink("../outside/secret", fixture.root.join("relative-escape")).unwrap(); + symlink( + fixture.root.join("src"), + fixture.root.join("absolute-inside"), + ) + .unwrap(); + + assert_eq!( + fixture + .confinement + .resolve(&path(&fixture.root, "inside/lib.rs")) + .unwrap(), + fixture.root.join("src/lib.rs") + ); + assert_eq!( + fixture + .confinement + .resolve(&path(&fixture.root, "src/deep/up")) + .unwrap(), + fixture.root.join("src/lib.rs") + ); + assert_eq!( + fixture + .confinement + .resolve_directory(&path(&fixture.root, "absolute-inside")) + .unwrap(), + fixture.root.join("src") + ); + for escape in [ + "absolute-escape/secret", + "relative-escape", + "absolute-escape", + ] { + let error = fixture + .confinement + .resolve(&path(&fixture.root, escape)) + .unwrap_err(); + assert_eq!( + permission_reason(&error), + Some(PermissionReason::OutsideRoot), + "{escape}" + ); + } + } + + #[test] + fn the_state_directory_is_never_addressed() { + let fixture = fixture(); + std::fs::write(fixture.root.join(".state/daemon-key"), "key").unwrap(); + symlink(".state", fixture.root.join("state-link")).unwrap(); + for subject in [".state", ".state/daemon-key", "state-link/daemon-key"] { + let error = fixture + .confinement + .resolve(&path(&fixture.root, subject)) + .unwrap_err(); + assert_eq!( + permission_reason(&error), + Some(PermissionReason::DaemonState), + "{subject}" + ); + } + let root = fixture + .confinement + .lookup(fixture.confinement.root_path(), Lookup::NO_FOLLOW) + .unwrap(); + assert!(root.is_root()); + assert!(fixture.confinement.contains_state(&root.stat.unwrap())); + let src = fixture + .confinement + .lookup(&path(&fixture.root, "src"), Lookup::NO_FOLLOW) + .unwrap(); + assert!(!fixture.confinement.contains_state(&src.stat.unwrap())); + } + + #[test] + fn the_state_directory_is_recognized_by_identity_not_by_name() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap().join("root"); + std::fs::create_dir_all(root.join("local/state")).unwrap(); + let confinement = Confinement::new(root.clone(), root.join("local/state/envd")).unwrap(); + std::fs::write(root.join("local/state/envd/daemon-key"), "key").unwrap(); + // Another name for the same directories: no recorded spelling of + // the state directory matches it any more. + std::fs::rename(root.join("local"), root.join("moved")).unwrap(); + // Where the filesystem folds case (APFS), other spellings reach the + // same entries; elsewhere they are missing. + for subject in [ + "moved/state/envd/daemon-key", + "moved/state/envd", + "MOVED/STATE/ENVD/daemon-key", + ] { + match confinement.resolve(&path(&root, subject)) { + Err(error) if error.kind() == ProtocolErrorKind::NotFound => { + assert!(subject.starts_with("MOVED"), "{subject}"); + } + Err(error) => assert_eq!( + permission_reason(&error), + Some(PermissionReason::DaemonState), + "{subject}" + ), + Ok(resolved) => panic!("{subject} resolved to {}", resolved.display()), + } + } + for ancestor in ["moved", "moved/state"] { + let resolved = confinement + .lookup(&path(&root, ancestor), Lookup::NO_FOLLOW) + .unwrap(); + assert!( + confinement + .refuse_containing_state(&resolved, &ancestor) + .is_err(), + "{ancestor}" + ); + assert_eq!( + confinement.holds_state(&resolved.stat.unwrap()), + ancestor == "moved/state" + ); + } + } + + #[test] + fn loops_and_missing_entries_are_typed() { + let fixture = fixture(); + symlink("b", fixture.root.join("a")).unwrap(); + symlink("a", fixture.root.join("b")).unwrap(); + let error = fixture + .confinement + .resolve(&path(&fixture.root, "a")) + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::SymlinkLoop + } + )); + let error = fixture + .confinement + .resolve(&path(&fixture.root, "missing")) + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + let error = fixture + .confinement + .resolve_directory(&path(&fixture.root, "src/lib.rs")) + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::NotDirectory + } + )); + } + + #[test] + fn dangling_in_root_links_resolve_to_their_missing_target() { + let fixture = fixture(); + symlink("src/new.rs", fixture.root.join("dangling")).unwrap(); + let resolved = fixture + .confinement + .lookup( + &path(&fixture.root, "dangling"), + Lookup::FOLLOW.allowing_missing(), + ) + .unwrap(); + assert!(resolved.stat.is_none()); + assert_eq!( + fixture.confinement.host_path(&resolved.segments), + fixture.root.join("src/new.rs") + ); + } +} diff --git a/crates/environment-daemon/src/filesystem/os.rs b/crates/environment-daemon/src/filesystem/os.rs new file mode 100644 index 000000000..f0bf7f017 --- /dev/null +++ b/crates/environment-daemon/src/filesystem/os.rs @@ -0,0 +1,487 @@ +//! Descriptor-relative filesystem primitives. Every operation names an entry +//! by a directory descriptor and one name, never by a host path, so nothing +//! here follows a symlink implicitly. Wire paths, inventories and receipts +//! never contain descriptors or inode numbers; only [`FileVersion`] tokens +//! are derived from them. + +use std::{ + ffi::{OsStr, OsString}, + fs::File, + io::{self, Read}, + os::{ + fd::OwnedFd, + unix::ffi::{OsStrExt, OsStringExt}, + }, + path::Path, +}; + +use environment_protocol::data::fs::{FileKind, FileVersion}; +use rustix::fs::{self as rfs, AtFlags, FileType, Mode, OFlags, RenameFlags}; + +/// An open directory. Children are opened beneath it without following +/// symlinks. +#[derive(Debug)] +pub(crate) struct Directory(OwnedFd); + +/// The entry a directory descriptor itself names. +pub(crate) const SELF: &str = "."; + +impl Directory { + /// Open an administrator-configured directory by its host path. Only the + /// filesystem root and the transfer journal are opened this way; every + /// wire path is resolved beneath the root's descriptor. + pub fn open_host(path: &Path) -> io::Result { + Ok(Self(rfs::open( + path, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + )?)) + } + + pub fn try_clone(&self) -> io::Result { + Ok(Self(self.0.try_clone()?)) + } + + /// Open the child directory `name`; a symlink or non-directory fails + /// (`ELOOP` or `ENOTDIR`). + pub fn child(&self, name: &OsStr) -> io::Result { + Ok(Self(rfs::openat( + &self.0, + name, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?)) + } + + /// `lstat` of `name` (`.` for the directory itself). + pub fn stat(&self, name: &OsStr) -> io::Result { + Ok(EntryStat::from_stat(&rfs::statat( + &self.0, + name, + AtFlags::SYMLINK_NOFOLLOW, + )?)) + } + + /// `lstat` plus the birth time where the filesystem records one. + pub fn stat_with_birth(&self, name: &OsStr) -> io::Result { + let stat = rfs::statat(&self.0, name, AtFlags::SYMLINK_NOFOLLOW)?; + let mut entry = EntryStat::from_stat(&stat); + entry.birth_ms = birth_ms(&self.0, name, &stat); + Ok(entry) + } + + pub fn read_link(&self, name: &OsStr) -> io::Result { + let target = rfs::readlinkat(&self.0, name, Vec::new())?; + Ok(OsString::from_vec(target.into_bytes())) + } + + /// Open `name` for reading without following a symlink and without + /// blocking on a FIFO or device; callers `fstat` before reading. + pub fn open_read(&self, name: &OsStr) -> io::Result { + Ok(rfs::openat( + &self.0, + name, + OFlags::RDONLY | OFlags::NONBLOCK | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )? + .into()) + } + + /// Create `name` exclusively, never following a symlink. + pub fn create_file(&self, name: &OsStr, mode: u32) -> io::Result { + Ok(rfs::openat( + &self.0, + name, + OFlags::RDWR | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::from_raw_mode(mode as _), + )? + .into()) + } + + pub fn mkdir(&self, name: &OsStr, mode: u32) -> io::Result<()> { + Ok(rfs::mkdirat(&self.0, name, Mode::from_raw_mode(mode as _))?) + } + + pub fn unlink(&self, name: &OsStr) -> io::Result<()> { + Ok(rfs::unlinkat(&self.0, name, AtFlags::empty())?) + } + + pub fn rmdir(&self, name: &OsStr) -> io::Result<()> { + Ok(rfs::unlinkat(&self.0, name, AtFlags::REMOVEDIR)?) + } + + /// Rename `from` in `self` to `to` in `destination`. + pub fn rename( + &self, + from: &OsStr, + destination: &Directory, + to: &OsStr, + mode: RenameMode, + ) -> io::Result<()> { + match mode { + RenameMode::Replace => rfs::renameat(&self.0, from, &destination.0, to)?, + RenameMode::NoReplace => { + rfs::renameat_with(&self.0, from, &destination.0, to, RenameFlags::NOREPLACE)? + } + RenameMode::Exchange => { + rfs::renameat_with(&self.0, from, &destination.0, to, RenameFlags::EXCHANGE)? + } + } + Ok(()) + } + + /// Every name except `.` and `..`, unsorted. A separate open file + /// description is used, so enumerations never share an offset. + pub fn names(&self) -> io::Result> { + let mut names = Vec::new(); + for entry in self.entries()? { + names.push(entry?); + } + Ok(names) + } + + fn entries(&self) -> io::Result { + let fd = rfs::openat( + &self.0, + SELF, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + )?; + Ok(Entries(rfs::Dir::new(fd)?)) + } + + pub fn sync(&self) -> io::Result<()> { + Ok(rfs::fsync(&self.0)?) + } + + /// Remove `name` and everything below it without following symlinks. + /// Streams each directory, so a large tree is never collected at once. + pub fn remove_tree(&self, name: &OsStr) -> io::Result<()> { + struct Frame { + name: OsString, + directory: Directory, + entries: Entries, + } + let directory = match self.child(name) { + Ok(directory) => directory, + Err(error) if is_not_directory(&error) => return self.unlink(name), + Err(error) => return Err(error), + }; + let mut stack = vec![Frame { + name: name.to_owned(), + entries: directory.entries()?, + directory, + }]; + while let Some(frame) = stack.last_mut() { + if let Some(child) = frame.entries.next() { + let child = child?; + match frame.directory.child(&child) { + Ok(directory) => { + if stack.len() >= MAX_REMOVE_DEPTH { + return Err(io::Error::other("directory tree is too deep to remove")); + } + stack.push(Frame { + name: child, + entries: directory.entries()?, + directory, + }); + } + Err(error) if is_not_directory(&error) => frame.directory.unlink(&child)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } else { + let frame = stack.pop().expect("frame exists"); + let parent = stack.last().map_or(self, |frame| &frame.directory); + match parent.rmdir(&frame.name) { + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + result => result?, + } + } + } + Ok(()) + } +} + +const MAX_REMOVE_DEPTH: usize = 4096; + +/// How [`Directory::rename`] treats an existing destination. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RenameMode { + /// Replace a file (`rename(2)`). + Replace, + /// Fail with `EEXIST` when the destination exists. + NoReplace, + /// Swap source and destination atomically. + Exchange, +} + +/// `ELOOP` (a symlink met with `O_NOFOLLOW`) or `ENOTDIR`. +pub(crate) fn is_not_directory(error: &io::Error) -> bool { + matches!( + rustix::io::Errno::from_io_error(error), + Some(rustix::io::Errno::LOOP | rustix::io::Errno::NOTDIR) + ) +} + +struct Entries(rfs::Dir); + +impl Iterator for Entries { + type Item = io::Result; + + fn next(&mut self) -> Option { + self.0.find_map(|entry| match entry { + Ok(entry) => { + let name = entry.file_name().to_bytes(); + (name != b"." && name != b"..").then(|| Ok(OsString::from_vec(name.to_vec()))) + } + Err(error) => Some(Err(error.into())), + }) + } +} + +/// What `lstat` or `fstat` reported about one entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct EntryStat { + pub kind: FileKind, + pub mode: u32, + pub size: u64, + pub dev: u64, + pub ino: u64, + pub modified: (i64, i64), + pub changed: (i64, i64), + pub birth_ms: Option, +} + +impl EntryStat { + #[allow(clippy::unnecessary_cast)] + pub fn from_stat(stat: &rfs::Stat) -> Self { + let kind = match FileType::from_raw_mode(stat.st_mode as _) { + FileType::RegularFile => FileKind::File, + FileType::Directory => FileKind::Directory, + FileType::Symlink => FileKind::Symlink, + _ => FileKind::Other, + }; + Self { + kind, + mode: stat.st_mode as u32 & 0o7777, + size: stat.st_size as u64, + dev: stat.st_dev as u64, + ino: stat.st_ino as u64, + modified: (stat.st_mtime as i64, stat.st_mtime_nsec as i64), + changed: (stat.st_ctime as i64, stat.st_ctime_nsec as i64), + birth_ms: None, + } + } + + pub fn of_file(file: &File) -> io::Result { + Ok(Self::from_stat(&rfs::fstat(file)?)) + } + + pub fn of_directory(directory: &Directory) -> io::Result { + Ok(Self::from_stat(&rfs::fstat(&directory.0)?)) + } + + pub fn is_dir(&self) -> bool { + self.kind == FileKind::Directory + } + + pub fn is_file(&self) -> bool { + self.kind == FileKind::File + } + + pub fn executable(&self) -> bool { + self.mode & 0o111 != 0 + } + + pub fn modified_ms(&self) -> i64 { + to_ms(self.modified) + } + + /// Device and inode. + pub fn identity(&self) -> (u64, u64) { + (self.dev, self.ino) + } + + /// Same inode with the same size, modification and change times: the + /// content observed earlier is still the content. + pub fn unchanged(&self, other: &Self) -> bool { + self.dev == other.dev + && self.ino == other.ino + && self.size == other.size + && self.modified == other.modified + && self.changed == other.changed + } + + /// The opaque `version` token: identity, size and both timestamps. + pub fn version(&self) -> FileVersion { + FileVersion::new(format!( + "v1-{:x}-{:x}-{:x}-{:x}.{:x}-{:x}.{:x}", + self.dev, + self.ino, + self.size, + self.modified.0 as u64, + self.modified.1 as u64, + self.changed.0 as u64, + self.changed.1 as u64, + )) + .expect("version tokens are short and printable") + } +} + +fn to_ms((seconds, nanoseconds): (i64, i64)) -> i64 { + seconds + .saturating_mul(1000) + .saturating_add(nanoseconds / 1_000_000) +} + +#[cfg(target_os = "linux")] +fn birth_ms(directory: &OwnedFd, name: &OsStr, _: &rfs::Stat) -> Option { + let statx = rfs::statx( + directory, + name, + AtFlags::SYMLINK_NOFOLLOW, + rfs::StatxFlags::BTIME, + ) + .ok()?; + statx + .stx_mask + .contains(rfs::StatxFlags::BTIME) + .then(|| to_ms((statx.stx_btime.tv_sec, statx.stx_btime.tv_nsec as i64))) +} + +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd"))] +#[allow(clippy::unnecessary_cast)] +fn birth_ms(_: &OwnedFd, _: &OsStr, stat: &rfs::Stat) -> Option { + // Filesystems without birth times report zero or a negative value. + (stat.st_birthtime > 0) + .then(|| to_ms((stat.st_birthtime as i64, stat.st_birthtime_nsec as i64))) +} + +#[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "ios", + target_os = "freebsd" +)))] +fn birth_ms(_: &OwnedFd, _: &OsStr, _: &rfs::Stat) -> Option { + None +} + +/// Read at most `max` bytes from the current position. +pub(crate) fn read_up_to(file: &mut File, max: u64) -> io::Result> { + let mut data = Vec::with_capacity(max.min(1024 * 1024) as usize); + file.take(max).read_to_end(&mut data)?; + Ok(data) +} + +/// A name as UTF-8, or `None` when it is not representable on the wire. +pub(crate) fn utf8_name(name: &OsStr) -> Option<&str> { + std::str::from_utf8(name.as_bytes()).ok() +} + +/// Set the permission bits of an open file. +pub(crate) fn set_mode(file: &File, mode: u32) -> io::Result<()> { + Ok(rfs::fchmod(file, Mode::from_raw_mode(mode as _))?) +} + +/// The mode with an execute bit added wherever a read bit is set. +pub(crate) fn with_execute_bits(mode: u32) -> u32 { + mode | ((mode & 0o444) >> 2) +} + +/// Mode of a file the daemon creates. +pub(crate) const DEFAULT_FILE_MODE: u32 = 0o644; +/// Mode of a directory the daemon creates. +pub(crate) const DEFAULT_DIRECTORY_MODE: u32 = 0o755; +/// Mode of the daemon's own staging directories. +pub(crate) const PRIVATE_DIRECTORY_MODE: u32 = 0o700; +/// Mode of a staged file before its final mode is set. +pub(crate) const PRIVATE_FILE_MODE: u32 = 0o600; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn removal_streams_nested_trees_without_following_symlinks() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("keep"), "untouched").unwrap(); + std::fs::create_dir_all(root.path().join("retired/nested")).unwrap(); + std::fs::write(root.path().join("retired/nested/file"), "content").unwrap(); + symlink(outside.path(), root.path().join("retired/link")).unwrap(); + symlink("missing", root.path().join("retired/dangling")).unwrap(); + let directory = Directory::open_host(root.path()).unwrap(); + directory.remove_tree(OsStr::new("retired")).unwrap(); + assert!(!root.path().join("retired").exists()); + assert_eq!( + std::fs::read_to_string(outside.path().join("keep")).unwrap(), + "untouched" + ); + } + + #[test] + fn enumerations_have_independent_offsets() { + let root = tempfile::tempdir().unwrap(); + for name in ["a", "b", "c"] { + std::fs::write(root.path().join(name), name).unwrap(); + } + let directory = Directory::open_host(root.path()).unwrap(); + let mut first = directory.entries().unwrap(); + let mut seen = vec![first.next().unwrap().unwrap()]; + let mut all = directory.names().unwrap(); + all.sort(); + assert_eq!(all, ["a", "b", "c"]); + seen.extend(first.map(Result::unwrap)); + seen.sort(); + assert_eq!(seen, ["a", "b", "c"]); + } + + #[test] + fn rename_modes_refuse_replace_and_exchange() { + let root = tempfile::tempdir().unwrap(); + let directory = Directory::open_host(root.path()).unwrap(); + std::fs::write(root.path().join("a"), "a").unwrap(); + std::fs::write(root.path().join("b"), "b").unwrap(); + let error = directory + .rename( + OsStr::new("a"), + &directory, + OsStr::new("b"), + RenameMode::NoReplace, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + directory + .rename( + OsStr::new("a"), + &directory, + OsStr::new("b"), + RenameMode::Exchange, + ) + .unwrap(); + assert_eq!(std::fs::read(root.path().join("a")).unwrap(), b"b"); + assert_eq!(std::fs::read(root.path().join("b")).unwrap(), b"a"); + } + + #[test] + fn versions_change_with_content_identity() { + let root = tempfile::tempdir().unwrap(); + let directory = Directory::open_host(root.path()).unwrap(); + std::fs::write(root.path().join("file"), "one").unwrap(); + let before = directory.stat(OsStr::new("file")).unwrap(); + assert_eq!( + before.version(), + directory.stat(OsStr::new("file")).unwrap().version() + ); + std::fs::write(root.path().join("staged"), "two!").unwrap(); + std::fs::rename(root.path().join("staged"), root.path().join("file")).unwrap(); + assert_ne!( + before.version(), + directory.stat(OsStr::new("file")).unwrap().version() + ); + } +} diff --git a/crates/environment-daemon/src/filesystem/scan.rs b/crates/environment-daemon/src/filesystem/scan.rs index 7d060f256..48eecb872 100644 --- a/crates/environment-daemon/src/filesystem/scan.rs +++ b/crates/environment-daemon/src/filesystem/scan.rs @@ -1,312 +1,546 @@ -//! Bounded endpoint-local observations through the shared scoped filesystem backend. -use super::backend::{ - self, Directory, Result, conflict, digest, error, invalid, io, relative, valid_path, +//! `fs/scan`: a bounded observation of several roots, each reported on its +//! own. A skipped entry never fails its root, and a failed root never hides +//! the others. + +use std::{ + sync::Arc, + time::{Duration, Instant}, }; -use environment_protocol::shared::EnvironmentPath; + use environment_protocol::{ - data::inventory::*, error::EnvironmentProtocolErrorCode as Code, shared::ByteChunk, + bytes::ByteChunk, + data::{ + fs::TreeContent, + handshake::ScanCeilings, + scan::{ + ScanEntry, ScanFingerprint, ScanParams, ScanResponse, ScanRoot, ScanRootResult, + ScanSkip, ScanSkipReason, + }, + }, + error::{ErrorData, PreconditionReason, ProtocolError}, + limits::{MAX_LISTED_SKIPS, MAX_RESULT_BYTES}, + paths::{ContentDigest, TreePath}, }; use sha2::{Digest, Sha256}; -use std::{collections::BTreeSet, path::PathBuf}; -use std::{ - io::Read, - path::Path, - time::{Duration, Instant}, +use tokio_util::sync::CancellationToken; + +use super::{ + confinement::{Confinement, Lookup}, + json_string_len, + os::{self, EntryStat}, + search::check_quota, + walk::{CompiledFilter, Event, LinkMode, Skip, Walker}, }; -pub fn scan(root: &Path, params: ScanParams) -> Result { - let ceiling = InventoryLimits::default(); - let limits = params.limits; - if limits.max_entries == 0 - || limits.max_entries > ceiling.max_entries - || limits.max_depth > ceiling.max_depth - || limits.max_file_bytes > ceiling.max_file_bytes - || limits.max_total_bytes > ceiling.max_total_bytes - || limits.max_manifest_bytes > ceiling.max_manifest_bytes - || limits.max_duration_ms == 0 - || limits.max_duration_ms > ceiling.max_duration_ms - || params.roots.len() > 32 - || params.include_patterns.len() > 32 - || params - .include_patterns - .iter() - .any(|p| p.len() > MAX_INVENTORY_PATH_BYTES) - { - return Err(invalid("scan quotas exceed daemon ceilings")); +/// Encoded overhead of one entry or skip besides its path and content. +const ENTRY_OVERHEAD: u64 = 96; + +pub(crate) fn check_scan( + params: &ScanParams, + ceilings: &ScanCeilings, +) -> Result<(), ProtocolError> { + let roots = params.roots.len() as u64; + if roots == 0 { + return Err(ProtocolError::invalid_field( + "roots", + "scan needs at least one root", + )); } - let patterns = params - .include_patterns - .iter() - .map(|p| glob::Pattern::new(p).map_err(|e| invalid(&e.to_string()))) - .collect::>>()?; - // Filename-only patterns select direct children, so unrelated subtrees do - // not consume scan budgets or cause failures for a nonrecursive query. - let direct_only = !params.include_patterns.is_empty() - && params - .include_patterns - .iter() - .all(|pattern| !pattern.contains('/') && !pattern.contains("**")); - let canonical_root = root.canonicalize().map_err(io)?; - let anchor = Directory::anchor(&canonical_root).map_err(io)?; - let mut response = ScanResponse { - fingerprint: None, - unchanged: false, - complete: true, - entries: vec![], - diagnostics: vec![], + if roots > u64::from(ceilings.max_roots) { + return Err(ProtocolError::above_ceiling( + "roots", + roots, + ceilings.max_roots.into(), + )); + } + let patterns = (params.filter.include.len() + params.filter.exclude.len()) as u64; + if patterns > u64::from(ceilings.max_patterns) { + return Err(ProtocolError::above_ceiling( + "filter", + patterns, + ceilings.max_patterns.into(), + )); + } + let limits = ¶ms.limits; + check_quota( + "limits.maxEntries", + limits.max_entries.into(), + ceilings.max_entries.into(), + )?; + if limits.max_file_bytes > ceilings.max_file_bytes { + return Err(ProtocolError::above_ceiling( + "limits.maxFileBytes", + limits.max_file_bytes, + ceilings.max_file_bytes, + )); + } + if limits.max_total_bytes > ceilings.max_total_bytes { + return Err(ProtocolError::above_ceiling( + "limits.maxTotalBytes", + limits.max_total_bytes, + ceilings.max_total_bytes, + )); + } + check_quota( + "limits.maxDurationMs", + limits.max_duration_ms, + ceilings.max_duration_ms, + ) +} + +/// Budgets shared by every root of one scan. +struct Totals { + content_bytes: u64, + result_bytes: u64, +} + +pub(crate) fn scan( + confinement: &Arc, + params: ScanParams, + cancel: &CancellationToken, +) -> Result { + let filter = CompiledFilter::compile(¶ms.filter, "filter")?; + let mut totals = Totals { + content_bytes: 0, + result_bytes: 0, }; - let start = Instant::now(); - let mut visited = 0u32; - let mut inspected = 0u64; - let mut response_bytes = 0u64; - let mut observations = Vec::new(); - let mut observation_bytes = 0u64; - for selected in ¶ms.roots { - let result = (|| -> Result<()> { - let selected_relative = relative(root, selected)?; - let selected_path = canonical_root.join(selected_relative); - // A missing configured root is a complete empty observation. A dangling - // link or a permission error is an incomplete observation. - match std::fs::symlink_metadata(&selected_path) { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - let mut ancestor = selected_path.parent(); - while let Some(path) = ancestor { - match std::fs::symlink_metadata(path) { - Ok(_) => { - let resolved = path.canonicalize().map_err(io)?; - if !resolved.starts_with(&canonical_root) { - return Err(error( - Code::Forbidden, - "root outside access scope", - )); - } - break; - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - ancestor = path.parent() - } - Err(e) => return Err(io(e)), - } - } - return Ok(()); - } - Err(e) => return Err(io(e)), - Ok(_) => {} + let mut roots = Vec::with_capacity(params.roots.len()); + for root in ¶ms.roots { + let result = match scan_root(confinement, ¶ms, root, &filter, &mut totals, cancel) { + Ok(result) => result, + Err(RootError::Cancelled) => return Err(ProtocolError::cancelled()), + Err(RootError::Failed(error)) => ScanRootResult::Failed { + path: root.path.clone(), + error, + }, + }; + roots.push(result); + } + Ok(ScanResponse { roots }) +} + +enum RootError { + Cancelled, + Failed(ProtocolError), +} + +impl From for RootError { + fn from(error: ProtocolError) -> Self { + Self::Failed(error) + } +} + +fn scan_root( + confinement: &Arc, + params: &ScanParams, + root: &ScanRoot, + filter: &CompiledFilter, + totals: &mut Totals, + cancel: &CancellationToken, +) -> Result { + let started = Instant::now(); + let deadline = Duration::from_millis(params.limits.max_duration_ms); + let resolved = match confinement.lookup(&root.path, Lookup::FOLLOW) { + Ok(resolved) => resolved, + Err(error) if matches!(error.data, ErrorData::NotFound { .. }) => { + return Ok(ScanRootResult::Missing { + path: root.path.clone(), + }); + } + Err(error) => return Err(error.into()), + }; + if !resolved.stat.is_some_and(|stat| stat.is_dir()) { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{} is not a directory", root.path), + ) + .into()); + } + let canonical_path = confinement + .wire_path(&resolved.segments) + .filter(|canonical| canonical != &root.path); + let links = if params.follow_symlinks { + LinkMode::Follow + } else { + LinkMode::Report + }; + let mut walker = Walker::new( + confinement.clone(), + &resolved, + filter.clone(), + links, + root.max_depth, + &root.path, + )?; + let limits = params.limits; + let read_files = params.read_content || params.include_digests; + let mut entries = Vec::new(); + let mut skipped = Vec::new(); + let mut skipped_entry_count = 0u32; + let mut content_bytes = 0u64; + let mut result_bytes = 0u64; + let mut fingerprint = Sha256::new(); + fingerprint.update([ + u8::from(params.read_content), + u8::from(params.include_digests), + ]); + let mut skip = |path: Option, reason: ScanSkipReason, result_bytes: &mut u64| { + skipped_entry_count = skipped_entry_count.saturating_add(1); + if (skipped.len() as u32) < MAX_LISTED_SKIPS { + *result_bytes += path + .as_ref() + .map_or(0, |path| json_string_len(path.as_str())) + + ENTRY_OVERHEAD; + skipped.push(ScanSkip { path, reason }); + } + }; + while let Some(event) = walker.next_event() { + if cancel.is_cancelled() { + return Err(RootError::Cancelled); + } + if started.elapsed() >= deadline { + return Err(ProtocolError::deadline_exceeded(format!( + "scanning {} took longer than {} ms", + root.path, limits.max_duration_ms + )) + .into()); + } + let entry = match event { + Event::Yield + | Event::Skip { + selected: false, .. + } => continue, + Event::Skip { path, reason, .. } => { + let reason = match reason { + Skip::NonUtf8Name => ScanSkipReason::NonUtf8Name, + Skip::Unreadable(_) | Skip::DanglingLink => ScanSkipReason::Unreadable, + Skip::LinkOutsideRoot(_) => ScanSkipReason::SymlinkOutsideRoot, + }; + skip(path, reason, &mut result_bytes); + continue; } - let mut pending = vec![(selected_path, String::new(), BTreeSet::::new())]; - while let Some((requested, path, mut ancestors)) = pending.pop() { - visited += 1; - if visited > limits.max_entries { - return Err(invalid("scan visited entry limit exceeded")); - } - if start.elapsed() > Duration::from_millis(limits.max_duration_ms.min(30_000)) { - return Err(error(Code::Timeout, "scan deadline exceeded")); - } - if !valid_path(&path) - || (!path.is_empty() && path.split('/').count() > limits.max_depth as usize) - { - return Err(invalid("scan depth/path limit exceeded")); - } - let canonical = requested.canonicalize().map_err(io)?; - let scoped = canonical - .strip_prefix(&canonical_root) - .map_err(|_| error(Code::Forbidden, "symlink target outside access scope"))?; - if !params.follow_symlinks && requested != canonical { - return Err(error(Code::Forbidden, "scan symlink following disabled")); - } - let scoped = scoped.to_str().ok_or_else(|| invalid("non-UTF-8 path"))?; - // The canonical spelling is still opened beneath an anchored directory - // without following symlinks, preventing a retarget race from escaping. - let mut file = if !params.read_content && params.digest_algorithm.is_none() { - anchor.metadata(scoped) - } else { - anchor.open(scoped) + Event::Entry(entry) => entry, + }; + if !entry.selected { + continue; + } + let (content, data) = match entry.stat.kind { + environment_protocol::data::fs::FileKind::Directory => { + (TreeContent::Directory {}, None) + } + environment_protocol::data::fs::FileKind::Symlink => { + skip(Some(entry.path), ScanSkipReason::Symlink, &mut result_bytes); + continue; + } + environment_protocol::data::fs::FileKind::Other => { + skip( + Some(entry.path), + ScanSkipReason::SpecialFile, + &mut result_bytes, + ); + continue; + } + environment_protocol::data::fs::FileKind::File => { + if entry.stat.size > limits.max_file_bytes { + skip( + Some(entry.path), + ScanSkipReason::FileTooLarge, + &mut result_bytes, + ); + continue; } - .map_err(io)?; - let observed = backend::observe(&file).map_err(io)?; - let matches = patterns.is_empty() || patterns.iter().any(|p| p.matches(&path)); - let mut data = None; - let content = if observed.is_dir() { - if direct_only && !path.is_empty() { - continue; - } - if !ancestors.insert(canonical.clone()) { - return Err(error(Code::Conflict, "scan symlink loop")); - } - let names = Directory::names(&file, limits.max_entries as usize).map_err(io)?; - if visited as usize + pending.len() + names.len() > limits.max_entries as usize - { - return Err(invalid("scan traversal limit exceeded")); - } - let pending_bytes: usize = pending - .iter() - .map(|(requested, path, ancestors)| { - requested.as_os_str().len() - + path.len() - + ancestors.iter().map(|p| p.as_os_str().len()).sum::() - }) - .sum(); - let added_bytes: usize = names - .iter() - .map(|name| { - requested.as_os_str().len() - + path.len() - + name.len() * 2 - + ancestors.iter().map(|p| p.as_os_str().len()).sum::() - }) - .sum(); - if (pending_bytes + added_bytes) as u64 > limits.max_manifest_bytes { - return Err(invalid("scan traversal byte limit exceeded")); - } - for name in names.into_iter().rev() { - if !valid_path(&name) { - return Err(invalid("unsupported filename")); - } - let child = if path.is_empty() { - name.clone() - } else { - format!("{path}/{name}") - }; - pending.push((requested.join(name), child, ancestors.clone())); - } - ScanContent::Directory + if !read_files { + ( + TreeContent::File { + size_bytes: entry.stat.size, + executable: entry.stat.executable(), + digest: None, + }, + None, + ) } else { - let digest = - if matches && (params.read_content || params.digest_algorithm.is_some()) { - if observed.size() > limits.max_file_bytes { - return Err(invalid("scan per-file byte limit exceeded")); - } - inspected = inspected - .checked_add(observed.size()) - .ok_or_else(|| invalid("byte overflow"))?; - if inspected > limits.max_total_bytes { - return Err(invalid("scan total byte limit exceeded")); - } - if params.read_content && observed.size() > MAX_CONTENT_CHUNK as u64 { - return Err(invalid("scan response file limit exceeded")); - } - let mut hash = Sha256::new(); - let mut bytes = Vec::new(); - let mut read = 0u64; - let mut buffer = [0u8; 65536]; - loop { - let count = file.read(&mut buffer).map_err(io)?; - if count == 0 { - break; - } - read += count as u64; - if read > observed.size() { - return Err(conflict()); - } - if start.elapsed() - > Duration::from_millis(limits.max_duration_ms.min(30_000)) - { - return Err(error(Code::Timeout, "scan deadline exceeded")); + match read_file( + &entry.parent, + &entry.name, + &entry.stat, + limits.max_file_bytes, + ) { + Ok(Some(bytes)) => { + let digest = ContentDigest::from_sha256(Sha256::digest(&bytes).into()); + fingerprint.update(digest.sha256()); + let data = if params.read_content { + content_bytes += bytes.len() as u64; + if totals.content_bytes + content_bytes > limits.max_total_bytes { + return Err(ProtocolError::payload_too_large( + limits.max_total_bytes, + format!( + "the content of {} exceeds the scan's remaining maxTotalBytes", + root.path + ), + ) + .into()); } - hash.update(&buffer[..count]); - if params.read_content { - bytes.extend_from_slice(&buffer[..count]); - } - } - if read != observed.size() - || !observed.matches(&backend::observe(&file).map_err(io)?) - { - return Err(conflict()); - } - if params.read_content { - data = Some(ByteChunk::from(bytes)); - } - params.digest_algorithm.map(|_| digest(hash)) - } else { - None - }; - ScanContent::File { - size_bytes: observed.size(), - executable: observed.executable(), - digest, - } - }; - observation_bytes += - (requested.as_os_str().len() + canonical.as_os_str().len() + 128) as u64; - if observation_bytes > limits.max_manifest_bytes { - return Err(invalid("scan observation memory limit exceeded")); - } - observations.push((requested, canonical.clone(), observed)); - if matches { - let row = ScanEntry { - root: selected.clone(), - path, - canonical_path: EnvironmentPath::new(canonical.to_string_lossy()) - .map_err(|e| invalid(&e.to_string()))?, - content, - data, - }; - response_bytes += serde_json::to_vec(&row) - .map_err(|e| invalid(&e.to_string()))? - .len() as u64; - if response_bytes > limits.max_manifest_bytes.min(4 * 1024 * 1024) - || response.entries.len() >= 10_000 - { - return Err(invalid("scan response byte limit exceeded")); + Some(ByteChunk::from(bytes)) + } else { + None + }; + ( + TreeContent::File { + size_bytes: entry.stat.size, + executable: entry.stat.executable(), + digest: params.include_digests.then_some(digest), + }, + data, + ) + } + Ok(None) => { + skip( + Some(entry.path), + ScanSkipReason::FileTooLarge, + &mut result_bytes, + ); + continue; + } + Err(()) => { + skip( + Some(entry.path), + ScanSkipReason::Unreadable, + &mut result_bytes, + ); + continue; + } } - response.entries.push(row); } } - Ok(()) - })(); - if let Err(error) = result { - response.complete = false; - response.diagnostics.push(ScanDiagnostic { - root: selected.clone(), - error, - }); + }; + if entries.len() as u32 >= limits.max_entries { + return Err(ProtocolError::too_many_entries( + limits.max_entries, + format!( + "{} selects more than {} entries", + root.path, limits.max_entries + ), + ) + .into()); } - } - // Validate identities and observations again, including nonmatching directories. - for (requested, canonical, observed) in observations { - let result = (|| -> Result<()> { - if start.elapsed() > Duration::from_millis(limits.max_duration_ms.min(30_000)) { - return Err(error(Code::Timeout, "scan deadline exceeded")); - } - if requested.canonicalize().map_err(io)? != canonical { - return Err(conflict()); - } - let path = canonical - .strip_prefix(&canonical_root) - .unwrap() - .to_str() - .ok_or_else(|| invalid("non-UTF-8 path"))?; - if !observed - .matches(&backend::observe(&anchor.metadata(path).map_err(io)?).map_err(io)?) - { - return Err(conflict()); + fingerprint.update(entry.path.as_str().as_bytes()); + fingerprint.update([0]); + match &content { + TreeContent::Directory {} => fingerprint.update(b"d"), + TreeContent::File { + size_bytes, + executable, + .. + } => { + fingerprint.update([b'f', u8::from(*executable)]); + fingerprint.update(size_bytes.to_le_bytes()); } - Ok(()) - })(); - if let Err(error) = result { - response.complete = false; - response.diagnostics.push(ScanDiagnostic { - root: EnvironmentPath::new(requested.to_string_lossy()) - .map_err(|e| invalid(&e.to_string()))?, - error, + } + result_bytes += json_string_len(entry.path.as_str()) + + ENTRY_OVERHEAD + + data.as_ref().map_or(0, |data: &ByteChunk| { + (data.as_slice().len() as u64).div_ceil(3) * 4 }); + if totals.result_bytes + result_bytes > MAX_RESULT_BYTES { + return Err(ProtocolError::payload_too_large( + MAX_RESULT_BYTES, + format!( + "the result of {} exceeds the scan's response budget", + root.path + ), + ) + .into()); } + entries.push(ScanEntry { + path: entry.path, + content, + data, + }); } - if response.complete { - response - .entries - .sort_by(|a, b| (a.root.as_str(), &a.path).cmp(&(b.root.as_str(), &b.path))); - let mut query = params.clone(); - query.if_none_match = None; - let fingerprint = digest(Sha256::new_with_prefix( - serde_json::to_vec(&(&canonical_root, query, &response.entries)) - .map_err(|e| invalid(&e.to_string()))?, - )); - response.unchanged = params.if_none_match.as_ref() == Some(&fingerprint); - response.fingerprint = Some(fingerprint); - if response.unchanged { - response.entries.clear(); + let fingerprint = ScanFingerprint::new(format!( + "{}{}", + ScanFingerprint::PREFIX, + hex::encode(fingerprint.finalize()) + )) + .expect("a hex digest is a valid fingerprint"); + if root.if_none_match.as_ref() == Some(&fingerprint) { + return Ok(ScanRootResult::Unchanged { + path: root.path.clone(), + fingerprint, + }); + } + totals.content_bytes += content_bytes; + totals.result_bytes += result_bytes; + Ok(ScanRootResult::Complete { + path: root.path.clone(), + canonical_path, + fingerprint, + entries, + skipped, + skipped_entry_count, + }) +} + +/// Read a selected file whole: `Ok(None)` when it grew past `max`, `Err` +/// when it vanished, changed or could not be read. +fn read_file( + parent: &os::Directory, + name: &std::ffi::OsStr, + observed: &EntryStat, + max: u64, +) -> Result>, ()> { + let mut file = parent.open_read(name).map_err(drop)?; + let opened = EntryStat::of_file(&file).map_err(drop)?; + if !opened.is_file() || !opened.unchanged(observed) { + return Err(()); + } + let bytes = os::read_up_to(&mut file, max + 1).map_err(drop)?; + if bytes.len() as u64 > max { + return Ok(None); + } + Ok(Some(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + use environment_protocol::{ + data::{fs::TreeFilter, scan::ScanLimits}, + error::ProtocolErrorKind, + paths::EnvironmentPath, + }; + + fn limits() -> ScanLimits { + ScanLimits { + max_entries: 100, + max_file_bytes: 1024, + max_total_bytes: 4096, + max_duration_ms: 10_000, } } - Ok(response) + + fn root(path: &std::path::Path) -> ScanRoot { + ScanRoot { + path: EnvironmentPath::new(path.to_str().unwrap()).unwrap(), + if_none_match: None, + max_depth: None, + } + } + + fn fixture() -> (tempfile::TempDir, std::path::PathBuf, Arc) { + let temp = tempfile::tempdir().unwrap(); + let host = temp.path().canonicalize().unwrap(); + let confinement = Arc::new(Confinement::new(host.clone(), host.join(".state")).unwrap()); + (temp, host, confinement) + } + + #[test] + fn one_bad_entry_is_skipped_and_one_bad_root_fails_alone() { + let (_temp, host, confinement) = fixture(); + std::fs::create_dir_all(host.join("skills/a")).unwrap(); + std::fs::create_dir_all(host.join("skills/b")).unwrap(); + std::fs::write(host.join("skills/a/SKILL.md"), "a").unwrap(); + std::fs::write(host.join("skills/b/SKILL.md"), vec![b'b'; 2048]).unwrap(); + std::fs::write(host.join("file"), "not a directory").unwrap(); + let params = ScanParams { + roots: vec![ + root(&host.join("skills")), + root(&host.join("missing")), + root(&host.join("file")), + root(&host.join(".state")), + ], + filter: TreeFilter { + include: vec!["**/SKILL.md".into()], + ..TreeFilter::default() + }, + read_content: true, + include_digests: false, + follow_symlinks: false, + limits: limits(), + }; + let response = scan(&confinement, params, &CancellationToken::new()).unwrap(); + let ScanRootResult::Complete { + entries, + skipped, + skipped_entry_count, + .. + } = &response.roots[0] + else { + panic!("{:?}", response.roots[0]); + }; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].path.as_str(), "a/SKILL.md"); + assert_eq!(entries[0].data.as_ref().unwrap().as_slice(), b"a"); + assert_eq!( + skipped, + &[ScanSkip { + path: Some(TreePath::new("b/SKILL.md").unwrap()), + reason: ScanSkipReason::FileTooLarge, + }] + ); + assert_eq!(*skipped_entry_count, 1); + assert!(matches!(response.roots[1], ScanRootResult::Missing { .. })); + let ScanRootResult::Failed { error, .. } = &response.roots[2] else { + panic!("{:?}", response.roots[2]); + }; + assert!(matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::NotDirectory + } + )); + let ScanRootResult::Failed { error, .. } = &response.roots[3] else { + panic!("{:?}", response.roots[3]); + }; + assert_eq!(error.kind(), ProtocolErrorKind::PermissionDenied); + } + + #[test] + fn depth_zero_probes_a_directory_and_fingerprints_detect_changes() { + let (_temp, host, confinement) = fixture(); + std::fs::create_dir_all(host.join("work")).unwrap(); + std::fs::write(host.join("work/a"), "one").unwrap(); + let probe = ScanRoot { + max_depth: Some(0), + ..root(&host.join("work")) + }; + let params = |roots: Vec| ScanParams { + roots, + filter: TreeFilter::default(), + read_content: false, + include_digests: true, + follow_symlinks: false, + limits: limits(), + }; + let response = scan( + &confinement, + params(vec![probe, root(&host.join("work"))]), + &CancellationToken::new(), + ) + .unwrap(); + let ScanRootResult::Complete { entries, .. } = &response.roots[0] else { + panic!("{:?}", response.roots[0]); + }; + assert!(entries.is_empty()); + let ScanRootResult::Complete { fingerprint, .. } = &response.roots[1] else { + panic!("{:?}", response.roots[1]); + }; + let again = ScanRoot { + if_none_match: Some(fingerprint.clone()), + ..root(&host.join("work")) + }; + let response = scan( + &confinement, + params(vec![again.clone()]), + &CancellationToken::new(), + ) + .unwrap(); + assert!(matches!( + response.roots[0], + ScanRootResult::Unchanged { .. } + )); + std::fs::write(host.join("work/a"), "two").unwrap(); + let response = scan(&confinement, params(vec![again]), &CancellationToken::new()).unwrap(); + assert!(matches!(response.roots[0], ScanRootResult::Complete { .. })); + } } diff --git a/crates/environment-daemon/src/filesystem/search.rs b/crates/environment-daemon/src/filesystem/search.rs new file mode 100644 index 000000000..49729fddc --- /dev/null +++ b/crates/environment-daemon/src/filesystem/search.rs @@ -0,0 +1,546 @@ +//! `fs/searchText` and `fs/globFiles`: bounded walks run on a blocking +//! thread that stop at their duration ceiling, on `$/cancelRequest` and +//! when their connection closes. + +use std::{ + io::{self, Read}, + sync::Arc, + time::{Duration, Instant}, +}; + +use environment_protocol::{ + data::{ + fs::{ + FileKind, GlobFilesParams, GlobFilesResponse, GlobFilesStop, SearchTextMatch, + SearchTextParams, SearchTextResponse, SearchTextStop, + }, + handshake::{GlobCeilings, SearchCeilings}, + }, + error::{PreconditionReason, ProtocolError}, + limits::MAX_RESULT_BYTES, + paths::EnvironmentPath, +}; +use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkMatch}; +use tokio_util::sync::CancellationToken; + +use super::{ + confinement::{Confinement, Lookup}, + json_string_len, + os::EntryStat, + walk::{CompiledFilter, Event, LinkMode, Walker}, +}; + +/// Encoded overhead of one search match besides its encoded path and line. +const SEARCH_MATCH_OVERHEAD: u64 = 64; +/// Encoded overhead of one glob match besides its encoded path. +const GLOB_MATCH_OVERHEAD: u64 = 16; +/// Longest line the searcher buffers; a file with a longer line is skipped. +const MAX_SEARCH_LINE_BUFFER: usize = 16 * 1024 * 1024; + +pub(crate) fn check_search( + params: &SearchTextParams, + ceilings: &SearchCeilings, +) -> Result<(), ProtocolError> { + if params.pattern.is_empty() { + return Err(ProtocolError::invalid_field( + "pattern", + "search pattern must not be empty", + )); + } + let limits = ¶ms.limits; + check_quota( + "limits.maxMatches", + limits.max_matches.into(), + ceilings.max_matches.into(), + )?; + check_quota( + "limits.maxFiles", + limits.max_files.into(), + ceilings.max_files.into(), + )?; + check_quota("limits.maxBytes", limits.max_bytes, ceilings.max_bytes)?; + check_quota( + "limits.maxDurationMs", + limits.max_duration_ms, + ceilings.max_duration_ms, + )?; + check_quota( + "limits.maxLineBytes", + limits.max_line_bytes.into(), + ceilings.max_line_bytes.into(), + ) +} + +pub(crate) fn check_glob( + params: &GlobFilesParams, + ceilings: &GlobCeilings, +) -> Result<(), ProtocolError> { + if params.filter.include.is_empty() { + return Err(ProtocolError::invalid_field( + "filter.include", + "globFiles needs at least one include pattern", + )); + } + let limits = ¶ms.limits; + check_quota( + "limits.maxMatches", + limits.max_matches.into(), + ceilings.max_matches.into(), + )?; + check_quota( + "limits.maxFiles", + limits.max_files.into(), + ceilings.max_files.into(), + )?; + check_quota( + "limits.maxDurationMs", + limits.max_duration_ms, + ceilings.max_duration_ms, + ) +} + +/// A caller quota: at least 1 and at most its ceiling. +pub(crate) fn check_quota(field: &str, value: u64, ceiling: u64) -> Result<(), ProtocolError> { + if value == 0 { + return Err(ProtocolError::invalid_field( + field, + format!("{field} must be at least 1"), + )); + } + if value > ceiling { + return Err(ProtocolError::above_ceiling(field, value, ceiling)); + } + Ok(()) +} + +/// Resolve a walk root: an existing directory under the confinement rule. +fn walker( + confinement: &Arc, + root: &EnvironmentPath, + filter: CompiledFilter, + links: LinkMode, +) -> Result { + let resolved = confinement.lookup(root, Lookup::FOLLOW)?; + if !resolved.stat.is_some_and(|stat| stat.is_dir()) { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{root} is not a directory"), + )); + } + Walker::new(confinement.clone(), &resolved, filter, links, None, root) +} + +pub(crate) fn search_text( + confinement: &Arc, + params: SearchTextParams, + cancel: &CancellationToken, +) -> Result { + let started = Instant::now(); + let limits = params.limits; + let matcher = grep_regex::RegexMatcherBuilder::new() + .case_insensitive(params.pattern_ignore_case) + .build(¶ms.pattern) + .map_err(|error| { + ProtocolError::invalid_field("pattern", format!("invalid search regex: {error}")) + })?; + let filter = CompiledFilter::compile(¶ms.filter, "filter")?; + let mut walker = walker(confinement, ¶ms.root, filter, LinkMode::Report)?; + let deadline = Duration::from_millis(limits.max_duration_ms); + let mut searcher = SearcherBuilder::new() + .line_number(true) + .binary_detection(BinaryDetection::quit(0)) + .heap_limit(Some(MAX_SEARCH_LINE_BUFFER)) + .build(); + let mut response = SearchTextResponse { + matches: Vec::new(), + searched_file_count: 0, + searched_bytes: 0, + skipped_file_count: 0, + skipped_entry_count: 0, + elapsed_ms: 0, + stopped: None, + }; + let mut result_bytes = 0u64; + while let Some(event) = walker.next_event() { + if cancel.is_cancelled() { + return Err(ProtocolError::cancelled()); + } + if started.elapsed() >= deadline { + response.stopped = Some(SearchTextStop::TimeLimit); + break; + } + let entry = match event { + Event::Yield => continue, + Event::Skip { .. } => { + response.skipped_entry_count = response.skipped_entry_count.saturating_add(1); + continue; + } + Event::Entry(entry) => entry, + }; + if entry.stat.kind != FileKind::File || !entry.selected { + continue; + } + let remaining = limits.max_bytes - response.searched_bytes; + if remaining == 0 { + response.stopped = Some(SearchTextStop::ByteLimit); + break; + } + if response.searched_file_count >= limits.max_files { + response.stopped = Some(SearchTextStop::FileLimit); + break; + } + if entry.stat.size > remaining { + response.skipped_file_count = response.skipped_file_count.saturating_add(1); + continue; + } + let Ok(file) = entry.parent.open_read(&entry.name) else { + response.skipped_file_count = response.skipped_file_count.saturating_add(1); + continue; + }; + if !EntryStat::of_file(&file).is_ok_and(|stat| stat.is_file()) { + response.skipped_file_count = response.skipped_file_count.saturating_add(1); + continue; + } + let path = params + .root + .join(&entry.path) + .map_err(|error| ProtocolError::internal(error.to_string()))?; + let mut sink = MatchSink { + path, + matches: &mut response.matches, + max_matches: limits.max_matches, + max_line_bytes: limits.max_line_bytes as usize, + result_bytes: &mut result_bytes, + stopped: None, + binary: false, + }; + let mut reader = Counting { + inner: file.take(remaining), + read: 0, + deadline: started + deadline, + cancel, + interrupted: None, + }; + let outcome = searcher.search_reader(&matcher, &mut reader, &mut sink); + let (stopped, binary) = (sink.stopped, sink.binary); + response.searched_bytes += reader.read; + match reader.interrupted { + Some(Interrupted::Cancelled) => return Err(ProtocolError::cancelled()), + Some(Interrupted::Deadline) => { + response.stopped = Some(SearchTextStop::TimeLimit); + break; + } + None => {} + } + if outcome.is_err() || binary { + response.skipped_file_count = response.skipped_file_count.saturating_add(1); + } else { + response.searched_file_count += 1; + } + if stopped.is_some() { + response.stopped = stopped; + break; + } + } + response.elapsed_ms = started.elapsed().as_millis() as u64; + Ok(response) +} + +/// Why a file's search stopped before its end. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Interrupted { + Cancelled, + Deadline, +} + +/// Counts the bytes read and ends the read once the walk's deadline passed +/// or its request was cancelled, so one large file cannot outlast either. +struct Counting<'a, R> { + inner: R, + read: u64, + deadline: Instant, + cancel: &'a CancellationToken, + interrupted: Option, +} + +impl Read for Counting<'_, R> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.cancel.is_cancelled() { + self.interrupted = Some(Interrupted::Cancelled); + } else if Instant::now() >= self.deadline { + self.interrupted = Some(Interrupted::Deadline); + } + if self.interrupted.is_some() { + // Not `Interrupted`: readers retry that kind. + return Err(io::Error::other("the search was stopped")); + } + let read = self.inner.read(buffer)?; + self.read += read as u64; + Ok(read) + } +} + +struct MatchSink<'a> { + path: EnvironmentPath, + matches: &'a mut Vec, + max_matches: u32, + max_line_bytes: usize, + result_bytes: &'a mut u64, + stopped: Option, + binary: bool, +} + +impl Sink for MatchSink<'_> { + type Error = io::Error; + + fn matched(&mut self, _: &Searcher, found: &SinkMatch<'_>) -> Result { + if self.matches.len() as u64 >= u64::from(self.max_matches) { + self.stopped = Some(SearchTextStop::MatchLimit); + return Ok(false); + } + let line = String::from_utf8_lossy(found.bytes()); + let line = line.trim_end_matches(['\r', '\n']); + let (line, line_truncated) = cut(line, self.max_line_bytes); + let size = + json_string_len(self.path.as_str()) + json_string_len(line) + SEARCH_MATCH_OVERHEAD; + if *self.result_bytes + size > MAX_RESULT_BYTES { + self.stopped = Some(SearchTextStop::ResponseLimit); + return Ok(false); + } + *self.result_bytes += size; + let line_number = found + .line_number() + .map_or(0, |number| u32::try_from(number).unwrap_or(u32::MAX)); + self.matches.push(SearchTextMatch { + path: self.path.clone(), + line_number, + line: line.to_owned(), + line_truncated, + }); + Ok(true) + } + + fn binary_data(&mut self, _: &Searcher, _: u64) -> Result { + self.binary = true; + Ok(false) + } +} + +/// Cut `line` to at most `max` bytes on a character boundary. +fn cut(line: &str, max: usize) -> (&str, bool) { + if line.len() <= max { + return (line, false); + } + let mut end = max; + while !line.is_char_boundary(end) { + end -= 1; + } + (&line[..end], true) +} + +pub(crate) fn glob_files( + confinement: &Arc, + params: GlobFilesParams, + cancel: &CancellationToken, +) -> Result { + let started = Instant::now(); + let limits = params.limits; + let filter = CompiledFilter::compile(¶ms.filter, "filter")?; + let mut walker = walker(confinement, ¶ms.root, filter, LinkMode::Report)?; + let deadline = Duration::from_millis(limits.max_duration_ms); + let mut response = GlobFilesResponse { + matches: Vec::new(), + visited_file_count: 0, + skipped_entry_count: 0, + elapsed_ms: 0, + stopped: None, + }; + let mut result_bytes = 0u64; + while let Some(event) = walker.next_event() { + if cancel.is_cancelled() { + return Err(ProtocolError::cancelled()); + } + if started.elapsed() >= deadline { + response.stopped = Some(GlobFilesStop::TimeLimit); + break; + } + let entry = match event { + Event::Yield => continue, + Event::Skip { .. } => { + response.skipped_entry_count = response.skipped_entry_count.saturating_add(1); + continue; + } + Event::Entry(entry) => entry, + }; + if entry.stat.is_dir() { + continue; + } + if response.visited_file_count >= limits.max_files { + response.stopped = Some(GlobFilesStop::FileLimit); + break; + } + response.visited_file_count += 1; + if !entry.selected { + continue; + } + if response.matches.len() as u64 >= u64::from(limits.max_matches) { + response.stopped = Some(GlobFilesStop::MatchLimit); + break; + } + let path = params + .root + .join(&entry.path) + .map_err(|error| ProtocolError::internal(error.to_string()))?; + let size = json_string_len(path.as_str()) + GLOB_MATCH_OVERHEAD; + if result_bytes + size > MAX_RESULT_BYTES { + response.stopped = Some(GlobFilesStop::ResponseLimit); + break; + } + result_bytes += size; + response.matches.push(path); + } + response.elapsed_ms = started.elapsed().as_millis() as u64; + Ok(response) +} + +#[cfg(test)] +mod tests { + use super::*; + use environment_protocol::data::fs::{GlobFilesLimits, SearchTextLimits, TreeFilter}; + + struct Fixture { + _temp: tempfile::TempDir, + root: EnvironmentPath, + host: std::path::PathBuf, + confinement: Arc, + } + + fn fixture() -> Fixture { + let temp = tempfile::tempdir().unwrap(); + let host = temp.path().canonicalize().unwrap(); + std::fs::create_dir_all(host.join("src/deep")).unwrap(); + std::fs::create_dir_all(host.join(".git/objects")).unwrap(); + std::fs::write( + host.join("src/lib.rs"), + "pub fn target() {}\nfn other() {}\n", + ) + .unwrap(); + std::fs::write(host.join("src/deep/nested.rs"), "target nested\n").unwrap(); + std::fs::write(host.join("readme.md"), "target readme\n").unwrap(); + std::fs::write(host.join("binary.bin"), b"tar\x00get\n").unwrap(); + // A pack file larger than any test budget; VCS metadata is pruned + // before the budget is charged. + std::fs::write(host.join(".git/objects/pack"), vec![b't'; 64 * 1024]).unwrap(); + let confinement = Arc::new(Confinement::new(host.clone(), host.join(".state")).unwrap()); + Fixture { + _temp: temp, + root: EnvironmentPath::new(host.to_str().unwrap()).unwrap(), + host, + confinement, + } + } + + fn search(root: &EnvironmentPath) -> SearchTextParams { + SearchTextParams { + root: root.clone(), + pattern: "target".into(), + pattern_ignore_case: false, + filter: TreeFilter::default(), + limits: SearchTextLimits { + max_matches: 100, + max_files: 100, + max_bytes: 1024, + max_duration_ms: 10_000, + max_line_bytes: 4096, + }, + } + } + + #[test] + fn search_skips_vcs_metadata_and_oversized_files_without_stopping() { + let fixture = fixture(); + std::fs::write(fixture.host.join("big.txt"), vec![b'x'; 2048]).unwrap(); + let response = search_text( + &fixture.confinement, + search(&fixture.root), + &CancellationToken::new(), + ) + .unwrap(); + let paths: Vec<_> = response + .matches + .iter() + .map(|found| { + found + .path + .as_str() + .trim_start_matches(fixture.root.as_str()) + }) + .collect(); + assert_eq!(paths, ["/readme.md", "/src/deep/nested.rs", "/src/lib.rs"]); + assert_eq!(response.stopped, None); + // big.txt exceeds the remaining budget and binary.bin is binary: + // both are skipped and counted, and the walk continues. + assert_eq!(response.skipped_file_count, 2); + assert!(response.searched_bytes < 1024); + } + + #[test] + fn search_stops_on_limits_and_cancellation() { + let fixture = fixture(); + let mut params = search(&fixture.root); + params.limits.max_matches = 1; + let response = + search_text(&fixture.confinement, params, &CancellationToken::new()).unwrap(); + assert_eq!(response.matches.len(), 1); + assert_eq!(response.stopped, Some(SearchTextStop::MatchLimit)); + + let mut params = search(&fixture.root); + params.limits.max_line_bytes = 3; + let response = + search_text(&fixture.confinement, params, &CancellationToken::new()).unwrap(); + assert!(response.matches.iter().all(|found| found.line_truncated)); + + let cancel = CancellationToken::new(); + cancel.cancel(); + let error = search_text(&fixture.confinement, search(&fixture.root), &cancel).unwrap_err(); + assert_eq!(error, ProtocolError::cancelled()); + } + + #[test] + fn glob_uses_the_tree_grammar_and_counts_only_files() { + let fixture = fixture(); + let params = |patterns: &[&str]| GlobFilesParams { + root: fixture.root.clone(), + filter: TreeFilter { + include: patterns.iter().map(|p| (*p).to_owned()).collect(), + ..TreeFilter::default() + }, + limits: GlobFilesLimits { + max_matches: 100, + max_files: 100, + max_duration_ms: 10_000, + }, + }; + let token = CancellationToken::new(); + let response = glob_files(&fixture.confinement, params(&["src/*.rs"]), &token).unwrap(); + assert_eq!(response.matches.len(), 1); + assert!(response.matches[0].as_str().ends_with("/src/lib.rs")); + let response = glob_files(&fixture.confinement, params(&["*.rs"]), &token).unwrap(); + assert_eq!(response.matches.len(), 2); + // Directories are free; the four visible files are counted. + assert_eq!(response.visited_file_count, 4); + let error = check_glob( + ¶ms(&[]), + &GlobCeilings { + max_matches: 10, + max_files: 10, + max_duration_ms: 10, + }, + ) + .unwrap_err(); + assert!(matches!( + error.data, + environment_protocol::error::ErrorData::InvalidParams { .. } + )); + } +} diff --git a/crates/environment-daemon/src/filesystem/transfer/capture.rs b/crates/environment-daemon/src/filesystem/transfer/capture.rs new file mode 100644 index 000000000..b7474a22d --- /dev/null +++ b/crates/environment-daemon/src/filesystem/transfer/capture.rs @@ -0,0 +1,416 @@ +//! Capture: walk a source under the shared traversal, hash every selected +//! file in bounded steps, and serve the sealed inventory's content by +//! digest. A file that changes between hashing and reading fails the +//! operation (`aborted`, `sourceChanged`). + +use std::{ + collections::BTreeMap, + ffi::OsString, + fs::File, + io::{Read, Seek, SeekFrom}, + sync::Arc, +}; + +use environment_protocol::{ + data::{ + fs::{FileKind, TreeContent, TreeFilter}, + transfer::{ + InventoryEntry, SpecialFilePolicy, SymlinkPolicy, TransferLimits, TransferSkip, + TransferSkipReason, + }, + }, + error::{AbortReason, PreconditionReason, ProtocolError}, + limits::MAX_LISTED_SKIPS, + paths::{ContentDigest, EnvironmentPath, TreePath}, +}; +use sha2::{Digest, Sha256}; + +use crate::filesystem::{ + confinement::{Confinement, Lookup}, + io_error, + os::{self, EntryStat}, + walk::{CompiledFilter, Event, LinkMode, Skip, Walker}, +}; + +use super::{STEP_BYTES, STEP_ENTRIES, check_entry_quota, entry_bytes}; + +/// Where a captured digest's bytes live. +#[derive(Clone, Debug)] +pub(super) struct Source { + pub canonical: Vec, + pub stat: EntryStat, +} + +struct Hashing { + path: TreePath, + source: Source, + file: File, + hash: Sha256, + read: u64, +} + +pub(super) struct Capture { + /// Canonical segments of the source; expanded links must stay inside. + selection: Vec, + walker: Option, + hashing: Option, + symlinks: SymlinkPolicy, + special_files: SpecialFilePolicy, + limits: TransferLimits, + pub entries: Vec, + pub sources: BTreeMap, + pub total_bytes: u64, + manifest_bytes: u64, + pub skipped: Vec, + pub skipped_count: u32, +} + +impl Capture { + pub fn begin( + confinement: Arc, + source: &EnvironmentPath, + filter: &TreeFilter, + symlinks: SymlinkPolicy, + special_files: SpecialFilePolicy, + limits: TransferLimits, + ) -> Result { + let compiled = CompiledFilter::compile(filter, "selection.filter")?; + let resolved = confinement.lookup(source, Lookup::FOLLOW)?; + confinement.refuse_containing_state(&resolved, source)?; + let stat = resolved.stat.expect("a followed lookup reports its target"); + let mut capture = Self { + selection: resolved.segments.clone(), + walker: None, + hashing: None, + symlinks, + special_files, + limits, + entries: Vec::new(), + sources: BTreeMap::new(), + total_bytes: 0, + manifest_bytes: 0, + skipped: Vec::new(), + skipped_count: 0, + }; + match stat.kind { + FileKind::Directory => { + capture.record(InventoryEntry { + path: TreePath::root(), + content: TreeContent::Directory {}, + })?; + let links = if symlinks == SymlinkPolicy::ExpandInside { + LinkMode::Follow + } else { + LinkMode::Report + }; + capture.walker = Some(Walker::new( + confinement, + &resolved, + compiled, + links, + None, + source, + )?); + } + FileKind::File => { + let file = resolved + .parent + .open_read(&resolved.name) + .map_err(|error| io_error(error, source))?; + capture.start_hashing( + TreePath::root(), + Source { + canonical: resolved.segments, + stat, + }, + file, + )?; + } + FileKind::Symlink | FileKind::Other => { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotRegularFile, + format!("{source} is not a regular file or directory"), + )); + } + } + Ok(capture) + } + + /// One bounded step; `true` once the inventory is sealed. + pub fn advance(&mut self) -> Result { + let mut allowance = STEP_BYTES; + for _ in 0..STEP_ENTRIES { + if self.hashing.is_some() { + if !self.hash_step(&mut allowance)? { + return Ok(false); + } + continue; + } + let Some(walker) = self.walker.as_mut() else { + return Ok(true); + }; + let Some(event) = walker.next_event() else { + self.walker = None; + return Ok(true); + }; + self.visit(event)?; + } + Ok(self.walker.is_none() && self.hashing.is_none()) + } + + /// Selection comes first: an entry or skip the filter does not select + /// is left out silently, before any policy, depth or readability check + /// could fail the capture. Directories are always walked. + fn visit(&mut self, event: Event) -> Result<(), ProtocolError> { + let entry = match event { + Event::Yield + | Event::Skip { + selected: false, .. + } => return Ok(()), + Event::Skip { + path, + reason: Skip::NonUtf8Name, + .. + } => { + self.skip(path, TransferSkipReason::NonUtf8Name); + return Ok(()); + } + Event::Skip { + path, + reason: Skip::DanglingLink, + .. + } => { + self.skip(path, TransferSkipReason::Symlink); + return Ok(()); + } + Event::Skip { + reason: Skip::Unreadable(error), + .. + } => return Err(error), + Event::Skip { + path, + reason: Skip::LinkOutsideRoot(error), + .. + } => { + return Err(symlink_rejected( + path.as_ref().map_or(".", TreePath::as_str), + &error.message, + )); + } + Event::Entry(entry) => entry, + }; + if !entry.selected && !entry.stat.is_dir() { + return Ok(()); + } + if entry.via_link && !entry.canonical.starts_with(&self.selection) { + return Err(symlink_rejected( + entry.path.as_str(), + "the link leaves the captured selection", + )); + } + if entry.depth > self.limits.max_depth { + return Err(ProtocolError::invalid_field( + "limits.maxDepth", + format!("{} is deeper than limits.maxDepth", entry.path), + )); + } + match entry.stat.kind { + FileKind::Directory => self.record(InventoryEntry { + path: entry.path, + content: TreeContent::Directory {}, + }), + FileKind::Symlink => match self.symlinks { + SymlinkPolicy::Skip => { + self.skip(Some(entry.path), TransferSkipReason::Symlink); + Ok(()) + } + SymlinkPolicy::Reject | SymlinkPolicy::ExpandInside => Err(symlink_rejected( + entry.path.as_str(), + "the capture refuses symlinks", + )), + }, + FileKind::Other => match self.special_files { + SpecialFilePolicy::Skip => { + self.skip(Some(entry.path), TransferSkipReason::SpecialFile); + Ok(()) + } + SpecialFilePolicy::Reject => Err(ProtocolError::failed_precondition( + PreconditionReason::NotRegularFile, + format!("{} is a special file", entry.path), + )), + }, + FileKind::File => { + let file = entry + .parent + .open_read(&entry.name) + .map_err(|error| io_error(error, &entry.path))?; + self.start_hashing( + entry.path, + Source { + canonical: entry.canonical, + stat: entry.stat, + }, + file, + ) + } + } + } + + fn start_hashing( + &mut self, + path: TreePath, + source: Source, + file: File, + ) -> Result<(), ProtocolError> { + let opened = EntryStat::of_file(&file).map_err(|error| io_error(error, &path))?; + if !opened.is_file() || !opened.unchanged(&source.stat) { + return Err(source_changed(&path)); + } + if source.stat.size > self.limits.max_file_bytes { + return Err(ProtocolError::payload_too_large( + self.limits.max_file_bytes, + format!("{path} is larger than limits.maxFileBytes"), + )); + } + self.total_bytes = self.total_bytes.saturating_add(source.stat.size); + if self.total_bytes > self.limits.max_total_bytes { + return Err(ProtocolError::payload_too_large( + self.limits.max_total_bytes, + "the capture exceeds limits.maxTotalBytes", + )); + } + self.hashing = Some(Hashing { + path, + source, + file, + hash: Sha256::new(), + read: 0, + }); + Ok(()) + } + + /// Hash within `allowance`; `true` when the file is done. + fn hash_step(&mut self, allowance: &mut u64) -> Result { + let task = self.hashing.as_mut().expect("caller checked"); + let mut buffer = vec![0u8; 64 * 1024]; + loop { + if *allowance == 0 { + return Ok(false); + } + let limit = (buffer.len() as u64).min(*allowance) as usize; + let read = task + .file + .read(&mut buffer[..limit]) + .map_err(|error| io_error(error, &task.path))?; + if read == 0 { + break; + } + task.read += read as u64; + if task.read > task.source.stat.size { + return Err(source_changed(&task.path)); + } + task.hash.update(&buffer[..read]); + *allowance -= read as u64; + } + let task = self.hashing.take().expect("hashing above"); + let now = EntryStat::of_file(&task.file).map_err(|error| io_error(error, &task.path))?; + if task.read != task.source.stat.size || !now.unchanged(&task.source.stat) { + return Err(source_changed(&task.path)); + } + let digest = ContentDigest::from_sha256(task.hash.finalize().into()); + let executable = task.source.stat.executable(); + let size_bytes = task.source.stat.size; + self.sources.entry(digest).or_insert(task.source); + self.record(InventoryEntry { + path: task.path, + content: TreeContent::File { + size_bytes, + executable, + digest: Some(digest), + }, + })?; + Ok(true) + } + + fn record(&mut self, entry: InventoryEntry) -> Result<(), ProtocolError> { + check_entry_quota(self.entries.len() + 1, &self.limits)?; + self.manifest_bytes += entry_bytes(&entry); + if self.manifest_bytes > self.limits.max_manifest_bytes { + return Err(ProtocolError::payload_too_large( + self.limits.max_manifest_bytes, + "the inventory exceeds limits.maxManifestBytes", + )); + } + self.entries.push(entry); + Ok(()) + } + + fn skip(&mut self, path: Option, reason: TransferSkipReason) { + self.skipped_count = self.skipped_count.saturating_add(1); + if (self.skipped.len() as u32) < MAX_LISTED_SKIPS { + self.skipped.push(TransferSkip { path, reason }); + } + } +} + +/// Why reading captured content failed. +pub(super) enum ReadFailure { + /// The bytes are not the bytes that were hashed. + Changed(ProtocolError), + Io(ProtocolError), +} + +/// Read `max` bytes of a captured source from `offset`: `(data, truncated)`. +pub(super) fn read_content( + confinement: &Confinement, + source: &Source, + offset: u64, + max: u64, +) -> Result<(Vec, bool), ReadFailure> { + let subject = format!("captured {}", source_label(source)); + let resolved = confinement + .reopen(&source.canonical, &subject) + .map_err(|_| ReadFailure::Changed(source_changed(&subject)))?; + let mut file = resolved + .parent + .open_read(&resolved.name) + .map_err(|error| ReadFailure::Io(io_error(error, &subject)))?; + let unchanged = |file: &File| { + EntryStat::of_file(file).is_ok_and(|stat| stat.is_file() && stat.unchanged(&source.stat)) + }; + if !unchanged(&file) { + return Err(ReadFailure::Changed(source_changed(&subject))); + } + file.seek(SeekFrom::Start(offset)) + .map_err(|error| ReadFailure::Io(io_error(error, &subject)))?; + let take = max.min(source.stat.size - offset); + let data = os::read_up_to(&mut file, take) + .map_err(|error| ReadFailure::Io(io_error(error, &subject)))?; + if data.len() as u64 != take || !unchanged(&file) { + return Err(ReadFailure::Changed(source_changed(&subject))); + } + let truncated = offset + take < source.stat.size; + Ok((data, truncated)) +} + +fn source_label(source: &Source) -> String { + source + .canonical + .last() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| ".".into()) +} + +pub(super) fn source_changed(subject: &dyn std::fmt::Display) -> ProtocolError { + ProtocolError::aborted( + Some(AbortReason::SourceChanged), + format!("{subject} changed during the transfer"), + ) +} + +fn symlink_rejected(path: &str, why: &str) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::SymlinkRejected, + format!("{path}: {why}"), + ) +} diff --git a/crates/environment-daemon/src/filesystem/transfer/journal.rs b/crates/environment-daemon/src/filesystem/transfer/journal.rs index 20d958f83..5dcb66c89 100644 --- a/crates/environment-daemon/src/filesystem/transfer/journal.rs +++ b/crates/environment-daemon/src/filesystem/transfer/journal.rs @@ -1,39 +1,247 @@ -//! Durable publication receipts. Interrupted in-flight operations fail closed after -//! restart; they are never silently re-executed against a possibly edited target. -use super::*; +//! Durable transfer records under `/transfers`. +//! +//! An operation is recorded at begin, before its staging directory exists, +//! so a restart can discard its staging directory and spool; when its commit +//! starts; and when it finishes, as its receipt. A receipt keeps the staging +//! and spool records until a sweep confirmed their removal. A completed +//! capture also keeps its sealed inventory beside its receipt, one entry +//! per line, so a retried capture can page it again. The +//! record is this daemon's own file: it is never read by another version, +//! and an undecodable record is renamed aside instead of failing startup. + +use std::{ + ffi::OsString, + fs::File, + io::{BufRead, BufReader, BufWriter, Read, Write}, + os::unix::ffi::{OsStrExt, OsStringExt}, + path::{Path, PathBuf}, +}; + +use environment_protocol::{ + data::transfer::{BeginTransferParams, InventoryEntry, TransferStatus}, + error::ProtocolError, + ids::TransferOperationId, +}; use serde::{Deserialize, Serialize}; -use std::path::PathBuf; -#[derive(Serialize, Deserialize)] -pub struct Record { - pub root: PathBuf, - pub request: TransferRequest, + +use crate::filesystem::io_error; + +/// Largest record read back; records hold no inventories. +const MAX_RECORD_BYTES: u64 = 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Record { + pub params: BeginTransferParams, pub status: TransferStatus, - pub expires_at_ms: u64, - pub stage_name: Option, - #[serde(default)] - pub spool_directory: Option, + /// The staging directory of an active publish. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stage: Option, + /// The upload spool of an active publish. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub spool: Option, + /// A publish's commit started: after a restart, whether it published + /// is unknown. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub committing: bool, } -pub fn now_ms() -> u64 { + +/// Where a staging directory lives: the canonical segments of its parent +/// below the filesystem root (raw bytes, names need not be UTF-8) and its +/// name. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct StageRecord { + pub parent: Vec>, + pub name: String, +} + +impl StageRecord { + pub fn new(parent: &[OsString], name: &str) -> Self { + Self { + parent: parent + .iter() + .map(|segment| segment.as_bytes().to_vec()) + .collect(), + name: name.to_owned(), + } + } + + pub fn parent_segments(&self) -> Vec { + self.parent + .iter() + .map(|segment| OsString::from_vec(segment.clone())) + .collect() + } +} + +pub(super) fn now_ms() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis().min(i64::MAX as u128) as i64) .unwrap_or_default() - .as_millis() as u64 } -pub fn write(directory: &Path, id: &str, record: &Record) -> Result<()> { + +fn record_path(directory: &Path, id: &TransferOperationId) -> PathBuf { + directory.join(format!("{}.json", id.as_str())) +} + +const INVENTORY_EXTENSION: &str = "inventory"; + +fn inventory_path(directory: &Path, id: &TransferOperationId) -> PathBuf { + directory.join(format!("{}.{INVENTORY_EXTENSION}", id.as_str())) +} + +/// Write a completed capture's inventory atomically and durably, before +/// its receipt. +pub(super) fn write_inventory( + directory: &Path, + id: &TransferOperationId, + entries: &[InventoryEntry], +) -> Result<(), ProtocolError> { + let io = |error| io_error(error, format!("transfer journal {}", directory.display())); + let temporary = tempfile::NamedTempFile::new_in(directory).map_err(io)?; + let mut writer = BufWriter::new(temporary); + for entry in entries { + serde_json::to_writer(&mut writer, entry).map_err(|error| { + ProtocolError::internal(format!("encode transfer inventory: {error}")) + })?; + writer.write_all(b"\n").map_err(io)?; + } + let temporary = writer + .into_inner() + .map_err(|error| io(error.into_error()))?; + temporary.as_file().sync_all().map_err(io)?; + temporary + .persist(inventory_path(directory, id)) + .map_err(|error| io(error.error))?; + File::open(directory) + .and_then(|file| file.sync_all()) + .map_err(io) +} + +/// Up to `limit` entries of a retained inventory from `index`, and whether +/// more follow; `Ok(None)` when none is retained. +pub(super) fn read_inventory( + directory: &Path, + id: &TransferOperationId, + index: usize, + limit: usize, +) -> Result, bool)>, ProtocolError> { + let path = inventory_path(directory, id); + let file = match File::open(&path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(io_error(error, path.display())), + }; + let mut lines = BufReader::new(file).lines().skip(index); + let mut entries = Vec::new(); + for line in lines.by_ref().take(limit) { + let line = line.map_err(|error| io_error(error, path.display()))?; + entries.push(serde_json::from_str(&line).map_err(|error| { + ProtocolError::internal(format!("decode transfer inventory: {error}")) + })?); + } + Ok(Some((entries, lines.next().is_some()))) +} + +/// Write a record atomically and durably. +pub(super) fn write( + directory: &Path, + id: &TransferOperationId, + record: &Record, +) -> Result<(), ProtocolError> { + let io = |error| io_error(error, format!("transfer journal {}", directory.display())); let mut temporary = tempfile::NamedTempFile::new_in(directory).map_err(io)?; - serde_json::to_writer(&mut temporary, record).map_err(|e| invalid(&e.to_string()))?; + serde_json::to_writer(&mut temporary, record) + .map_err(|error| ProtocolError::internal(format!("encode transfer record: {error}")))?; temporary.as_file().sync_all().map_err(io)?; temporary - .persist(directory.join(format!("{id}.json"))) - .map_err(|e| io(e.error))?; - backend::sync_directory(directory).map_err(io) -} -pub fn read(directory: &Path, id: &str) -> Result> { - match File::open(directory.join(format!("{id}.json"))) { - Ok(file) => Ok(Some( - serde_json::from_reader(file.take(64 * 1024)).map_err(|e| invalid(&e.to_string()))?, - )), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(io(e)), + .persist(record_path(directory, id)) + .map_err(|error| io(error.error))?; + File::open(directory) + .and_then(|file| file.sync_all()) + .map_err(io) +} + +/// Read a record; `Ok(None)` when there is none or it was quarantined. +pub(super) fn read( + directory: &Path, + id: &TransferOperationId, +) -> Result, ProtocolError> { + read_path(&record_path(directory, id)) +} + +fn read_path(path: &Path) -> Result, ProtocolError> { + let file = match File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(io_error(error, path.display())), + }; + match serde_json::from_reader(file.take(MAX_RECORD_BYTES)) { + Ok(record) => Ok(Some(record)), + Err(error) => { + quarantine(path, &error.to_string()); + Ok(None) + } + } +} + +/// Remove a record and any inventory retained with it. +pub(super) fn remove(directory: &Path, id: &TransferOperationId) { + let _ = std::fs::remove_file(record_path(directory, id)); + let _ = std::fs::remove_file(inventory_path(directory, id)); +} + +/// Retained inventories whose record is gone (a crash between writing the +/// inventory and its receipt, or between removing the two). +pub(super) fn remove_orphan_inventories(directory: &Path) { + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path + .extension() + .is_some_and(|extension| extension == INVENTORY_EXTENSION) + && !path.with_extension("json").exists() + { + let _ = std::fs::remove_file(path); + } + } +} + +fn quarantine(path: &Path, reason: &str) { + let aside = path.with_extension("quarantined"); + eprintln!( + "lightspeed-envd: unreadable transfer record {} moved to {}: {reason}", + path.display(), + aside.display() + ); + let _ = std::fs::rename(path, aside); +} + +/// Every readable record with its id, in no particular order. +pub(super) fn all(directory: &Path) -> Vec<(TransferOperationId, Record)> { + let Ok(entries) = std::fs::read_dir(directory) else { + return Vec::new(); + }; + let mut records = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().is_none_or(|extension| extension != "json") { + continue; + } + let Some(id) = path + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| TransferOperationId::new(stem).ok()) + else { + continue; + }; + if let Ok(Some(record)) = read_path(&path) { + records.push((id, record)); + } } + records } diff --git a/crates/environment-daemon/src/filesystem/transfer/mod.rs b/crates/environment-daemon/src/filesystem/transfer/mod.rs index 712cf9f78..7bfb321c4 100644 --- a/crates/environment-daemon/src/filesystem/transfer/mod.rs +++ b/crates/environment-daemon/src/filesystem/transfer/mod.rs @@ -1,252 +1,1177 @@ -//! Compatibility adapters for small inline copies. All confinement, hashing and -//! publication use the same backend as paged transfers on Linux and macOS. -#[cfg(any(target_os = "linux", target_os = "macos"))] -pub(super) mod session; -use super::*; -use environment_protocol::data::{inventory::*, transfer_session::*}; -use sha2::{Digest, Sha256}; -fn invalid(message: &str) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::InvalidRequest, message) -} -fn limits(value: TransferLimits) -> Result { - if value.max_entries == 0 - || value.max_entries > MAX_TRANSFER_ENTRIES - || value.max_depth > MAX_TRANSFER_DEPTH - || value.max_file_bytes > MAX_TRANSFER_BYTES - || value.max_total_bytes > MAX_TRANSFER_BYTES - || value.max_duration_ms == 0 - || value.max_duration_ms > MAX_TRANSFER_DURATION_MS - { - return Err(invalid("inline transfer limits exceed protocol ceilings")); - } - Ok(InventoryLimits { - max_entries: value.max_entries, - max_depth: value.max_depth, - max_file_bytes: value.max_file_bytes, - max_total_bytes: value.max_total_bytes, - max_duration_ms: value.max_duration_ms, - ..Default::default() - }) -} -async fn advance( - fs: &LocalFileSystem, - id: &str, - phase: TransferPhase, -) -> Result<(), EnvironmentProtocolError> { - loop { - if let TransferResponse::Status(status) = fs - .transfer(TransferRequest::Advance { - operation_id: id.into(), - }) - .await? - && status.phase == phase +//! `transfer/*`: one logical capture, materialize or file write as many +//! bounded calls. +//! +//! Each operation has its own lock; uploads and captured reads of +//! different digests proceed concurrently. Only active operations are held +//! in memory and count against `maxActiveOperations`: a finished operation +//! (complete, aborted or failed) is journaled as its receipt, evicted, and +//! answered from the journal for `receiptRetentionMs` after it finished. +//! A completed capture's sealed inventory is retained with its receipt and +//! stays pageable, so a caller retrying a capture whose answer it lost +//! gets the tree it first captured without revisiting the live source. +//! +//! Errors of the operation's own work (walking, hashing, staging, +//! publishing, a source that changed, a digest mismatch, the duration +//! running out) move it to `failed`; every later call except +//! `transfer/getStatus` and `transfer/abort` then answers that error. A +//! request the daemon refuses without acting on it (invalid params, a call +//! in the wrong phase, an unexpected offset, an unknown digest) leaves the +//! operation as it was. + +mod capture; +mod journal; +mod publish; +#[cfg(test)] +mod tests; + +use std::{ + collections::BTreeMap, + path::PathBuf, + sync::{ + Arc, Mutex, MutexGuard, Weak, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use environment_protocol::{ + bytes::ByteChunk, + data::{ + handshake::TransferCeilings, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, InventoryEntry, + ListTransferMissingParams, ListTransferMissingResponse, ReadTransferContentParams, + ReadTransferContentResponse, ReadTransferInventoryParams, + ReadTransferInventoryResponse, TransferLimits, TransferOperationParams, TransferPhase, + TransferSelection, TransferStatus, WriteTransferContentParams, + WriteTransferContentResponse, + }, + }, + error::{BusyReason, Capability, NotFoundReason, PreconditionReason, ProtocolError}, + ids::TransferOperationId, +}; + +use super::{confinement::Confinement, io_error, search::check_quota}; +use capture::{Capture, ReadFailure}; +use journal::Record; +use publish::{ChunkFailure, Kind, Publish}; + +/// Content hashed or copied by one `transfer/advance`. +const STEP_BYTES: u64 = 4 * 1024 * 1024; +/// Entries visited by one `transfer/advance`. +const STEP_ENTRIES: usize = 1024; +/// How often finished receipts and overdue operations are swept. +const SWEEP_INTERVAL: Duration = Duration::from_secs(30); + +fn check_entry_quota(count: usize, limits: &TransferLimits) -> Result<(), ProtocolError> { + if count as u64 > u64::from(limits.max_entries) { + return Err(ProtocolError::too_many_entries( + limits.max_entries, + "the inventory exceeds limits.maxEntries", + )); + } + Ok(()) +} + +fn entry_bytes(entry: &InventoryEntry) -> u64 { + serde_json::to_vec(entry).map_or(0, |encoded| encoded.len() as u64) +} + +/// The daemon's transfer operations. +#[derive(Clone)] +pub(crate) struct Transfers { + shared: Arc, +} + +struct Shared { + confinement: Arc, + journal: PathBuf, + ceilings: TransferCeilings, + writable: bool, + active: Mutex>>, +} + +struct Operation { + id: TransferOperationId, + params: BeginTransferParams, + /// Canonical destination of a publish, for overlap checks. + destination: Option>, + /// Set (under `state`) once the operation finished; read without it. + finished: AtomicBool, + state: Mutex, +} + +struct State { + status: TransferStatus, + deadline_ms: i64, + work: Work, + /// The receipt of a finished operation whose journal write failed. The + /// operation stays in memory, answering from here, until a retry + /// persists it. + unpersisted: Option, +} + +enum Work { + Capture(Box), + Publish(Box), + Finished, +} + +/// How a finished operation answers each call. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Call { + Begin, + Abort, + Commit, + ReadInventory, + Other, +} + +enum Found { + Active(Arc), + Finished(TransferStatus), +} + +impl Transfers { + /// Journal under `/transfers`. Operations a restart + /// interrupted are recorded as failed (`aborted`) and their staging is + /// discarded. + pub fn new( + confinement: Arc, + ceilings: TransferCeilings, + writable: bool, + ) -> Result { + let journal = confinement.state_dir().join("transfers"); { - return Ok(()); + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new() + .recursive(true) + .mode(super::os::PRIVATE_DIRECTORY_MODE) + .create(&journal) + .map_err(|error| io_error(error, journal.display()))?; + } + let shared = Arc::new(Shared { + confinement, + journal, + ceilings, + writable, + active: Mutex::new(BTreeMap::new()), + }); + shared.recover(); + let weak = Arc::downgrade(&shared); + std::thread::Builder::new() + .name("transfer-sweep".into()) + .spawn(move || sweep_loop(weak)) + .map_err(|error| ProtocolError::internal(format!("start transfer sweep: {error}")))?; + Ok(Self { shared }) + } + + pub fn begin(&self, params: BeginTransferParams) -> Result { + self.shared.begin(params) + } + + pub fn advance( + &self, + params: TransferOperationParams, + ) -> Result { + self.shared.advance(¶ms.operation_id) + } + + pub fn get_status( + &self, + params: TransferOperationParams, + ) -> Result { + match self.shared.find(¶ms.operation_id)? { + Found::Finished(status) => Ok(status), + Found::Active(operation) => Ok(operation.lock().status.clone()), } } + + pub fn read_inventory( + &self, + params: ReadTransferInventoryParams, + ) -> Result { + self.shared.read_inventory(params) + } + + pub fn append( + &self, + params: AppendTransferInventoryParams, + ) -> Result { + self.shared.append(params) + } + + pub fn list_missing( + &self, + params: ListTransferMissingParams, + ) -> Result { + self.shared.list_missing(params) + } + + pub fn read( + &self, + params: ReadTransferContentParams, + ) -> Result { + self.shared.read(params) + } + + pub fn write( + &self, + params: WriteTransferContentParams, + ) -> Result { + self.shared.write(params) + } + + pub fn commit(&self, params: TransferOperationParams) -> Result { + self.shared.commit(¶ms.operation_id) + } + + pub fn abort(&self, params: TransferOperationParams) -> Result { + self.shared.abort(¶ms.operation_id) + } +} + +impl Operation { + fn lock(&self) -> MutexGuard<'_, State> { + self.state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } } -pub(super) async fn capture( - fs: &LocalFileSystem, - params: CaptureParams, -) -> Result { - let id = format!("inline-{:032x}", rand::random::()); - let result = async { - fs.transfer(TransferRequest::Begin { + +impl Shared { + fn retention_ms(&self) -> i64 { + i64::try_from(self.ceilings.receipt_retention_ms).unwrap_or(i64::MAX) + } + + fn find(&self, id: &TransferOperationId) -> Result { + if let Some(operation) = self.active_map().get(id).cloned() { + return Ok(Found::Active(operation)); + } + let Some(record) = journal::read(&self.journal, id)? else { + return Err(ProtocolError::not_found(format!("unknown transfer {id}"))); + }; + if !record.status.phase.is_finished() { + // Only a record left by an earlier daemon instance can be + // active without being in memory; recovery normally converts + // it already. + return Ok(Found::Finished(self.interrupted(id, record))); + } + if record + .status + .receipt_expires_at_ms + .is_some_and(|expires| expires <= journal::now_ms()) + { + return Err(ProtocolError::not_found_because( + NotFoundReason::Expired, + format!("the receipt of transfer {id} expired"), + )); + } + Ok(Found::Finished(record.status)) + } + + fn active_map(&self) -> MutexGuard<'_, BTreeMap>> { + self.active + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// Drop a finished operation from memory once its receipt is on disk; + /// retries a receipt write that failed. + fn evict(&self, operation: &Arc) { + if !operation.finished.load(Ordering::Acquire) { + return; + } + { + let mut state = operation.lock(); + if let Some(record) = &state.unpersisted { + if let Err(error) = journal::write(&self.journal, &operation.id, record) { + eprintln!( + "lightspeed-envd: journaling the receipt of transfer {}: {error}", + operation.id + ); + return; + } + state.unpersisted = None; + } + } + let mut active = self.active_map(); + if active + .get(&operation.id) + .is_some_and(|current| Arc::ptr_eq(current, operation)) + { + active.remove(&operation.id); + } + } + + /// Run `body` on an active operation, answering finished ones per + /// `call`, failing an overdue one, and evicting it once finished. + fn with_active( + &self, + id: &TransferOperationId, + call: Call, + body: impl FnOnce(&Self, &Arc, &mut State) -> Result, + finished: impl FnOnce(TransferStatus) -> Result, + ) -> Result { + let operation = match self.find(id)? { + Found::Finished(status) => return answer_finished(status, call).and_then(finished), + Found::Active(operation) => operation, + }; + let result = { + let mut state = operation.lock(); + if state.status.phase.is_finished() { + answer_finished(state.status.clone(), call).and_then(finished) + } else if call != Call::Abort && journal::now_ms() >= state.deadline_ms { + let error = ProtocolError::deadline_exceeded(format!( + "transfer {id} ran past its maxDurationMs" + )); + Err(self.fail(&operation, &mut state, error)) + } else { + body(self, &operation, &mut state) + } + }; + self.evict(&operation); + result + } + + /// Record `phase` (complete, aborted, failed), release the work and + /// journal the receipt. + fn finish( + &self, + operation: &Operation, + state: &mut State, + phase: TransferPhase, + error: Option, + ) { + state.status.phase = phase; + state.status.error = error; + state.status.deadline_at_ms = None; + state.status.receipt_expires_at_ms = + Some(journal::now_ms().saturating_add(self.retention_ms())); + // The staging directory and spool are removed in the background; + // the receipt names them until a sweep confirms they are gone. + let (stage, spool) = match &state.work { + Work::Publish(publish) => (publish.created_stage_record(), publish.spool_path()), + Work::Capture(_) | Work::Finished => (None, None), + }; + state.work = Work::Finished; + let record = Record { + params: operation.params.clone(), + status: state.status.clone(), + stage, + spool, + committing: false, + }; + if let Err(error) = journal::write(&self.journal, &operation.id, &record) { + eprintln!( + "lightspeed-envd: journaling the receipt of transfer {}: {error}", + operation.id + ); + state.unpersisted = Some(record); + } + operation.finished.store(true, Ordering::Release); + } + + fn fail( + &self, + operation: &Operation, + state: &mut State, + error: ProtocolError, + ) -> ProtocolError { + self.finish(operation, state, TransferPhase::Failed, Some(error.clone())); + error + } + + fn begin(&self, params: BeginTransferParams) -> Result { + self.check_limits(¶ms.limits)?; + let publishes = !matches!(params.selection, TransferSelection::Capture { .. }); + if publishes && !self.writable { + return Err(ProtocolError::capability_unavailable( + Capability::FilesystemWrite, + "the filesystem is read-only", + )); + } + let id = params.operation_id.clone(); + if let Some(existing) = self.existing(¶ms)? { + return Ok(existing); + } + // Checked again at insert; this avoids staging work that would be + // refused anyway. + self.check_capacity(&self.active_map())?; + let (mut work, destination) = self.start(¶ms)?; + let now = journal::now_ms(); + let deadline_ms = + now.saturating_add(i64::try_from(params.limits.max_duration_ms).unwrap_or(i64::MAX)); + let (phase, stage, spool) = match &work { + Work::Capture(_) => (TransferPhase::Scanning, None, None), + Work::Publish(publish) => ( + TransferPhase::Inventory, + Some(publish.stage_record()), + publish.spool_path(), + ), + Work::Finished => unreachable!("start returns work"), + }; + let status = TransferStatus { operation_id: id.clone(), - selection: TransferSelection::Capture { - source: params.source.clone(), - }, - limits: limits(params.limits)?, - }) - .await?; - advance(fs, &id, TransferPhase::Ready).await?; - let mut entries = Vec::new(); - let mut offset = 0; - let mut bytes = 0; - loop { - let TransferResponse::Inventory { - entries: page, - next_offset, - } = fs - .transfer(TransferRequest::Inventory { - operation_id: id.clone(), - offset, + phase, + entry_count: 0, + total_bytes: 0, + transferred_bytes: 0, + reused_bytes: 0, + skipped: Vec::new(), + skipped_entry_count: 0, + error: None, + deadline_at_ms: Some(deadline_ms), + receipt_expires_at_ms: None, + }; + let mut active = self.active_map(); + if active.contains_key(&id) { + drop(active); + discard(work); + return self.existing(¶ms)?.ok_or_else(|| { + ProtocolError::internal(format!("transfer {id} vanished during begin")) + }); + } + let admitted = self.check_capacity(&active).and_then(|()| { + let overlaps = destination.as_ref().is_some_and(|destination| { + active.values().any(|other| { + !other.finished.load(Ordering::Acquire) + && other.destination.as_ref().is_some_and(|other| { + other.starts_with(destination) || destination.starts_with(other) + }) }) - .await? - else { - return Err(invalid("unexpected inventory")); + }); + if overlaps { + return Err(ProtocolError::busy( + Some(BusyReason::TransferCapacity), + None, + "the destination overlaps an active transfer", + )); + } + Ok(()) + }); + // The record names the staging directory before it exists, so an + // exit at any later point leaves nothing a restart cannot find. + let recorded = admitted.and_then(|()| { + journal::write( + &self.journal, + &id, + &Record { + params: params.clone(), + status: status.clone(), + stage, + spool, + committing: false, + }, + ) + }); + if let Err(error) = recorded { + drop(active); + discard(work); + return Err(error); + } + if let Work::Publish(publish) = &mut work + && let Err(error) = publish.create_stage() + { + journal::remove(&self.journal, &id); + drop(active); + discard(work); + return Err(error); + } + active.insert( + id.clone(), + Arc::new(Operation { + id, + params, + destination, + finished: AtomicBool::new(false), + state: Mutex::new(State { + status: status.clone(), + deadline_ms, + work, + unpersisted: None, + }), + }), + ); + Ok(status) + } + + fn check_capacity( + &self, + active: &BTreeMap>, + ) -> Result<(), ProtocolError> { + let running = active + .values() + .filter(|operation| !operation.finished.load(Ordering::Acquire)) + .count(); + if running >= self.ceilings.max_active_operations as usize { + return Err(ProtocolError::busy( + Some(BusyReason::TransferCapacity), + None, + format!( + "{running} transfers are active; the ceiling is {}", + self.ceilings.max_active_operations + ), + )); + } + Ok(()) + } + + /// The answer to a repeated begin, or `None` for a new id. + fn existing( + &self, + params: &BeginTransferParams, + ) -> Result, ProtocolError> { + let id = ¶ms.operation_id; + let in_use = || { + ProtocolError::failed_precondition( + PreconditionReason::OperationIdInUse, + format!("transfer {id} was begun with different input"), + ) + }; + let active = self.active_map().get(id).cloned(); + if let Some(operation) = active { + if operation.params != *params { + return Err(in_use()); + } + let status = operation.lock().status.clone(); + return answer_finished_or_active(status, Call::Begin).map(Some); + } + match journal::read(&self.journal, id)? { + None => Ok(None), + Some(record) if record.params != *params => Err(in_use()), + Some(_) => match self.find(id)? { + Found::Finished(status) => answer_finished(status, Call::Begin).map(Some), + Found::Active(operation) => { + let status = operation.lock().status.clone(); + answer_finished_or_active(status, Call::Begin).map(Some) + } + }, + } + } + + fn check_limits(&self, limits: &TransferLimits) -> Result<(), ProtocolError> { + let ceilings = &self.ceilings; + check_quota( + "limits.maxEntries", + limits.max_entries.into(), + ceilings.max_entries.into(), + )?; + let above = |field: &str, value: u64, ceiling: u64| { + if value > ceiling { + Err(ProtocolError::above_ceiling(field, value, ceiling)) + } else { + Ok(()) + } + }; + above( + "limits.maxDepth", + limits.max_depth.into(), + ceilings.max_depth.into(), + )?; + above( + "limits.maxFileBytes", + limits.max_file_bytes, + ceilings.max_file_bytes, + )?; + above( + "limits.maxTotalBytes", + limits.max_total_bytes, + ceilings.max_total_bytes, + )?; + check_quota( + "limits.maxManifestBytes", + limits.max_manifest_bytes, + ceilings.max_manifest_bytes, + )?; + check_quota( + "limits.maxDurationMs", + limits.max_duration_ms, + ceilings.max_duration_ms, + ) + } + + fn start( + &self, + params: &BeginTransferParams, + ) -> Result<(Work, Option>), ProtocolError> { + let confinement = self.confinement.clone(); + let limits = params.limits; + let (destination, kind) = match ¶ms.selection { + TransferSelection::Capture { + source, + filter, + symlinks, + special_files, + } => { + let capture = Capture::begin( + confinement, + source, + filter, + *symlinks, + *special_files, + limits, + )?; + return Ok((Work::Capture(Box::new(capture)), None)); + } + TransferSelection::Materialize { + destination, + on_existing, + } => ( + destination, + Kind::Materialize { + on_existing: *on_existing, + }, + ), + TransferSelection::WriteFile { + destination, + create, + if_match, + executable, + } => ( + destination, + Kind::WriteFile { + create: *create, + if_match: if_match.clone(), + executable: *executable, + }, + ), + }; + let spool = tempfile::Builder::new() + .prefix(&format!("{}.spool-", params.operation_id)) + .tempdir_in(&self.journal) + .map_err(|error| io_error(error, self.journal.display()))?; + let publish = Publish::begin(confinement, destination, kind, limits, spool)?; + let segments = publish.destination_segments.clone(); + Ok((Work::Publish(Box::new(publish)), Some(segments))) + } + + fn advance(&self, id: &TransferOperationId) -> Result { + self.with_active( + id, + Call::Other, + |shared, operation, state| { + let step = match &mut state.work { + Work::Capture(capture) => advance_capture(capture, &mut state.status), + Work::Publish(publish) => advance_publish(publish, &mut state.status), + Work::Finished => Ok(()), + }; + match step { + Ok(()) => Ok(state.status.clone()), + Err(error) => Err(shared.fail(operation, state, error)), + } + }, + Ok, + ) + } + + fn read_inventory( + &self, + params: ReadTransferInventoryParams, + ) -> Result { + check_quota( + "limit", + params.limit.into(), + self.ceilings.max_inventory_page_entries.into(), + )?; + let retained = |status: TransferStatus| { + let index = params.index as usize; + let page = journal::read_inventory( + &self.journal, + ¶ms.operation_id, + index, + params.limit as usize, + )?; + let Some((entries, more)) = page else { + return Err(phase_error("the transfer is finished")); }; - for entry in page { - let content = match entry.content { - InventoryContent::Directory => TransferContent::Directory, - InventoryContent::File { - digest, - size_bytes, - executable, - } => { - let mut data = Vec::with_capacity(size_bytes as usize); - loop { - let TransferResponse::Chunk { data: chunk, eof } = fs - .transfer(TransferRequest::Read { - operation_id: id.clone(), - digest: digest.clone(), - offset: data.len() as u64, - }) - .await? - else { - return Err(invalid("unexpected chunk")); - }; - data.extend(chunk.into_inner()); - if eof { - break; - } - } - bytes += size_bytes; - TransferContent::File { - data: ByteChunk::from(data), - executable, - } + if entries.is_empty() && index > status.entry_count as usize { + return Err(ProtocolError::invalid_field( + "index", + format!("the inventory has {} entries", status.entry_count), + )); + } + let next_index = more.then_some(params.index + entries.len() as u32); + Ok(ReadTransferInventoryResponse { + entries, + next_index, + }) + }; + self.with_active( + ¶ms.operation_id, + Call::ReadInventory, + |_, _, state| { + let Work::Capture(capture) = &state.work else { + return Err(phase_error("only a capture has an inventory to read")); + }; + if state.status.phase != TransferPhase::Inventory { + return Err(phase_error("the capture's inventory is not sealed yet")); + } + let count = capture.entries.len(); + let start = params.index as usize; + if start > count { + return Err(ProtocolError::invalid_field( + "index", + format!("the inventory has {count} entries"), + )); + } + let end = (start + params.limit as usize).min(count); + Ok(ReadTransferInventoryResponse { + entries: capture.entries[start..end].to_vec(), + next_index: (end < count).then_some(end as u32), + }) + }, + retained, + ) + } + + fn append( + &self, + params: AppendTransferInventoryParams, + ) -> Result { + let page = self.ceilings.max_inventory_page_entries; + if params.entries.len() as u64 > u64::from(page) { + return Err(ProtocolError::above_ceiling( + "entries", + params.entries.len() as u64, + page.into(), + )); + } + self.with_active( + ¶ms.operation_id, + Call::Other, + |shared, operation, state| { + let Work::Publish(publish) = &mut state.work else { + return Err(phase_error("a capture takes no inventory")); + }; + // A retried page, the sealing one included, answers the + // current status in any phase. + if publish.is_repeat(params.index, ¶ms.entries, params.last) { + return Ok(state.status.clone()); + } + if state.status.phase != TransferPhase::Inventory { + return Err(phase_error("the inventory is already sealed")); + } + let sealed = publish.append(params.index, params.entries, params.last)?; + state.status.entry_count = publish.entries.len() as u32; + state.status.total_bytes = publish.total_bytes; + if sealed { + if let Err(error) = publish.start_scan() { + return Err(shared.fail(operation, state, error)); } + state.status.phase = TransferPhase::Scanning; + } + Ok(state.status.clone()) + }, + |_| Err(phase_error("the transfer is finished")), + ) + } + + fn list_missing( + &self, + params: ListTransferMissingParams, + ) -> Result { + check_quota( + "limit", + params.limit.into(), + self.ceilings.max_inventory_page_entries.into(), + )?; + self.with_active( + ¶ms.operation_id, + Call::Other, + |_, _, state| { + let Work::Publish(publish) = &state.work else { + return Err(phase_error("a capture needs no content")); }; - entries.push(TransferEntry { - path: entry.path, - content, - }); + if !matches!( + state.status.phase, + TransferPhase::Content | TransferPhase::Staging | TransferPhase::Ready + ) { + return Err(phase_error("the destination scan has not finished")); + } + let missing = publish.missing_digests(); + let limit = params.limit as usize; + Ok(ListTransferMissingResponse { + remaining_count: missing.len().saturating_sub(limit) as u32, + digests: missing.into_iter().take(limit).collect(), + }) + }, + |_| Err(phase_error("the transfer is finished")), + ) + } + + fn read( + &self, + params: ReadTransferContentParams, + ) -> Result { + check_quota( + "maxBytes", + params.max_bytes, + self.ceilings.max_content_chunk_bytes, + )?; + let id = params.operation_id.clone(); + let (operation, source) = self.with_active( + &id, + Call::Other, + |_, operation, state| { + let Work::Capture(capture) = &state.work else { + return Err(phase_error("only a capture serves content")); + }; + if state.status.phase != TransferPhase::Inventory { + return Err(phase_error("the capture's inventory is not sealed yet")); + } + let source = capture + .sources + .get(¶ms.digest) + .cloned() + .ok_or_else(|| { + ProtocolError::not_found(format!("{} is not in the capture", params.digest)) + })?; + if params.offset > source.stat.size { + return Err(ProtocolError::invalid_field( + "offset", + format!("the content has {} bytes", source.stat.size), + )); + } + Ok((operation.clone(), source)) + }, + |_| Err(phase_error("the transfer is finished")), + )?; + let read = + capture::read_content(&self.confinement, &source, params.offset, params.max_bytes); + let mut state = operation.lock(); + let result = match read { + Ok((data, truncated)) => { + if state.status.phase == TransferPhase::Inventory { + state.status.transferred_bytes += data.len() as u64; + } + Ok(ReadTransferContentResponse { + data: ByteChunk::from(data), + truncated, + }) } - if let Some(next) = next_offset { - offset = next; - } else { - break; + Err(ReadFailure::Changed(error) | ReadFailure::Io(error)) => { + if state.status.phase.is_finished() { + Err(error) + } else { + Err(self.fail(&operation, &mut state, error)) + } } + }; + drop(state); + self.evict(&operation); + result + } + + fn write( + &self, + params: WriteTransferContentParams, + ) -> Result { + let chunk = self.ceilings.max_content_chunk_bytes; + if params.data.len() as u64 > chunk { + return Err(ProtocolError::above_ceiling( + "data", + params.data.len() as u64, + chunk, + )); } - fs.transfer(TransferRequest::Commit { - operation_id: id.clone(), - }) - .await?; - Ok(CaptureResponse { - source: params.source, - entries, - bytes, - }) - } - .await; - if result.is_err() { - let _ = fs - .transfer(TransferRequest::Abort { operation_id: id }) - .await; - } - result -} -pub(super) async fn materialize( - fs: &LocalFileSystem, - params: MaterializeParams, -) -> Result { - let quota = limits(params.limits)?; - if params.entries.len() > quota.max_entries as usize { - return Err(invalid("inline entry quota exceeded")); - } - let id = format!("inline-{:032x}", rand::random::()); - let result = async { - let mut content = std::collections::BTreeMap::new(); - let mut entries = Vec::new(); - let mut bytes = 0u64; - for entry in ¶ms.entries { - let kind = match &entry.content { - TransferContent::Directory => InventoryContent::Directory, - TransferContent::File { data, executable } => { - bytes += data.as_slice().len() as u64; - if bytes > quota.max_total_bytes { - return Err(invalid("inline byte quota exceeded")); + let id = params.operation_id.clone(); + let (operation, upload) = self.with_active( + &id, + Call::Other, + |shared, operation, state| { + let Work::Publish(publish) = &mut state.work else { + return Err(phase_error("a capture takes no content")); + }; + if state.status.phase != TransferPhase::Content { + return Err(phase_error("the operation is not collecting content")); + } + let upload = match publish.upload(¶ms.digest) { + Ok(Some(upload)) => upload, + Ok(None) => { + return Err(ProtocolError::not_found(format!( + "{} is not in the inventory", + params.digest + ))); + } + Err(error) => return Err(shared.fail(operation, state, error)), + }; + Ok((operation.clone(), upload)) + }, + |_| Err(phase_error("the transfer is finished")), + )?; + let written = { + let mut upload = upload + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let before = upload.length(); + upload + .write(¶ms.digest, params.offset, params.data.as_slice()) + .map(|next| (next, next.saturating_sub(before))) + }; + let mut state = operation.lock(); + let result = match written { + Ok((next_offset, added)) => { + state.status.transferred_bytes += added; + Ok(WriteTransferContentResponse { next_offset }) + } + Err(ChunkFailure::Offset(error)) => Err(error), + Err(ChunkFailure::Mismatch(error) | ChunkFailure::Io(error)) => { + if state.status.phase.is_finished() { + Err(error) + } else { + Err(self.fail(&operation, &mut state, error)) + } + } + }; + drop(state); + self.evict(&operation); + result + } + + fn commit(&self, id: &TransferOperationId) -> Result { + self.with_active( + id, + Call::Commit, + |shared, operation, state| { + let outcome = match &mut state.work { + // The inventory is retained before the receipt that + // answers for it. + Work::Capture(capture) if state.status.phase == TransferPhase::Inventory => { + journal::write_inventory(&shared.journal, &operation.id, &capture.entries) } - let digest = format!("sha256:{}", hex::encode(Sha256::digest(data.as_slice()))); - content.insert(digest.clone(), data.as_slice()); - InventoryContent::File { - digest, - size_bytes: data.as_slice().len() as u64, - executable: *executable, + Work::Publish(publish) if state.status.phase == TransferPhase::Ready => { + // Recorded first: a restart during the commit must + // not claim that nothing was published. + journal::write( + &shared.journal, + &operation.id, + &Record { + params: operation.params.clone(), + status: state.status.clone(), + stage: publish.created_stage_record(), + spool: publish.spool_path(), + committing: true, + }, + )?; + publish.commit() } + _ => return Err(phase_error("the operation is not ready to commit")), + }; + match outcome { + Ok(()) => { + shared.finish(operation, state, TransferPhase::Complete, None); + Ok(state.status.clone()) + } + Err(error) => Err(shared.fail(operation, state, error)), } - }; - entries.push(InventoryEntry { - path: entry.path.clone(), - content: kind, - }); + }, + Ok, + ) + } + + fn abort(&self, id: &TransferOperationId) -> Result { + self.with_active( + id, + Call::Abort, + |shared, operation, state| { + shared.finish(operation, state, TransferPhase::Aborted, None); + Ok(state.status.clone()) + }, + Ok, + ) + } + + /// Convert operations left active by an earlier daemon instance and + /// drop expired receipts. + fn recover(&self) { + for (id, record) in journal::all(&self.journal) { + if !record.status.phase.is_finished() { + self.interrupted(&id, record); + } else { + self.settle_receipt(&id, record, journal::now_ms()); + } } - if entries.is_empty() { - return Err(invalid("empty inventory")); + journal::remove_orphan_inventories(&self.journal); + // Spools of operations without a record (a crash between creating + // the spool and the first journal write). + if let Ok(entries) = std::fs::read_dir(&self.journal) { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some((id, _)) = name.split_once(".spool-") + && let Ok(id) = TransferOperationId::new(id) + && journal::read(&self.journal, &id).ok().flatten().is_none() + { + let _ = std::fs::remove_dir_all(entry.path()); + } + } } - fs.transfer(TransferRequest::Begin { - operation_id: id.clone(), - selection: TransferSelection::Materialize { - destination: params.destination.clone(), - on_existing: params.on_existing, + } + + /// Remove the staging directory and spool a record names. + fn release(&self, record: &mut Record) { + if let Some(stage) = record.stage.take() { + publish::discard_stage(&self.confinement, &stage); + } + if let Some(spool) = record.spool.take() + && spool.parent() == Some(self.journal.as_path()) + { + let _ = std::fs::remove_dir_all(spool); + } + } + + /// A journaled receipt: removed once expired; otherwise its staging + /// and spool, which the finishing instance removes in the background, + /// are made sure of and dropped from the record. + fn settle_receipt(&self, id: &TransferOperationId, mut record: Record, now: i64) { + let expired = record + .status + .receipt_expires_at_ms + .is_none_or(|expires| expires <= now); + if expired { + self.release(&mut record); + journal::remove(&self.journal, id); + } else if record.stage.is_some() || record.spool.is_some() { + self.release(&mut record); + if let Err(error) = journal::write(&self.journal, id, &record) { + eprintln!("lightspeed-envd: settling the receipt of transfer {id}: {error}"); + } + } + } + + /// Record an operation a restart interrupted as failed (`aborted`) and + /// discard its staging and spool. + fn interrupted(&self, id: &TransferOperationId, mut record: Record) -> TransferStatus { + self.release(&mut record); + record.status.phase = TransferPhase::Failed; + record.status.error = Some(ProtocolError::aborted( + None, + if record.committing { + "the daemon restarted while the transfer was committing; the destination may \ + hold the published tree" + } else { + "the daemon restarted while the transfer was active; nothing was committed" }, - limits: quota, - }) - .await?; - advance(fs, &id, TransferPhase::Inventory).await?; - for (index, page) in entries.chunks(MAX_INVENTORY_PAGE).enumerate() { - fs.transfer(TransferRequest::Append { - operation_id: id.clone(), - offset: (index * MAX_INVENTORY_PAGE) as u32, - entries: page.to_vec(), - last: (index + 1) * MAX_INVENTORY_PAGE >= entries.len(), - }) - .await?; + )); + record.committing = false; + record.status.deadline_at_ms = None; + record.status.receipt_expires_at_ms = + Some(journal::now_ms().saturating_add(self.retention_ms())); + if let Err(error) = journal::write(&self.journal, id, &record) { + eprintln!("lightspeed-envd: recording interrupted transfer {id}: {error}"); } - loop { - let TransferResponse::Missing { digests, .. } = fs - .transfer(TransferRequest::Missing { - operation_id: id.clone(), - offset: 0, - }) - .await? - else { - return Err(invalid("unexpected missing response")); - }; - if digests.is_empty() { - break; - } - for digest in digests { - let data = content[&digest]; - if data.is_empty() { - fs.transfer(TransferRequest::Write { - operation_id: id.clone(), - digest, - offset: 0, - data: ByteChunk::from(vec![]), - }) - .await?; - } else { - for (index, chunk) in data.chunks(MAX_CONTENT_CHUNK).enumerate() { - fs.transfer(TransferRequest::Write { - operation_id: id.clone(), - digest: digest.clone(), - offset: (index * MAX_CONTENT_CHUNK) as u64, - data: ByteChunk::from(chunk), - }) - .await?; - } + record.status + } + + /// Fail overdue operations and remove expired receipts. + fn sweep(&self) { + let operations: Vec<_> = self.active_map().values().cloned().collect(); + for operation in operations { + { + let mut state = operation.lock(); + if !state.status.phase.is_finished() && journal::now_ms() >= state.deadline_ms { + let error = ProtocolError::deadline_exceeded(format!( + "transfer {} ran past its maxDurationMs", + operation.id + )); + self.fail(&operation, &mut state, error); } } + self.evict(&operation); } - advance(fs, &id, TransferPhase::Ready).await?; - fs.transfer(TransferRequest::Commit { - operation_id: id.clone(), - }) - .await?; - Ok(MaterializeResponse { - destination: params.destination, - entries: entries.len() as u32, - bytes, - retired_directory: None, - }) - } - .await; - if result.is_err() { - let _ = fs - .transfer(TransferRequest::Abort { operation_id: id }) - .await; - } - result + let now = journal::now_ms(); + // Receipts that finished at least a sweep ago: the background + // removal of their staging had its time. + let finished_before = now.saturating_sub(SWEEP_INTERVAL.as_millis() as i64); + let retention = self.retention_ms(); + for (id, record) in journal::all(&self.journal) { + let due = record.status.receipt_expires_at_ms.is_none_or(|expires| { + expires <= now || expires.saturating_sub(retention) <= finished_before + }); + if record.status.phase.is_finished() && due && !self.active_map().contains_key(&id) { + self.settle_receipt(&id, record, now); + } + } + } +} + +fn sweep_loop(shared: Weak) { + loop { + std::thread::sleep(SWEEP_INTERVAL); + let Some(shared) = shared.upgrade() else { + return; + }; + shared.sweep(); + } +} + +/// Undo a begin refused before its operation was recorded. +fn discard(work: Work) { + if let Work::Publish(publish) = work { + publish.discard(); + } +} + +fn phase_error(message: &str) -> ProtocolError { + ProtocolError::failed_precondition(PreconditionReason::TransferPhase, message) +} + +/// A finished operation's answer: its status for abort, an identical begin +/// and (when complete) commit and inventory reads; its error when it failed; otherwise a phase +/// error. `transfer/getStatus` always answers the status. +fn answer_finished(status: TransferStatus, call: Call) -> Result { + match (call, status.phase) { + (Call::Abort, _) => Ok(status), + (_, TransferPhase::Failed) => Err(status + .error + .clone() + .unwrap_or_else(|| ProtocolError::internal("the transfer failed"))), + (Call::Begin, _) | (Call::Commit | Call::ReadInventory, TransferPhase::Complete) => { + Ok(status) + } + _ => Err(phase_error("the transfer is finished")), + } +} + +fn answer_finished_or_active( + status: TransferStatus, + call: Call, +) -> Result { + if status.phase.is_finished() { + answer_finished(status, call) + } else { + Ok(status) + } +} + +fn advance_capture( + capture: &mut Capture, + status: &mut TransferStatus, +) -> Result<(), ProtocolError> { + if status.phase == TransferPhase::Scanning && capture.advance()? { + status.phase = TransferPhase::Inventory; + } + status.entry_count = capture.entries.len() as u32; + status.total_bytes = capture.total_bytes; + status.skipped = capture.skipped.clone(); + status.skipped_entry_count = capture.skipped_count; + Ok(()) +} + +fn advance_publish( + publish: &mut Publish, + status: &mut TransferStatus, +) -> Result<(), ProtocolError> { + if status.phase == TransferPhase::Scanning && publish.scan_step()? { + status.phase = TransferPhase::Content; + } + if status.phase == TransferPhase::Content && publish.missing_digests().is_empty() { + status.phase = TransferPhase::Staging; + } + if status.phase == TransferPhase::Staging && publish.stage_step()? { + status.phase = TransferPhase::Ready; + } + status.reused_bytes = publish.reused_bytes; + Ok(()) } diff --git a/crates/environment-daemon/src/filesystem/transfer/publish.rs b/crates/environment-daemon/src/filesystem/transfer/publish.rs new file mode 100644 index 000000000..1435f6bbe --- /dev/null +++ b/crates/environment-daemon/src/filesystem/transfer/publish.rs @@ -0,0 +1,970 @@ +//! Materialize and writeFile: append the incoming inventory, scan the +//! destination (hashing only files whose size matches an incoming file), +//! collect missing content, stage the tree next to the destination and +//! publish it with one rename at commit. + +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + ffi::{OsStr, OsString}, + fs::File, + io::{Read, Seek, SeekFrom, Write}, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use environment_protocol::{ + data::{ + fs::{FileKind, FileVersion, TreeContent, WriteCondition}, + transfer::{InventoryEntry, TransferLimits, TransferOnExisting}, + }, + error::{PreconditionReason, ProtocolError}, + paths::{ContentDigest, EnvironmentPath, TreePath}, +}; +use sha2::{Digest, Sha256}; + +use crate::filesystem::{ + confinement::{Confinement, Lookup, Resolved}, + io_error, + os::{self, DEFAULT_DIRECTORY_MODE, DEFAULT_FILE_MODE, Directory, EntryStat, RenameMode}, + walk::{CompiledFilter, Event, LinkMode, Walker}, + write_preconditions, +}; + +use super::{ + STEP_BYTES, STEP_ENTRIES, + capture::{Source, source_changed}, + check_entry_quota, entry_bytes, + journal::StageRecord, +}; + +/// Prefix of staging directories next to a destination. +pub(super) const STAGE_PREFIX: &str = ".lightspeed-transfer-"; +/// The staged tree inside a staging directory. +const STAGED_TREE: &str = "tree"; + +/// What a publish writes. +#[derive(Clone, Debug)] +pub(super) enum Kind { + Materialize { + on_existing: TransferOnExisting, + }, + WriteFile { + create: WriteCondition, + if_match: Option, + executable: bool, + }, +} + +struct Stage { + parent: Arc, + name: OsString, + directory: Directory, +} + +/// Content uploaded for one digest, spooled under the journal directory. +pub(super) struct Upload { + file: File, + path: PathBuf, + hash: Sha256, + length: u64, + size: u64, + complete: bool, +} + +/// Why one uploaded chunk was refused. +pub(super) enum ChunkFailure { + /// The offset is not the digest's next offset and the chunk is not an + /// identical repeat; nothing changed. + Offset(ProtocolError), + /// The completed content does not hash to its digest. + Mismatch(ProtocolError), + Io(ProtocolError), +} + +impl Upload { + pub fn is_complete(&self) -> bool { + self.complete + } + + /// Bytes written so far: the next expected offset. + pub fn length(&self) -> u64 { + self.length + } + + /// Append a chunk at `offset`; returns the next expected offset. + pub fn write( + &mut self, + digest: &ContentDigest, + offset: u64, + data: &[u8], + ) -> Result { + let io = |error| ChunkFailure::Io(io_error(error, digest)); + let end = offset.saturating_add(data.len() as u64); + if offset < self.length { + // An identical repeat of bytes already written is accepted. + if end > self.length { + return Err(ChunkFailure::Offset(offset_error(digest, self.length))); + } + let mut existing = vec![0; data.len()]; + self.file.seek(SeekFrom::Start(offset)).map_err(io)?; + self.file.read_exact(&mut existing).map_err(io)?; + if existing != data { + return Err(ChunkFailure::Offset(offset_error(digest, self.length))); + } + return Ok(self.length); + } + if offset != self.length || self.complete || end > self.size { + return Err(ChunkFailure::Offset(offset_error(digest, self.length))); + } + self.file.seek(SeekFrom::Start(offset)).map_err(io)?; + self.file.write_all(data).map_err(io)?; + self.hash.update(data); + self.length = end; + if self.length == self.size { + let actual = ContentDigest::from_sha256(self.hash.clone().finalize().into()); + if actual != *digest { + return Err(ChunkFailure::Mismatch(ProtocolError::failed_precondition( + PreconditionReason::DigestMismatch, + format!("uploaded content hashes to {actual}, not {digest}"), + ))); + } + self.file.sync_all().map_err(io)?; + self.complete = true; + } + Ok(self.length) + } +} + +fn offset_error(digest: &ContentDigest, next: u64) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::TransferOffset, + format!("the next offset of {digest} is {next}"), + ) +} + +struct Copying { + source: File, + destination: File, + hash: Sha256, + expected: ContentDigest, + copied: u64, + size: u64, + mode: u32, + changed: Option, +} + +struct DestinationScan { + walker: Option, + single: Option, + hashing: Option<(Source, File, Sha256, u64)>, +} + +pub(super) struct Publish { + confinement: Arc, + pub kind: Kind, + destination: EnvironmentPath, + /// The destination's parent directory and its name there. + parent: Arc, + name: OsString, + parent_segments: Vec, + pub destination_segments: Vec, + /// Directories `begin` created for the destination; removed again + /// unless a commit published into them. + created_parents: Vec>, + /// Name of the staging directory, chosen at begin and created by + /// [`Publish::create_stage`] once it is journaled. + stage_name: OsString, + stage: Option, + limits: TransferLimits, + pub entries: Vec, + pages: BTreeMap, bool)>, + paths: HashMap, + sizes: BTreeMap, + pub total_bytes: u64, + manifest_bytes: u64, + scan: Option, + reusable: BTreeMap, + pub uploads: BTreeMap>>, + spool: Option, + staging_index: usize, + copying: Option, + pub reused_bytes: u64, +} + +impl Publish { + pub fn begin( + confinement: Arc, + destination: &EnvironmentPath, + kind: Kind, + limits: TransferLimits, + spool: tempfile::TempDir, + ) -> Result { + let resolved = match &kind { + Kind::Materialize { on_existing } => { + let resolved = confinement.lookup( + destination, + Lookup::NO_FOLLOW.allowing_missing().creating_parents(), + )?; + if let Some(stat) = &resolved.stat { + if stat.kind == FileKind::Symlink { + return Err(ProtocolError::failed_precondition( + PreconditionReason::SymlinkRejected, + format!("{destination} is a symlink"), + )); + } + if stat.kind == FileKind::Other { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotRegularFile, + format!("{destination} is a special file"), + )); + } + if *on_existing == TransferOnExisting::Error { + return Err(ProtocolError::already_exists(format!( + "{destination} exists" + ))); + } + } + resolved + } + Kind::WriteFile { + create, if_match, .. + } => { + let resolved = + confinement.lookup(destination, Lookup::FOLLOW.allowing_missing())?; + write_preconditions( + destination, + resolved.stat.as_ref(), + *create, + if_match.as_ref(), + )?; + resolved + } + }; + if resolved.is_root() { + return Err(ProtocolError::failed_precondition( + PreconditionReason::FilesystemRoot, + "the filesystem root cannot be replaced", + )); + } + if let Err(error) = confinement.refuse_containing_state(&resolved, destination) { + confinement.remove_created(&resolved.created); + return Err(error); + } + let Resolved { + parent, + name, + segments, + created, + .. + } = resolved; + Ok(Self { + confinement, + kind, + destination: destination.clone(), + parent_segments: segments[..segments.len() - 1].to_vec(), + destination_segments: segments, + created_parents: created, + parent: Arc::new(parent), + name, + stage_name: OsString::from(format!("{STAGE_PREFIX}{:032x}", rand::random::())), + stage: None, + limits, + entries: Vec::new(), + pages: BTreeMap::new(), + paths: HashMap::new(), + sizes: BTreeMap::new(), + total_bytes: 0, + manifest_bytes: 0, + scan: None, + reusable: BTreeMap::new(), + uploads: BTreeMap::new(), + spool: Some(spool), + staging_index: 0, + copying: None, + reused_bytes: 0, + }) + } + + /// Create the staging directory next to the destination. Called once + /// its record is journaled, so a restart always finds and removes it. + pub fn create_stage(&mut self) -> Result<(), ProtocolError> { + let io = |error| io_error(error, &self.destination); + self.parent + .mkdir(&self.stage_name, os::PRIVATE_DIRECTORY_MODE) + .map_err(io)?; + let directory = self.parent.child(&self.stage_name).map_err(io)?; + self.stage = Some(Stage { + parent: self.parent.clone(), + name: self.stage_name.clone(), + directory, + }); + Ok(()) + } + + /// Remove everything `begin` created, at once: a begin refused before + /// the operation was recorded leaves nothing behind. + pub fn discard(mut self) { + if let Some(stage) = self.stage.take() { + remove_stage(stage); + } + self.confinement + .remove_created(&std::mem::take(&mut self.created_parents)); + drop(self.spool.take()); + } + + /// Where the staging directory is (or is about to be) created. + pub fn stage_record(&self) -> StageRecord { + StageRecord::new( + &self.parent_segments, + self.stage_name.to_str().expect("stage names are ASCII"), + ) + } + + /// [`Self::stage_record`] once the staging directory exists. + pub fn created_stage_record(&self) -> Option { + self.stage.as_ref().map(|_| self.stage_record()) + } + + /// The page at `index` was already appended with these entries. + pub fn is_repeat(&self, index: u32, entries: &[InventoryEntry], last: bool) -> bool { + self.pages + .get(&index) + .is_some_and(|(previous, sealed)| previous == entries && *sealed == last) + } + + pub fn spool_path(&self) -> Option { + self.spool.as_ref().map(|spool| spool.path().to_owned()) + } + + /// Validate and add one inventory page; `Ok(true)` when it sealed the + /// inventory. A repeat of an identical page changes nothing. + pub fn append( + &mut self, + index: u32, + entries: Vec, + last: bool, + ) -> Result { + if let Some((previous, sealed)) = self.pages.get(&index) { + if *previous == entries && *sealed == last { + return Ok(false); + } + return Err(ProtocolError::failed_precondition( + PreconditionReason::TransferOffset, + format!("page {index} was appended with different entries"), + )); + } + if index as usize != self.entries.len() { + return Err(ProtocolError::failed_precondition( + PreconditionReason::TransferOffset, + format!("the next inventory index is {}", self.entries.len()), + )); + } + if entries.is_empty() && !(last && !self.entries.is_empty()) { + return Err(ProtocolError::invalid_field( + "entries", + "an inventory page needs at least one entry", + )); + } + let single_file = matches!(self.kind, Kind::WriteFile { .. }); + let mut paths = HashMap::new(); + let mut sizes = BTreeMap::new(); + let mut total = self.total_bytes; + let mut manifest = self.manifest_bytes; + for (offset, entry) in entries.iter().enumerate() { + let position = self.entries.len() + offset; + let field = |member: &str| format!("entries[{offset}]{member}"); + if entry.path.depth() > self.limits.max_depth { + return Err(ProtocolError::invalid_field( + field(".path"), + "deeper than limits.maxDepth", + )); + } + if position == 0 { + if !entry.path.is_root() { + return Err(ProtocolError::invalid_field( + field(".path"), + "the inventory starts with the root entry `.`", + )); + } + } else { + if single_file { + return Err(ProtocolError::invalid_field( + "entries", + "a writeFile inventory holds exactly one file", + )); + } + if self.paths.contains_key(&entry.path) || paths.contains_key(&entry.path) { + return Err(ProtocolError::invalid_field( + field(".path"), + "duplicate path", + )); + } + let parent = parent_of(&entry.path); + let parent_is_directory = self + .paths + .get(&parent) + .or_else(|| paths.get(&parent)) + .copied() + .unwrap_or(false); + if !parent_is_directory { + return Err(ProtocolError::invalid_field( + field(".path"), + "a parent directory must precede its children", + )); + } + } + let is_directory = matches!(entry.content, TreeContent::Directory {}); + if single_file && is_directory { + return Err(ProtocolError::invalid_field( + field(".content"), + "a writeFile inventory holds one file", + )); + } + paths.insert(entry.path.clone(), is_directory); + if let TreeContent::File { + size_bytes, digest, .. + } = &entry.content + { + let Some(digest) = digest else { + return Err(ProtocolError::invalid_field( + field(".content.digest"), + "inventory files carry their digest", + )); + }; + if *size_bytes > self.limits.max_file_bytes { + return Err(ProtocolError::invalid_field( + field(".content.sizeBytes"), + "larger than limits.maxFileBytes", + )); + } + let known = self.sizes.get(digest).or_else(|| sizes.get(digest)); + if known.is_some_and(|known| known != size_bytes) { + return Err(ProtocolError::invalid_field( + field(".content.sizeBytes"), + "one digest with two sizes", + )); + } + sizes.insert(*digest, *size_bytes); + total = total.saturating_add(*size_bytes); + } + manifest += entry_bytes(entry); + } + check_entry_quota(self.entries.len() + entries.len(), &self.limits)?; + if total > self.limits.max_total_bytes { + return Err(ProtocolError::invalid_field( + "entries", + "the inventory exceeds limits.maxTotalBytes", + )); + } + if manifest > self.limits.max_manifest_bytes { + return Err(ProtocolError::invalid_field( + "entries", + "the inventory exceeds limits.maxManifestBytes", + )); + } + if single_file && !last { + return Err(ProtocolError::invalid_field( + "last", + "a writeFile inventory is one page", + )); + } + self.pages.insert(index, (entries.clone(), last)); + self.paths.extend(paths); + self.sizes.extend(sizes); + self.entries.extend(entries); + self.total_bytes = total; + self.manifest_bytes = manifest; + Ok(last) + } + + /// Start the destination scan after the inventory was sealed. + pub fn start_scan(&mut self) -> Result<(), ProtocolError> { + let current = self.parent.stat(&self.name); + let stat = match current { + Ok(stat) => Some(stat), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(io_error(error, &self.destination)), + }; + let mut scan = DestinationScan { + walker: None, + single: None, + hashing: None, + }; + match stat { + Some(stat) if stat.is_dir() && matches!(self.kind, Kind::Materialize { .. }) => { + let resolved = self + .confinement + .reopen(&self.destination_segments, &self.destination)?; + scan.walker = Some(Walker::new( + self.confinement.clone(), + &resolved, + CompiledFilter::everything(), + LinkMode::Report, + None, + &self.destination, + )?); + } + Some(stat) if stat.is_file() => { + scan.single = Some(Source { + canonical: self.destination_segments.clone(), + stat, + }); + } + _ => {} + } + self.scan = Some(scan); + Ok(()) + } + + /// One bounded step of the destination scan; `true` when it is done. + /// Reuse is an optimisation: unreadable destination entries are left + /// out, never fatal. + pub fn scan_step(&mut self) -> Result { + let wanted: HashSet = self + .missing_digests() + .iter() + .map(|digest| self.sizes[digest]) + .collect(); + let mut allowance = STEP_BYTES; + let Some(scan) = self.scan.as_mut() else { + return Ok(true); + }; + for _ in 0..STEP_ENTRIES { + if let Some((_, file, hash, read)) = scan.hashing.as_mut() { + let mut buffer = vec![0u8; 64 * 1024]; + let mut finished = false; + while allowance > 0 { + let limit = (buffer.len() as u64).min(allowance) as usize; + match file.read(&mut buffer[..limit]) { + Ok(0) => { + finished = true; + break; + } + Ok(count) => { + hash.update(&buffer[..count]); + *read += count as u64; + allowance -= count as u64; + } + Err(_) => { + scan.hashing = None; + break; + } + } + } + if !finished { + if scan.hashing.is_some() { + return Ok(false); + } + continue; + } + let (source, file, hash, read) = scan.hashing.take().expect("hashing above"); + let digest = ContentDigest::from_sha256(hash.finalize().into()); + let unchanged = EntryStat::of_file(&file) + .is_ok_and(|stat| read == source.stat.size && stat.unchanged(&source.stat)); + if unchanged && self.sizes.contains_key(&digest) { + self.reusable.entry(digest).or_insert(source); + } + continue; + } + let candidate = if let Some(single) = scan.single.take() { + Some(( + single.clone(), + self.confinement + .reopen(&single.canonical, &self.destination), + )) + } else if let Some(walker) = scan.walker.as_mut() { + match walker.next_event() { + Some(Event::Entry(entry)) + if entry.stat.is_file() && wanted.contains(&entry.stat.size) => + { + let source = Source { + canonical: entry.canonical.clone(), + stat: entry.stat, + }; + let file = entry.parent.open_read(&entry.name); + if let Ok(file) = file { + scan.hashing = Some((source, file, Sha256::new(), 0)); + } + continue; + } + Some(_) => continue, + None => { + scan.walker = None; + None + } + } + } else { + None + }; + match candidate { + Some((source, Ok(resolved))) if wanted.contains(&source.stat.size) => { + if let Ok(file) = resolved.parent.open_read(&resolved.name) { + scan.hashing = Some((source, file, Sha256::new(), 0)); + } + } + Some(_) => {} + None => { + self.scan = None; + return Ok(true); + } + } + } + Ok(false) + } + + /// Digests whose content is neither empty, reusable nor uploaded. + pub fn missing_digests(&self) -> Vec { + self.sizes + .iter() + .filter(|(digest, size)| { + **size > 0 + && !self.reusable.contains_key(*digest) + && !self.uploads.get(*digest).is_some_and(|upload| { + upload.lock().is_ok_and(|upload| upload.is_complete()) + }) + }) + .map(|(digest, _)| *digest) + .collect() + } + + /// The upload of `digest`, created on first use. `None` when the digest + /// is not in the inventory. + pub fn upload( + &mut self, + digest: &ContentDigest, + ) -> Result>>, ProtocolError> { + let Some(size) = self.sizes.get(digest).copied() else { + return Ok(None); + }; + if let Some(upload) = self.uploads.get(digest) { + return Ok(Some(upload.clone())); + } + let spool = self + .spool + .as_ref() + .ok_or_else(|| ProtocolError::internal("the upload spool is gone"))?; + let path = spool.path().join(digest.hex()); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .map_err(|error| io_error(error, "the upload spool"))?; + let upload = Arc::new(Mutex::new(Upload { + file, + path, + hash: Sha256::new(), + length: 0, + size, + complete: false, + })); + self.uploads.insert(*digest, upload.clone()); + Ok(Some(upload)) + } + + /// One bounded staging step; `true` when the tree is staged. + pub fn stage_step(&mut self) -> Result { + let mut allowance = STEP_BYTES; + for _ in 0..STEP_ENTRIES { + if self.copying.is_some() { + if !self.copy_step(&mut allowance)? { + return Ok(false); + } + continue; + } + let Some(entry) = self.entries.get(self.staging_index).cloned() else { + return Ok(true); + }; + let stage = self.stage.as_ref().expect("staging before commit"); + let (directory, name) = staged_parent(&stage.directory, &entry.path) + .map_err(|error| io_error(error, &entry.path))?; + match entry.content { + TreeContent::Directory {} => { + directory + .mkdir(&name, DEFAULT_DIRECTORY_MODE) + .map_err(|error| io_error(error, &entry.path))?; + directory + .sync() + .map_err(|error| io_error(error, &entry.path))?; + self.staging_index += 1; + } + TreeContent::File { + size_bytes, + executable, + digest, + } => { + let digest = digest.expect("appended files carry digests"); + let mode = if executable { + os::with_execute_bits(DEFAULT_FILE_MODE) + } else { + DEFAULT_FILE_MODE + }; + let destination = directory + .create_file(&name, os::PRIVATE_FILE_MODE) + .map_err(|error| io_error(error, &entry.path))?; + let (source, changed) = self.content_source(&digest, size_bytes)?; + self.copying = Some(Copying { + source, + destination, + hash: Sha256::new(), + expected: digest, + copied: 0, + size: size_bytes, + mode, + changed, + }); + } + } + } + Ok(false) + } + + /// The bytes of `digest`: an upload, a reusable destination file or + /// nothing for empty content (an empty spool file stands in). + fn content_source( + &mut self, + digest: &ContentDigest, + size: u64, + ) -> Result<(File, Option), ProtocolError> { + if let Some(upload) = self.uploads.get(digest) { + let upload = upload + .lock() + .map_err(|_| ProtocolError::internal("upload lock poisoned"))?; + if upload.complete { + let file = File::open(&upload.path) + .map_err(|error| io_error(error, "the upload spool"))?; + return Ok((file, None)); + } + } + if let Some(source) = self.reusable.get(digest).cloned() { + let resolved = self + .confinement + .reopen(&source.canonical, &self.destination) + .map_err(|_| source_changed(&self.destination))?; + let file = resolved + .parent + .open_read(&resolved.name) + .map_err(|error| io_error(error, &self.destination))?; + if !EntryStat::of_file(&file).is_ok_and(|stat| stat.unchanged(&source.stat)) { + return Err(source_changed(&self.destination)); + } + self.reused_bytes += size; + return Ok((file, Some(source))); + } + if size == 0 { + let spool = self + .spool + .as_ref() + .ok_or_else(|| ProtocolError::internal("the upload spool is gone"))?; + let path = spool.path().join("empty"); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(path) + .map_err(|error| io_error(error, "the upload spool"))?; + return Ok((file, None)); + } + Err(ProtocolError::internal(format!( + "content of {digest} is missing at staging" + ))) + } + + fn copy_step(&mut self, allowance: &mut u64) -> Result { + let copy = self.copying.as_mut().expect("caller checked"); + let mut buffer = vec![0u8; 64 * 1024]; + loop { + if *allowance == 0 { + return Ok(false); + } + let limit = (buffer.len() as u64).min(*allowance) as usize; + let read = copy + .source + .read(&mut buffer[..limit]) + .map_err(|error| io_error(error, &self.destination))?; + if read == 0 { + break; + } + copy.copied += read as u64; + if copy.copied > copy.size { + return Err(copy_mismatch(copy, &self.destination)); + } + copy.hash.update(&buffer[..read]); + copy.destination + .write_all(&buffer[..read]) + .map_err(|error| io_error(error, &self.destination))?; + *allowance -= read as u64; + } + let copy = self.copying.take().expect("copying above"); + let digest = ContentDigest::from_sha256(copy.hash.clone().finalize().into()); + if copy.copied != copy.size || digest != copy.expected { + return Err(copy_mismatch(©, &self.destination)); + } + os::set_mode(©.destination, copy.mode) + .and_then(|()| copy.destination.sync_all()) + .map_err(|error| io_error(error, &self.destination))?; + self.staging_index += 1; + Ok(true) + } + + /// Publish the staged tree with one rename. + pub fn commit(&mut self) -> Result<(), ProtocolError> { + let stage = self.stage.as_ref().expect("commit before release"); + let staged = OsStr::new(STAGED_TREE); + let io = |error| io_error(error, &self.destination); + match &self.kind { + Kind::Materialize { on_existing } => { + let published = + stage + .directory + .rename(staged, &self.parent, &self.name, RenameMode::NoReplace); + match published { + Ok(()) => {} + Err(error) + if error.kind() == std::io::ErrorKind::AlreadyExists + && *on_existing == TransferOnExisting::Replace => + { + // The previous tree lands in the staging directory, + // which is removed with the operation. + stage + .directory + .rename(staged, &self.parent, &self.name, RenameMode::Exchange) + .map_err(io)?; + } + Err(error) => return Err(io(error)), + } + } + Kind::WriteFile { + create, + if_match, + executable, + } => { + let current = match self.parent.stat(&self.name) { + Ok(stat) => Some(stat), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(io(error)), + }; + write_preconditions( + &self.destination, + current.as_ref(), + *create, + if_match.as_ref(), + )?; + let base = current.map_or(DEFAULT_FILE_MODE, |stat| stat.mode); + let mode = if *executable { + os::with_execute_bits(base) + } else { + base + }; + let file = stage.directory.open_read(staged).map_err(io)?; + os::set_mode(&file, mode) + .and_then(|()| file.sync_all()) + .map_err(io)?; + let rename = if *create == WriteCondition::IfAbsent { + RenameMode::NoReplace + } else { + RenameMode::Replace + }; + stage + .directory + .rename(staged, &self.parent, &self.name, rename) + .map_err(io)?; + } + } + self.parent.sync().map_err(io)?; + // The published tree lives in the directories begin created. + self.created_parents.clear(); + Ok(()) + } +} + +impl Drop for Publish { + fn drop(&mut self) { + let stage = self.stage.take(); + let created = std::mem::take(&mut self.created_parents); + if stage.is_some() || !created.is_empty() { + // A replaced tree may be large; removal never delays a receipt. + // The receipt keeps the stage's record until a sweep confirms + // it is gone, so an exit before this thread finishes leaks + // nothing. + let confinement = self.confinement.clone(); + let _ = std::thread::Builder::new() + .name("transfer-stage-cleanup".into()) + .spawn(move || { + if let Some(stage) = stage { + remove_stage(stage); + } + confinement.remove_created(&created); + }); + } + if let Some(spool) = self.spool.take() { + let _ = std::thread::Builder::new() + .name("transfer-spool-cleanup".into()) + .spawn(move || drop(spool)); + } + } +} + +fn remove_stage(stage: Stage) { + drop(stage.directory); + if let Err(error) = stage.parent.remove_tree(&stage.name) + && error.kind() != std::io::ErrorKind::NotFound + { + eprintln!( + "lightspeed-envd: removing transfer staging {}: {error}", + stage.name.to_string_lossy() + ); + } +} + +fn copy_mismatch(copy: &Copying, destination: &EnvironmentPath) -> ProtocolError { + match ©.changed { + Some(_) => source_changed(destination), + None => ProtocolError::failed_precondition( + PreconditionReason::DigestMismatch, + format!( + "staged content of {} does not match its digest", + copy.expected + ), + ), + } +} + +fn parent_of(path: &TreePath) -> TreePath { + match path.as_str().rsplit_once('/') { + Some((parent, _)) => TreePath::new(parent).expect("a parent of a tree path is one"), + None => TreePath::root(), + } +} + +/// The directory that holds `path` in the staged tree, and its name there. +fn staged_parent(stage: &Directory, path: &TreePath) -> std::io::Result<(Directory, OsString)> { + if path.is_root() { + return Ok((stage.try_clone()?, OsString::from(STAGED_TREE))); + } + let mut directory = stage.child(OsStr::new(STAGED_TREE))?; + let segments: Vec<&str> = path.segments().collect(); + let (name, parents) = segments + .split_last() + .expect("a non-root path has a segment"); + for segment in parents { + directory = directory.child(OsStr::new(segment))?; + } + Ok((directory, OsString::from(name))) +} + +/// Remove a staging directory recorded by an operation a restart +/// interrupted. +pub(super) fn discard_stage(confinement: &Confinement, record: &StageRecord) { + if !record.name.starts_with(STAGE_PREFIX) || record.name.contains('/') { + return; + } + let segments = record.parent_segments(); + let Ok(resolved) = confinement.reopen(&segments, &"transfer staging") else { + return; + }; + if let Ok(parent) = resolved.open_directory() { + let _ = parent.remove_tree(OsStr::new(&record.name)); + } +} diff --git a/crates/environment-daemon/src/filesystem/transfer/session.rs b/crates/environment-daemon/src/filesystem/transfer/session.rs deleted file mode 100644 index e09db6002..000000000 --- a/crates/environment-daemon/src/filesystem/transfer/session.rs +++ /dev/null @@ -1,1531 +0,0 @@ -//! Bounded transfer sessions retain inventories and publish through the filesystem backend. -use crate::filesystem::backend::{ - self, Result, conflict, digest, error, invalid, io, relative, valid_digest, valid_path, -}; -use environment_protocol::{ - data::{inventory::*, transfer::TransferOnExisting, transfer_session::*}, - error::EnvironmentProtocolErrorCode as Code, - shared::ByteChunk, -}; -use sha2::{Digest, Sha256}; -use std::{ - collections::BTreeMap, - fs::File, - io::{Read, Seek, SeekFrom, Write}, - path::Path, - time::{Duration, Instant}, -}; -#[path = "journal.rs"] -mod journal; -use backend::{Directory, Observation}; -#[derive(Clone)] -struct Source { - path: String, - observed: Observation, -} -struct Hashing { - source: Source, - file: File, - hash: Sha256, - read: u64, -} -struct Scanner { - anchor: Option, - pending: Vec, - pending_bytes: usize, - hashing: Option, - entries: Vec, - files: BTreeMap, - observed: Vec, - limits: InventoryLimits, - bytes: u64, - manifest_bytes: u64, - selected: String, - ignore_errors: bool, - patterns: Vec, - visited: u32, - metadata_only: bool, - skipped: bool, -} -impl Scanner { - fn new( - anchor: Directory, - selected: String, - limits: InventoryLimits, - ignore_errors: bool, - ) -> Self { - Self { - anchor: Some(anchor), - pending_bytes: selected.len(), - pending: vec![selected.clone()], - hashing: None, - entries: vec![], - files: BTreeMap::new(), - observed: vec![], - limits, - bytes: 0, - manifest_bytes: 0, - selected, - ignore_errors, - patterns: vec![], - visited: 0, - metadata_only: false, - skipped: false, - } - } - fn record(&mut self, source: Source, content: InventoryContent) -> Result<()> { - let relative = source - .path - .strip_prefix(&self.selected) - .unwrap() - .trim_start_matches('/') - .to_owned(); - if !valid_path(&relative) { - return Err(invalid("invalid inventory path")); - } - let entry = InventoryEntry { - path: relative, - content, - }; - self.manifest_bytes += serde_json::to_vec(&entry) - .map_err(|e| invalid(&e.to_string()))? - .len() as u64; - if self.manifest_bytes > self.limits.max_manifest_bytes { - return Err(invalid("manifest memory quota exceeded")); - } - self.entries.push(entry); - self.observed.push(source); - Ok(()) - } - /// At most 4 MiB of hashing and 128 visited nodes per exchange. Directory - /// enumeration is additionally bounded by the inventory entry/manifest quotas. - fn advance(&mut self) -> Result { - let mut allowance = 4 * 1024 * 1024; - for _ in 0..MAX_INVENTORY_PAGE { - if let Some(mut task) = self.hashing.take() { - let mut buffer = [0u8; 64 * 1024]; - while allowance > 0 { - let limit = buffer.len().min(allowance); - let read = task.file.read(&mut buffer[..limit]).map_err(io)?; - if read == 0 { - if task.read != task.source.observed.size() - || !task - .source - .observed - .matches(&backend::observe(&task.file).map_err(io)?) - { - return Err(conflict()); - } - let hash = digest(task.hash); - self.files - .entry(hash.clone()) - .or_insert_with(|| task.source.clone()); - let content = InventoryContent::File { - size_bytes: task.read, - executable: task.source.observed.executable(), - digest: hash, - }; - self.record(task.source, content)?; - break; - } - task.read += read as u64; - if task.read > task.source.observed.size() { - return Err(conflict()); - } - task.hash.update(&buffer[..read]); - allowance -= read; - if allowance == 0 { - self.hashing = Some(task); - return Ok(false); - } - } - continue; - } - let Some(path) = self.pending.pop() else { - return Ok(true); - }; - self.pending_bytes -= path.len(); - let anchor = self.anchor.as_ref().unwrap(); - let file = match if self.metadata_only { - anchor.metadata(&path) - } else { - anchor.open(&path) - } { - Ok(file) => file, - Err(_) if self.ignore_errors => { - self.skipped = true; - continue; - } - Err(e) => return Err(io(e)), - }; - let observed = backend::observe(&file).map_err(io)?; - self.visited += 1; - if self.visited > self.limits.max_entries { - return Err(invalid("entry quota exceeded")); - } - let rel = path - .strip_prefix(&self.selected) - .unwrap() - .trim_start_matches('/'); - if !valid_path(rel) - || (!rel.is_empty() && rel.split('/').count() > self.limits.max_depth as usize) - { - return Err(invalid("path/depth quota exceeded")); - } - let source = Source { - path: path.clone(), - observed, - }; - if source.observed.is_dir() { - let names = - Directory::names(&file, self.limits.max_entries as usize).map_err(io)?; - if self.pending.len() + self.entries.len() + names.len() - > self.limits.max_entries as usize - { - return Err(invalid("entry quota exceeded")); - } - // Bound the pending traversal too, before any names enter the queue. - let pending_bytes = self.pending_bytes; - let added_bytes: usize = names - .iter() - .map(|n| path.len() + usize::from(!path.is_empty()) + n.len()) - .sum(); - if (pending_bytes + added_bytes) as u64 > self.limits.max_manifest_bytes { - return Err(invalid("traversal memory quota exceeded")); - } - self.pending_bytes += added_bytes; - for name in names.into_iter().rev() { - if !valid_path(&name) { - return Err(invalid("unsupported filename")); - } - self.pending.push(if path.is_empty() { - name - } else { - format!("{path}/{name}") - }); - } - self.record(source, InventoryContent::Directory)?; - } else { - if !self.patterns.is_empty() && !self.patterns.iter().any(|p| p.matches(rel)) { - continue; - } - self.bytes = self - .bytes - .checked_add(source.observed.size()) - .ok_or_else(|| invalid("size overflow"))?; - if source.observed.size() > self.limits.max_file_bytes - || self.bytes > self.limits.max_total_bytes - { - return Err(invalid("content quota exceeded")); - } - if self.metadata_only { - let content = InventoryContent::File { - size_bytes: source.observed.size(), - executable: source.observed.executable(), - digest: String::new(), - }; - self.record(source, content)?; - continue; - } - self.hashing = Some(Hashing { - source, - file, - hash: Sha256::new(), - read: 0, - }); - } - } - Ok(false) - } - fn verify(&self) -> Result<()> { - for source in &self.observed { - let file = self - .anchor - .as_ref() - .unwrap() - .open(&source.path) - .map_err(io)?; - if !source - .observed - .matches(&backend::observe(&file).map_err(io)?) - { - return Err(conflict()); - } - } - Ok(()) - } -} -struct Stage { - parent: Directory, - directory: Directory, - name: String, - target: String, -} -impl Drop for Stage { - fn drop(&mut self) { - if let Ok(parent) = self.parent.clone_dir() { - let name = self.name.clone(); - // Retired trees may be large. Cleanup cannot delay the publication receipt. - let _ = std::thread::Builder::new() - .name("transfer-cleanup".into()) - .spawn(move || { - if let Err(error) = parent.remove_tree(&name) - && error.kind() != std::io::ErrorKind::NotFound - { - eprintln!("lightspeed-envd retirement cleanup {name}: {error}"); - } - }); - } - } -} -struct Upload { - path: std::path::PathBuf, - hash: Sha256, - length: u64, - complete: bool, -} -struct Copying { - source: File, - destination: File, - hash: Sha256, - expected: String, - bytes: u64, - size: u64, - executable: bool, -} -struct Operation { - request: TransferRequest, - status: TransferStatus, - started: Instant, - limits: InventoryLimits, - scanner: Scanner, - stage: Option, - entries: Vec, - manifest_bytes: u64, - paths: BTreeMap, - sizes: BTreeMap, - pages: BTreeMap, bool)>, - uploads: BTreeMap, - spool: Option, - retained_inventory: Option, - staging_index: usize, - copying: Option, - unchanged: bool, - staging_name: Option, -} -#[derive(Default)] -pub struct TransferManager { - operations: BTreeMap, - journal: Option, -} -impl TransferManager { - pub fn with_journal(directory: std::path::PathBuf) -> Result { - backend::create_private_directory(&directory).map_err(io)?; - Ok(Self { - operations: BTreeMap::new(), - journal: Some(directory), - }) - } - pub fn expire(&mut self) -> Option { - self.operations.retain(|_, op| { - op.started.elapsed() < Duration::from_millis(op.limits.max_duration_ms) - }); - self.journal.clone() - } - pub fn cleanup_expired(directory: &Path) { - if let Ok(entries) = std::fs::read_dir(directory) { - for entry in entries.flatten().take(8192) { - if entry.path().extension().is_some_and(|s| s == "json") { - let id = entry - .file_name() - .to_string_lossy() - .trim_end_matches(".json") - .to_owned(); - if let Ok(Some(record)) = journal::read(directory, &id) - && record.expires_at_ms < journal::now_ms() - { - if let ( - Some(name), - TransferRequest::Begin { - selection: - TransferSelection::Materialize { destination, .. } - | TransferSelection::WriteFile { destination }, - .. - }, - ) = (&record.stage_name, &record.request) - && name.starts_with(".env-transfer-") - && valid_path(name) - && !name.contains('/') - && let Ok(anchor) = Directory::anchor(&record.root) - && let Ok(relative) = relative(&record.root, destination) - && let Ok((parent, _)) = anchor.parent(&relative) - && let Err(error) = parent.remove_tree(name) - && error.kind() != std::io::ErrorKind::NotFound - { - continue; - } - if let Some(spool) = &record.spool_directory - && spool.parent() == Some(directory) - && spool.file_name().is_some_and(|name| { - name.to_string_lossy() - .starts_with(&format!("{id}.content-")) - }) - { - let _ = std::fs::remove_dir_all(spool); - } - let _ = std::fs::remove_file(directory.join(format!("{id}.inventory"))); - let _ = std::fs::remove_file(entry.path()); - } - } - } - } - } - fn save(&self, root: &Path, id: &str) -> Result<()> { - let Some(directory) = &self.journal else { - return Ok(()); - }; - let op = &self.operations[id]; - if let Some(path) = &op.retained_inventory { - let mut copy = tempfile::NamedTempFile::new_in(directory).map_err(io)?; - std::io::copy(&mut File::open(path).map_err(io)?, &mut copy).map_err(io)?; - copy.as_file().sync_all().map_err(io)?; - copy.persist(directory.join(format!("{id}.inventory"))) - .map_err(|e| io(e.error))?; - } - journal::write( - directory, - id, - &journal::Record { - root: root.to_path_buf(), - request: op.request.clone(), - status: op.status.clone(), - expires_at_ms: journal::now_ms() - + op.limits - .max_duration_ms - .saturating_sub(op.started.elapsed().as_millis() as u64), - stage_name: op.staging_name.clone(), - spool_directory: op.spool.as_ref().map(|s| s.path().to_path_buf()), - }, - ) - } - pub fn execute(&mut self, root: &Path, request: TransferRequest) -> Result { - let id = request.operation_id().to_owned(); - if id.is_empty() - || id.len() > 128 - || !id - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b)) - { - return Err(invalid("invalid operation id")); - } - if !self.operations.contains_key(&id) - && let Some(directory) = &self.journal - && let Some(record) = journal::read(directory, &id)? - { - if record.root != root { - return Err(error( - Code::Forbidden, - "receipt belongs to a different filesystem scope", - )); - } - if record.expires_at_ms < journal::now_ms() { - return Err(error(Code::Timeout, "transfer receipt expired")); - } - if matches!(&request, TransferRequest::Begin { .. }) && request != record.request { - return Err(error( - Code::Conflict, - "operation id already used for different input", - )); - } - if matches!( - request, - TransferRequest::Begin { .. } - | TransferRequest::Status { .. } - | TransferRequest::Commit { .. } - | TransferRequest::Abort { .. } - ) && record.status.phase == TransferPhase::Complete - { - return Ok(TransferResponse::Status(record.status)); - } - if let TransferRequest::Inventory { offset, .. } = &request - && record.status.phase == TransferPhase::Complete - { - return retained_page( - &directory.join(format!("{id}.inventory")), - record.status.entries, - *offset, - ); - } - if matches!(request, TransferRequest::Abort { .. }) { - if let ( - Some(name), - TransferRequest::Begin { - selection: - TransferSelection::Materialize { destination, .. } - | TransferSelection::WriteFile { destination }, - .. - }, - ) = (&record.stage_name, &record.request) - && name.starts_with(".env-transfer-") - && valid_path(name) - && !name.contains('/') - { - let anchor = Directory::anchor(root).map_err(io)?; - let (parent, _) = anchor.parent(&relative(root, destination)?).map_err(io)?; - let _ = parent.remove_tree(name); - } - let mut record = record; - record.status.phase = TransferPhase::Aborted; - record.stage_name = None; - if let Some(spool) = record.spool_directory.take() - && spool.parent() == Some(directory.as_path()) - && spool.file_name().is_some_and(|name| { - name.to_string_lossy() - .starts_with(&format!("{id}.content-")) - }) - { - let _ = std::fs::remove_dir_all(spool); - } - journal::write(directory, &id, &record)?; - return Ok(TransferResponse::Status(record.status)); - } - return Err(error( - Code::Conflict, - "daemon restarted during transfer; inspect the destination and abort the old operation before starting a new one", - )); - } - if let Some(op) = self.operations.get(&id) - && op.started.elapsed() >= Duration::from_millis(op.limits.max_duration_ms) - { - return Err(error(Code::Timeout, "transfer expired")); - } - if let TransferRequest::Begin { - selection, limits, .. - } = &request - { - if let Some(op) = self.operations.get(&id) { - return if op.request == request { - Ok(TransferResponse::Status(op.status.clone())) - } else { - Err(error( - Code::Conflict, - "operation id already used for different input", - )) - }; - } - self.operations.retain(|_, op| { - op.started.elapsed() < Duration::from_millis(op.limits.max_duration_ms) - }); - if self.operations.len() >= 4096 - || self - .operations - .values() - .filter(|op| { - !matches!( - op.status.phase, - TransferPhase::Complete | TransferPhase::Aborted - ) - }) - .count() - >= 8 - { - return Err(error(Code::Conflict, "transfer capacity reached")); - } - let ceiling = InventoryLimits::default(); - if limits.max_entries == 0 - || limits.max_entries > ceiling.max_entries - || limits.max_depth > ceiling.max_depth - || limits.max_file_bytes > ceiling.max_file_bytes - || limits.max_total_bytes > ceiling.max_total_bytes - || limits.max_manifest_bytes > ceiling.max_manifest_bytes - || limits.max_duration_ms == 0 - || limits.max_duration_ms > ceiling.max_duration_ms - { - return Err(invalid("operation quotas exceed daemon ceilings")); - } - let anchor = Directory::anchor(root).map_err(io)?; - let (selected, stage, ignore_errors) = match selection { - TransferSelection::Capture { source } => (relative(root, source)?, None, false), - TransferSelection::Materialize { destination, .. } - | TransferSelection::WriteFile { destination } => { - let path = relative(root, destination)?; - if path.is_empty() { - return Err(invalid("cannot replace the filesystem root")); - } - for operation in self.operations.values().filter(|op| op.stage.is_some()) { - let other = &operation.scanner.selected; - if path == *other - || path - .strip_prefix(other) - .is_some_and(|tail| tail.starts_with('/')) - || other - .strip_prefix(&path) - .is_some_and(|tail| tail.starts_with('/')) - { - return Err(error( - Code::Conflict, - "destination overlaps an active materialization", - )); - } - } - let (parent, target) = anchor.ensure_parent(&path).map_err(io)?; - if parent.target_exists(&target).map_err(io)? { - if matches!( - selection, - TransferSelection::Materialize { - on_existing: TransferOnExisting::Error, - .. - } - ) { - return Err(error(Code::Conflict, "destination exists")); - } - if matches!(selection, TransferSelection::WriteFile { .. }) { - parent.writable_file(&target).map_err(io)?; - } - } - let name = format!(".env-transfer-{:032x}", rand::random::()); - let directory = parent.mkdir(&name, true).map_err(io)?; - ( - path, - Some(Stage { - parent, - directory, - name, - target, - }), - true, - ) - } - }; - let op = Operation { - request: request.clone(), - status: TransferStatus { - operation_id: id.clone(), - phase: TransferPhase::Scanning, - entries: 0, - bytes: 0, - transferred_bytes: 0, - reused_bytes: 0, - }, - started: Instant::now(), - limits: *limits, - scanner: Scanner::new(anchor, selected, *limits, ignore_errors), - staging_name: stage.as_ref().map(|stage| stage.name.clone()), - stage, - entries: vec![], - manifest_bytes: 0, - paths: BTreeMap::new(), - sizes: BTreeMap::new(), - pages: BTreeMap::new(), - uploads: BTreeMap::new(), - spool: Some(if let Some(directory) = &self.journal { - tempfile::Builder::new() - .prefix(&format!("{id}.content-")) - .tempdir_in(directory) - .map_err(io)? - } else { - tempfile::tempdir().map_err(io)? - }), - retained_inventory: None, - staging_index: 0, - copying: None, - unchanged: false, - }; - let response = TransferResponse::Status(op.status.clone()); - self.operations.insert(id.clone(), op); - if let Err(error) = self.save(root, &id) { - self.operations.remove(&id); - return Err(error); - } - return Ok(response); - } - let op = self.operations.get_mut(&id).ok_or_else(|| { - error( - Code::NotFound, - "unknown transfer; operation receipts may have expired", - ) - })?; - if matches!(request, TransferRequest::Status { .. }) { - return Ok(TransferResponse::Status(op.status.clone())); - } - if matches!(request, TransferRequest::Abort { .. }) { - if op.status.phase != TransferPhase::Complete { - op.status.phase = TransferPhase::Aborted; - op.release(); - } - let response = TransferResponse::Status(op.status.clone()); - self.save(root, &id)?; - return Ok(response); - } - if op.started.elapsed() >= Duration::from_millis(op.limits.max_duration_ms) { - return Err(error(Code::Timeout, "transfer expired")); - } - if op.status.phase == TransferPhase::Aborted { - return Err(error(Code::Conflict, "transfer aborted")); - } - let terminal = matches!(request, TransferRequest::Commit { .. }); - let response = op.execute(request)?; - if terminal { - self.save(root, &id)?; - } - Ok(response) - } -} -impl Drop for Operation { - fn drop(&mut self) { - self.release(); - } -} -impl Operation { - fn retain_inventory(&mut self) -> Result<()> { - let mut spool = tempfile::NamedTempFile::new().map_err(io)?; - let count = self.entries.len(); - spool - .seek(SeekFrom::Start((8 * (count + 1)) as u64)) - .map_err(io)?; - let mut positions = Vec::with_capacity(count + 1); - for entry in &self.entries { - positions.push(spool.stream_position().map_err(io)?); - serde_json::to_writer(&mut spool, entry).map_err(|e| invalid(&e.to_string()))?; - } - positions.push(spool.stream_position().map_err(io)?); - spool.seek(SeekFrom::Start(0)).map_err(io)?; - for position in positions { - spool.write_all(&position.to_le_bytes()).map_err(io)?; - } - self.retained_inventory = Some(spool.into_temp_path()); - Ok(()) - } - fn retained_page(&self, offset: u32) -> Result { - let path = self.retained_inventory.as_ref().ok_or_else(|| { - error( - Code::Conflict, - "completed materialization retains only its receipt", - ) - })?; - retained_page(path, self.status.entries, offset) - } - fn release(&mut self) { - self.stage.take(); - self.uploads.clear(); - self.copying.take(); - if let Some(spool) = self.spool.take() { - let _ = std::thread::Builder::new() - .name("transfer-spool-cleanup".into()) - .spawn(move || drop(spool)); - } - self.scanner.anchor.take(); - self.scanner.pending = Vec::new(); - self.scanner.hashing.take(); - self.scanner.files.clear(); - self.scanner.observed = Vec::new(); - self.scanner.entries = Vec::new(); - self.paths.clear(); - self.sizes.clear(); - self.pages.clear(); - self.entries = Vec::new(); - } - fn status(&self) -> TransferResponse { - TransferResponse::Status(self.status.clone()) - } - fn execute(&mut self, request: TransferRequest) -> Result { - match request { - TransferRequest::Advance { .. } => { - if self.status.phase == TransferPhase::Scanning - && match self.scanner.advance() { - Ok(done) => done, - Err(_) if self.stage.is_some() => { - self.scanner.skipped = true; - self.scanner.pending.clear(); - self.scanner.hashing.take(); - true - } - Err(error) => return Err(error), - } - { - if self.stage.is_some() { - self.status.phase = TransferPhase::Inventory; - } else { - self.scanner.verify()?; - self.entries = self.scanner.entries.clone(); - self.status.entries = self.entries.len() as u32; - self.status.bytes = self.scanner.bytes; - self.status.phase = TransferPhase::Ready; - } - } - if self.status.phase == TransferPhase::Content && self.missing().is_empty() { - if !self.scanner.skipped && self.entries == self.scanner.entries { - self.scanner.verify()?; - self.unchanged = true; - self.status.reused_bytes = self.status.bytes; - self.status.phase = TransferPhase::Ready; - } else { - self.status.phase = TransferPhase::Staging; - } - } - if self.status.phase == TransferPhase::Staging { - self.advance_staging()?; - } - Ok(self.status()) - } - TransferRequest::Inventory { offset, .. } => { - if self.status.phase == TransferPhase::Complete { - return self.retained_page(offset); - } - if matches!( - self.status.phase, - TransferPhase::Scanning | TransferPhase::Inventory - ) { - return Err(error(Code::Conflict, "inventory not complete")); - } - let offset = offset as usize; - if offset > self.entries.len() { - return Err(invalid("inventory offset")); - } - let end = (offset + MAX_INVENTORY_PAGE).min(self.entries.len()); - Ok(TransferResponse::Inventory { - entries: self.entries[offset..end].to_vec(), - next_offset: (end < self.entries.len()).then_some(end as u32), - }) - } - TransferRequest::Append { - offset, - entries, - last, - .. - } => { - if self.stage.is_none() || entries.len() > MAX_INVENTORY_PAGE { - return Err(invalid("invalid inventory page")); - } - if let Some((previous, sealed)) = self.pages.get(&offset) { - if previous != &entries || *sealed != last { - return Err(error(Code::Conflict, "inventory retry differs")); - } - return Ok(self.status()); - } - let offset = offset as usize; - if self.status.phase != TransferPhase::Inventory || offset != self.entries.len() { - return Err(error(Code::Conflict, "inventory offset/phase mismatch")); - } - if entries.is_empty() { - return Err(invalid("empty inventory page")); - } - if self.is_file_write() - && (offset != 0 - || !last - || !matches!(entries.as_slice(), [InventoryEntry { path, content: InventoryContent::File { .. } }] if path.is_empty())) - { - return Err(invalid("file write requires exactly one file")); - } - // Validate the complete page before mutating the operation. - let mut total = self.status.bytes; - let mut meta = self.manifest_bytes; - let mut paths = BTreeMap::new(); - let mut sizes = BTreeMap::new(); - for (index, entry) in entries.iter().enumerate() { - if !valid_path(&entry.path) - || (!entry.path.is_empty() - && entry.path.split('/').count() > self.limits.max_depth as usize) - { - return Err(invalid("invalid inventory path")); - } - if offset + index == 0 { - if !entry.path.is_empty() { - return Err(invalid("inventory must start with root")); - } - } else { - let parent = entry.path.rsplit_once('/').map_or("", |(p, _)| p); - if self.paths.contains_key(&entry.path) || paths.contains_key(&entry.path) { - return Err(invalid("duplicate inventory path")); - } - if !self - .paths - .get(parent) - .or_else(|| paths.get(parent)) - .copied() - .unwrap_or(false) - { - return Err(invalid("parent must precede child")); - } - } - paths.insert( - entry.path.clone(), - entry.content == InventoryContent::Directory, - ); - if let InventoryContent::File { - size_bytes, digest, .. - } = &entry.content - { - if !valid_digest(digest) || *size_bytes > self.limits.max_file_bytes { - return Err(invalid("invalid file identity/size")); - } - if self - .sizes - .get(digest) - .or_else(|| sizes.get(digest)) - .is_some_and(|s| s != size_bytes) - || self - .scanner - .files - .get(digest) - .is_some_and(|f| f.observed.size() != *size_bytes) - { - return Err(invalid("same digest has different sizes")); - } - sizes.insert(digest.clone(), *size_bytes); - total = total - .checked_add(*size_bytes) - .ok_or_else(|| invalid("size overflow"))?; - } - meta += serde_json::to_vec(entry) - .map_err(|e| invalid(&e.to_string()))? - .len() as u64; - } - if self.entries.len() + entries.len() > self.limits.max_entries as usize - || total > self.limits.max_total_bytes - || meta > self.limits.max_manifest_bytes - { - return Err(invalid("inventory quota exceeded")); - } - self.pages.insert(offset as u32, (entries.clone(), last)); - self.paths.extend(paths); - self.sizes.extend(sizes); - self.entries.extend(entries); - self.status.entries = self.entries.len() as u32; - self.status.bytes = total; - self.manifest_bytes = meta; - if last { - if self.entries.is_empty() { - return Err(invalid("empty inventory")); - } - self.status.phase = TransferPhase::Content; - } - Ok(self.status()) - } - TransferRequest::Missing { offset, .. } => { - if !matches!( - self.status.phase, - TransferPhase::Content - | TransferPhase::Staging - | TransferPhase::Ready - | TransferPhase::Complete - ) { - return Err(error(Code::Conflict, "inventory not sealed")); - } - let missing = self.missing(); - let offset = offset as usize; - if offset > missing.len() { - return Err(invalid("missing offset")); - } - let end = (offset + MAX_INVENTORY_PAGE).min(missing.len()); - Ok(TransferResponse::Missing { - digests: missing[offset..end].to_vec(), - next_offset: (end < missing.len()).then_some(end as u32), - }) - } - TransferRequest::Read { digest, offset, .. } => { - if self.stage.is_some() || self.status.phase != TransferPhase::Ready { - return Err(error(Code::Conflict, "capture is not ready")); - } - let source = self - .scanner - .files - .get(&digest) - .ok_or_else(|| invalid("digest outside capture"))?; - let mut file = self - .scanner - .anchor - .as_ref() - .unwrap() - .open(&source.path) - .map_err(io)?; - if !source - .observed - .matches(&backend::observe(&file).map_err(io)?) - || offset > source.observed.size() - { - return Err(conflict()); - } - file.seek(SeekFrom::Start(offset)).map_err(io)?; - let mut data = vec![ - 0; - (source.observed.size() - offset).min(MAX_CONTENT_CHUNK as u64) - as usize - ]; - file.read_exact(&mut data).map_err(io)?; - if !source - .observed - .matches(&backend::observe(&file).map_err(io)?) - { - return Err(conflict()); - } - self.status.transferred_bytes += data.len() as u64; - Ok(TransferResponse::Chunk { - eof: offset + data.len() as u64 == source.observed.size(), - data: ByteChunk::from(data), - }) - } - TransferRequest::Write { - digest: expected, - offset, - data, - .. - } => { - if self.status.phase != TransferPhase::Content - || data.as_slice().len() > MAX_CONTENT_CHUNK - { - return Err(invalid("invalid upload chunk/phase")); - } - let size = self - .file_size(&expected) - .ok_or_else(|| invalid("digest outside inventory"))?; - if !self.uploads.contains_key(&expected) { - if offset != 0 { - return Err(error(Code::Conflict, "upload starts at offset zero")); - } - let path = self.spool.as_ref().unwrap().path().join(&expected[7..]); - let file = std::fs::OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .open(&path) - .map_err(io)?; - drop(file); - self.uploads.insert( - expected.clone(), - Upload { - path, - hash: Sha256::new(), - length: 0, - complete: false, - }, - ); - } - let upload = self.uploads.get_mut(&expected).unwrap(); - let mut file = std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(&upload.path) - .map_err(io)?; - let bytes = data.as_slice(); - let end = offset - .checked_add(bytes.len() as u64) - .ok_or_else(|| invalid("chunk offset overflow"))?; - if offset < upload.length || upload.complete { - if end > upload.length { - return Err(error(Code::Conflict, "overlapping chunk")); - } - file.seek(SeekFrom::Start(offset)).map_err(io)?; - let mut existing = vec![0; bytes.len()]; - file.read_exact(&mut existing).map_err(io)?; - if existing != bytes { - return Err(error(Code::Conflict, "chunk retry differs")); - } - return Ok(TransferResponse::Written { - next_offset: upload.length, - }); - } - if offset != upload.length || end > size || (bytes.is_empty() && size != 0) { - return Err(invalid("chunk offset/size mismatch")); - } - file.seek(SeekFrom::Start(offset)).map_err(io)?; - file.write_all(bytes).map_err(io)?; - upload.hash.update(bytes); - upload.length += bytes.len() as u64; - self.status.transferred_bytes += bytes.len() as u64; - if upload.length == size { - if digest(upload.hash.clone()) != expected { - self.uploads.remove(&expected); - let _ = std::fs::remove_file( - self.spool.as_ref().unwrap().path().join(&expected[7..]), - ); - return Err(error(Code::Conflict, "file digest mismatch")); - } - file.sync_all().map_err(io)?; - upload.complete = true; - } - Ok(TransferResponse::Written { next_offset: end }) - } - TransferRequest::Commit { .. } => { - if self.status.phase == TransferPhase::Complete { - return Ok(self.status()); - } - if self.status.phase != TransferPhase::Ready { - return Err(error(Code::Conflict, "transfer is not ready to commit")); - } - if self.stage.is_none() { - self.retain_inventory()?; - } - if self.unchanged { - self.scanner.verify()?; - } else if let Some(stage) = &self.stage { - let replace = matches!( - &self.request, - TransferRequest::Begin { - selection: TransferSelection::Materialize { - on_existing: TransferOnExisting::Replace, - .. - }, - .. - } - ); - stage.parent.target_exists(&stage.target).map_err(io)?; - if self.is_file_write() { - match stage.parent.writable_file(&stage.target) { - Ok(target) => { - let permissions = target.metadata().map_err(io)?.permissions(); - let staged = stage.directory.open("tree").map_err(io)?; - staged.set_permissions(permissions).map_err(io)?; - staged.sync_all().map_err(io)?; - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), - Err(error) => return Err(io(error)), - } - } - if self.is_file_write() { - stage - .parent - .publish_file(&stage.directory, &stage.target) - .map_err(io)?; - } else { - stage - .parent - .publish(&stage.directory, &stage.target, replace) - .map_err(io)?; - } - // Rename has succeeded. A later durability error must never cause - // a second swap on retry. - self.status.phase = TransferPhase::Complete; - stage.parent.sync().map_err(io)?; - stage.directory.sync().map_err(io)?; - } else { - self.scanner.verify()?; - } - self.status.phase = TransferPhase::Complete; - self.release(); - Ok(self.status()) - } - _ => Err(invalid("unexpected transfer action")), - } - } - fn file_size(&self, digest: &str) -> Option { - self.sizes.get(digest).copied() - } - - fn is_file_write(&self) -> bool { - matches!( - &self.request, - TransferRequest::Begin { - selection: TransferSelection::WriteFile { .. }, - .. - } - ) - } - fn missing(&self) -> Vec { - let mut missing = std::collections::BTreeSet::new(); - for digest in self.sizes.keys() { - if !self.scanner.files.contains_key(digest) - && !self.uploads.get(digest).is_some_and(|u| u.complete) - { - missing.insert(digest.clone()); - } - } - missing.into_iter().collect() - } - fn advance_staging(&mut self) -> Result<()> { - let mut allowance = 4 * 1024 * 1024; - for _ in 0..MAX_INVENTORY_PAGE { - if let Some(mut copy) = self.copying.take() { - let mut buffer = [0u8; 64 * 1024]; - while allowance > 0 { - let limit = buffer.len().min(allowance); - let read = copy.source.read(&mut buffer[..limit]).map_err(io)?; - if read == 0 { - if copy.bytes != copy.size || digest(copy.hash) != copy.expected { - return Err(conflict()); - } - backend::set_executable(©.destination, copy.executable).map_err(io)?; - copy.destination.sync_all().map_err(io)?; - let entry = &self.entries[self.staging_index]; - let path = if entry.path.is_empty() { - "tree".into() - } else { - format!("tree/{}", entry.path) - }; - self.stage - .as_ref() - .unwrap() - .directory - .parent(&path) - .map_err(io)? - .0 - .sync() - .map_err(io)?; - self.staging_index += 1; - break; - } - copy.bytes += read as u64; - if copy.bytes > copy.size { - return Err(conflict()); - } - copy.hash.update(&buffer[..read]); - copy.destination.write_all(&buffer[..read]).map_err(io)?; - allowance -= read; - if allowance == 0 { - self.copying = Some(copy); - return Ok(()); - } - } - continue; - } - let Some(entry) = self.entries.get(self.staging_index) else { - self.status.phase = TransferPhase::Ready; - return Ok(()); - }; - let stage = &self.stage.as_ref().unwrap().directory; - let path = if entry.path.is_empty() { - "tree".into() - } else { - format!("tree/{}", entry.path) - }; - match &entry.content { - InventoryContent::Directory => { - let (parent, name) = stage.parent(&path).map_err(io)?; - parent.mkdir(&name, false).map_err(io)?.sync().map_err(io)?; - parent.sync().map_err(io)?; - self.staging_index += 1; - } - InventoryContent::File { - digest, - size_bytes, - executable, - } => { - let source = - if let Some(upload) = self.uploads.get(digest).filter(|u| u.complete) { - let mut f = File::open(&upload.path).map_err(io)?; - f.seek(SeekFrom::Start(0)).map_err(io)?; - f - } else { - let source = self - .scanner - .files - .get(digest) - .ok_or_else(|| invalid("missing content"))?; - let file = self - .scanner - .anchor - .as_ref() - .unwrap() - .open(&source.path) - .map_err(io)?; - if !source - .observed - .matches(&backend::observe(&file).map_err(io)?) - { - return Err(conflict()); - } - self.status.reused_bytes += *size_bytes; - file - }; - self.copying = Some(Copying { - source, - destination: stage.create(&path).map_err(io)?, - hash: Sha256::new(), - expected: digest.clone(), - bytes: 0, - size: *size_bytes, - executable: *executable, - }); - } - } - } - Ok(()) - } -} - -fn retained_page(path: &Path, count: u32, offset: u32) -> Result { - if offset > count { - return Err(invalid("inventory offset")); - } - let end = (offset + MAX_INVENTORY_PAGE as u32).min(count); - let mut file = File::open(path).map_err(io)?; - file.seek(SeekFrom::Start(offset as u64 * 8)).map_err(io)?; - let mut positions = Vec::new(); - for _ in offset..=end { - let mut bytes = [0; 8]; - file.read_exact(&mut bytes).map_err(io)?; - positions.push(u64::from_le_bytes(bytes)); - } - let mut entries = Vec::new(); - for pair in positions.windows(2) { - let size = pair[1] - pair[0]; - if size > MAX_INVENTORY_PATH_BYTES as u64 + 1024 { - return Err(invalid("corrupt retained inventory")); - } - file.seek(SeekFrom::Start(pair[0])).map_err(io)?; - let mut bytes = vec![0; size as usize]; - file.read_exact(&mut bytes).map_err(io)?; - entries.push(serde_json::from_slice(&bytes).map_err(|e| invalid(&e.to_string()))?); - } - Ok(TransferResponse::Inventory { - entries, - next_offset: (end < count).then_some(end), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use environment_protocol::shared::EnvironmentPath; - - #[test] - fn expired_transfer_rejects_progress_and_publication() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - std::fs::write(root.join("destination"), b"keep").unwrap(); - let mut manager = TransferManager::default(); - manager - .execute( - &root, - TransferRequest::Begin { - operation_id: "expired".into(), - selection: TransferSelection::Materialize { - destination: EnvironmentPath::new( - root.join("destination").to_str().unwrap(), - ) - .unwrap(), - on_existing: TransferOnExisting::Replace, - }, - limits: InventoryLimits { - max_duration_ms: 1000, - ..Default::default() - }, - }, - ) - .unwrap(); - // Exercise expiry without waiting for wall-clock time or depending on - // filesystem throughput and test-runner scheduling. - manager.operations.get_mut("expired").unwrap().started = - Instant::now() - Duration::from_secs(2); - for request in [ - TransferRequest::Advance { - operation_id: "expired".into(), - }, - TransferRequest::Commit { - operation_id: "expired".into(), - }, - ] { - assert_eq!( - manager.execute(&root, request).unwrap_err().code, - Code::Timeout - ); - } - assert_eq!(std::fs::read(root.join("destination")).unwrap(), b"keep"); - } - - #[test] - fn scan_respects_remaining_budget_when_a_large_file_follows_a_small_file() { - let root = tempfile::tempdir().unwrap(); - std::fs::create_dir(root.path().join("source")).unwrap(); - std::fs::write(root.path().join("source/small"), [1; 17]).unwrap(); - std::fs::write( - root.path().join("source/large"), - vec![2; 4 * 1024 * 1024 + 13], - ) - .unwrap(); - let mut scanner = Scanner::new( - Directory::anchor(root.path()).unwrap(), - "source".into(), - InventoryLimits::default(), - false, - ); - // Make traversal order explicit: pending nodes are popped from the end. - scanner.pending = vec!["source/large".into(), "source/small".into()]; - scanner.pending_bytes = scanner.pending.iter().map(String::len).sum(); - let progress = |scanner: &Scanner| { - scanner - .entries - .iter() - .map(|entry| match entry.content { - InventoryContent::File { size_bytes, .. } => size_bytes, - InventoryContent::Directory => 0, - }) - .sum::() - + scanner.hashing.as_ref().map_or(0, |task| task.read) - }; - assert!(!scanner.advance().unwrap()); - assert_eq!(progress(&scanner), 4 * 1024 * 1024); - assert!(scanner.advance().unwrap()); - assert_eq!(progress(&scanner), 4 * 1024 * 1024 + 30); - scanner.verify().unwrap(); - } - - #[test] - fn staging_respects_remaining_budget_when_a_large_file_follows_a_small_file() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - std::fs::create_dir(root.join("target")).unwrap(); - let small = vec![1; 17]; - let large = vec![2; 4 * 1024 * 1024 + 13]; - std::fs::write(root.join("target/small"), &small).unwrap(); - std::fs::write(root.join("target/large"), &large).unwrap(); - let mut manager = TransferManager::default(); - manager - .execute( - &root, - TransferRequest::Begin { - operation_id: "bounded".into(), - selection: TransferSelection::Materialize { - destination: EnvironmentPath::new(root.join("target").to_str().unwrap()) - .unwrap(), - on_existing: TransferOnExisting::Replace, - }, - limits: InventoryLimits::default(), - }, - ) - .unwrap(); - while manager.operations["bounded"].status.phase == TransferPhase::Scanning { - manager - .execute( - &root, - TransferRequest::Advance { - operation_id: "bounded".into(), - }, - ) - .unwrap(); - } - let mut entries = vec![InventoryEntry { - path: String::new(), - content: InventoryContent::Directory, - }]; - for (name, bytes) in [("renamed-small", &small), ("renamed-large", &large)] { - entries.push(InventoryEntry { - path: name.into(), - content: InventoryContent::File { - size_bytes: bytes.len() as u64, - executable: false, - digest: harness::BlobRef::from_bytes(bytes).to_string(), - }, - }); - } - manager - .execute( - &root, - TransferRequest::Append { - operation_id: "bounded".into(), - offset: 0, - entries, - last: true, - }, - ) - .unwrap(); - manager - .execute( - &root, - TransferRequest::Advance { - operation_id: "bounded".into(), - }, - ) - .unwrap(); - let op = &manager.operations["bounded"]; - assert_eq!(op.status.phase, TransferPhase::Staging); - let copied: u64 = op.entries[..op.staging_index] - .iter() - .map(|entry| match entry.content { - InventoryContent::File { size_bytes, .. } => size_bytes, - InventoryContent::Directory => 0, - }) - .sum::() - + op.copying.as_ref().map_or(0, |copy| copy.bytes); - assert_eq!(copied, 4 * 1024 * 1024); - manager - .execute( - &root, - TransferRequest::Advance { - operation_id: "bounded".into(), - }, - ) - .unwrap(); - assert_eq!( - manager.operations["bounded"].status.phase, - TransferPhase::Ready - ); - manager - .execute( - &root, - TransferRequest::Commit { - operation_id: "bounded".into(), - }, - ) - .unwrap(); - assert_eq!( - std::fs::read(root.join("target/renamed-small")).unwrap(), - small - ); - assert_eq!( - std::fs::read(root.join("target/renamed-large")).unwrap(), - large - ); - } - - #[test] - fn expiry_sweep_reclaims_abandoned_state_and_preserves_live_work() { - let root = tempfile::tempdir().unwrap(); - let state = root.path().join("state"); - std::fs::create_dir(&state).unwrap(); - std::fs::write(root.path().join("destination"), b"keep").unwrap(); - for (id, expires_at_ms) in [ - ("expired", journal::now_ms() - 1), - ("live", journal::now_ms() + 60_000), - ] { - let stage = format!(".env-transfer-{id}"); - std::fs::create_dir_all(root.path().join(&stage).join("nested")).unwrap(); - std::fs::write(root.path().join(&stage).join("nested/file"), b"staged").unwrap(); - let spool = state.join(format!("{id}.content-private")); - std::fs::create_dir(&spool).unwrap(); - std::fs::write(spool.join("file"), b"uploaded").unwrap(); - std::fs::write(state.join(format!("{id}.inventory")), b"inventory").unwrap(); - journal::write( - &state, - id, - &journal::Record { - root: root.path().into(), - request: TransferRequest::Begin { - operation_id: id.into(), - selection: TransferSelection::Materialize { - destination: EnvironmentPath::new( - root.path().join("destination").to_str().unwrap(), - ) - .unwrap(), - on_existing: TransferOnExisting::Replace, - }, - limits: Default::default(), - }, - status: TransferStatus { - operation_id: id.into(), - phase: TransferPhase::Scanning, - entries: 0, - bytes: 0, - transferred_bytes: 0, - reused_bytes: 0, - }, - expires_at_ms, - stage_name: Some(stage), - spool_directory: Some(spool), - }, - ) - .unwrap(); - } - TransferManager::cleanup_expired(&state); - for suffix in ["json", "inventory", "content-private"] { - assert!(!state.join(format!("expired.{suffix}")).exists()); - assert!(state.join(format!("live.{suffix}")).exists()); - } - assert!(!root.path().join(".env-transfer-expired").exists()); - assert!(root.path().join(".env-transfer-live/nested/file").exists()); - assert_eq!( - std::fs::read(root.path().join("destination")).unwrap(), - b"keep" - ); - } -} diff --git a/crates/environment-daemon/src/filesystem/transfer/tests.rs b/crates/environment-daemon/src/filesystem/transfer/tests.rs index e6c6603d7..fc7813aa6 100644 --- a/crates/environment-daemon/src/filesystem/transfer/tests.rs +++ b/crates/environment-daemon/src/filesystem/transfer/tests.rs @@ -1,1331 +1,964 @@ -use super::*; -use async_trait::async_trait; -use environment_client::{EnvironmentClientResult, EnvironmentDataClient, JsonRpcTransport}; +//! Transfer state machine tests against the handlers, without sockets. + +use std::{os::unix::fs::symlink, path::Path}; + use environment_protocol::{ - data::{inventory::*, transfer_session::*}, - shared::EnvironmentPath, -}; -use harness::{ - BlobRef, - storage::{BlobSource, BlobStore, BlobStoreError}, -}; -use serde_json::json; -use std::path::Path; -use std::{ - os::unix::fs::{PermissionsExt, symlink}, - sync::{ - Arc, - atomic::{AtomicBool, AtomicUsize, Ordering}, + data::{ + fs::{TreeContent, TreeFilter, WriteCondition}, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, InventoryEntry, + ListTransferMissingParams, ReadTransferContentParams, ReadTransferInventoryParams, + SpecialFilePolicy, SymlinkPolicy, TransferLimits, TransferOnExisting, + TransferOperationParams, TransferPhase, TransferSelection, TransferSkipReason, + TransferStatus, WriteTransferContentParams, + }, + }, + error::{ + AbortReason, BusyReason, ErrorData, PreconditionReason, ProtocolError, ProtocolErrorKind, }, + ids::TransferOperationId, + paths::{ContentDigest, EnvironmentPath, TreePath}, }; -use tools::{ - environment_protocol::RemoteEnvironmentConnection, - transfer::{self}, +use sha2::{Digest, Sha256}; + +use crate::filesystem::{ + LocalFileSystem, + tests::{context, filesystem}, }; -struct Transport { - runtime: DaemonRuntime, - pending: Option, - largest: Arc, - lose_commit: Arc, +fn id(value: &str) -> TransferOperationId { + TransferOperationId::new(value).unwrap() } -#[async_trait] -impl JsonRpcTransport for Transport { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()> { - let wire = serde_json::to_vec(&message)?; - self.largest.fetch_max(wire.len(), Ordering::Relaxed); - let message: Value = serde_json::from_slice(&wire)?; - let result = handle_data( - &self.runtime, - message["method"].as_str().unwrap(), - message["params"].clone(), - ) - .await; - if message["params"]["action"] == "commit" - && self.lose_commit.swap(false, Ordering::Relaxed) - { - return Err(environment_client::EnvironmentClientError::TransportClosed); - } - self.pending = Some(match result { - Ok(result) => success_response(message["id"].clone(), result), - Err(error) => error_response(Some(message["id"].clone()), error), - }); - Ok(()) - } - async fn recv(&mut self) -> EnvironmentClientResult> { - let Some(response) = self.pending.take() else { - return Ok(None); - }; - let wire = serde_json::to_vec(&response)?; - self.largest.fetch_max(wire.len(), Ordering::Relaxed); - Ok(Some(serde_json::from_slice(&wire)?)) + +fn path(root: &Path, relative: &str) -> EnvironmentPath { + EnvironmentPath::new(format!("{}/{relative}", root.display())).unwrap() +} + +fn limits() -> TransferLimits { + TransferLimits { + max_entries: 1000, + max_depth: 16, + max_file_bytes: 64 * 1024 * 1024, + max_total_bytes: 256 * 1024 * 1024, + max_manifest_bytes: 1024 * 1024, + max_duration_ms: 600_000, } } -fn runtime(root: &Path, read_only: bool) -> DaemonRuntime { - DaemonRuntime::new(crate::config::DaemonConfig { - listen: None, - cwd: root.into(), - fs_root: root.into(), - state_dir: root.join("state"), - read_only_fs: read_only, - registration: None, - scrubbed_env: vec![], - }) - .unwrap() + +fn digest(bytes: &[u8]) -> ContentDigest { + ContentDigest::from_sha256(Sha256::digest(bytes).into()) } -fn path(value: &str) -> EnvironmentPath { - EnvironmentPath::new(value).unwrap() + +fn operation(value: &str) -> TransferOperationParams { + TransferOperationParams { + operation_id: id(value), + } } -fn remote( - runtime: DaemonRuntime, -) -> ( - RemoteEnvironmentConnection, - Arc, - Arc, -) { - let largest = Arc::new(AtomicUsize::new(0)); - let lose_commit = Arc::new(AtomicBool::new(false)); - let caps = runtime.capabilities(); - ( - RemoteEnvironmentConnection::new( - EnvironmentDataClient::new(Transport { - runtime, - pending: None, - largest: largest.clone(), - lose_commit: lose_commit.clone(), - }), - caps, - ), - largest, - lose_commit, + +fn capture(source: EnvironmentPath, filter: TreeFilter) -> TransferSelection { + TransferSelection::Capture { + source, + filter, + symlinks: SymlinkPolicy::ExpandInside, + special_files: SpecialFilePolicy::Reject, + } +} + +async fn begin( + fs: &LocalFileSystem, + operation_id: &str, + selection: TransferSelection, +) -> Result { + fs.transfer_begin( + &context(), + BeginTransferParams { + operation_id: id(operation_id), + selection, + limits: limits(), + }, ) + .await } -struct Repeated { - remaining: u64, + +async fn advance_until( + fs: &LocalFileSystem, + operation_id: &str, + phase: TransferPhase, +) -> Result { + for _ in 0..10_000 { + let status = fs + .transfer_advance(&context(), operation(operation_id)) + .await?; + if status.phase == phase { + return Ok(status); + } + } + panic!("{operation_id} never reached {phase:?}"); } -#[async_trait] -impl BlobSource for Repeated { - async fn read_chunk(&mut self, max_bytes: usize) -> Result, BlobStoreError> { - let size = self.remaining.min(max_bytes as u64) as usize; - self.remaining -= size as u64; - Ok(vec![0x83; size]) + +async fn inventory(fs: &LocalFileSystem, operation_id: &str) -> Vec { + let mut entries = Vec::new(); + let mut index = 0; + loop { + let page = fs + .transfer_read_inventory( + &context(), + ReadTransferInventoryParams { + operation_id: id(operation_id), + index, + limit: 2, + }, + ) + .await + .unwrap(); + entries.extend(page.entries); + match page.next_index { + Some(next) => index = next, + None => return entries, + } } } -fn repeated_ref(size: u64) -> BlobRef { - use sha2::{Digest, Sha256}; - let mut hash = Sha256::new(); - let chunk = [0x83; 65536]; - let mut remaining = size; - while remaining > 0 { - let n = remaining.min(chunk.len() as u64) as usize; - hash.update(&chunk[..n]); - remaining -= n as u64; + +fn reason(error: &ProtocolError) -> Option { + match error.data { + ErrorData::FailedPrecondition { reason } => Some(reason), + _ => None, } - BlobRef::parse(format!("sha256:{}", hex::encode(hash.finalize()))).unwrap() } #[tokio::test(flavor = "current_thread")] -async fn blob_file_tools_stream_without_vfs_and_keep_receipts_across_retries() { - use tools::builtin::{ - BuiltinTool, BuiltinToolContext, BuiltinToolOperation as Op, BuiltinToolSurface as Surface, +async fn captures_expand_inside_links_skip_ignored_trees_and_serve_ranged_content() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + std::fs::create_dir_all(root.join("project/src")).unwrap(); + std::fs::create_dir_all(root.join("project/node_modules/.bin")).unwrap(); + std::fs::create_dir_all(root.join("project/.git")).unwrap(); + std::fs::write(root.join("project/.gitignore"), "node_modules/\n").unwrap(); + std::fs::write(root.join("project/src/lib.rs"), b"0123456789").unwrap(); + std::fs::write(root.join("project/.git/HEAD"), b"ref").unwrap(); + symlink("src/lib.rs", root.join("project/alias.rs")).unwrap(); + symlink( + "../src/missing", + root.join("project/node_modules/.bin/tool"), + ) + .unwrap(); + + let filter = TreeFilter { + include_hidden: true, + ..TreeFilter::default() }; - for surface in [ - Surface::Canonical, - Surface::CodexLike, - Surface::ClaudeCodeLike, - ] { - let environment = tempfile::tempdir().unwrap(); - let cas = tempfile::tempdir().unwrap(); - let store = Arc::new(store_fs::FsBlobStore::open(cas.path()).await.unwrap()); - let size = 9 * 1024 * 1024 + 7; - let content = repeated_ref(size); - store - .put_stream(&content, size, &mut Repeated { remaining: size }) - .await - .unwrap(); - let (connection, largest, lose_commit) = remote(runtime(environment.path(), false)); - let (_, context) = connection - .with_cwd(tools::fs::FsPath::new(environment.path().to_str().unwrap()).unwrap()) - .into_contexts(store.clone()); - let context = context.with_environment_id("test-env"); - let key = if surface == Surface::ClaudeCodeLike { - "file_path" - } else { - "path" - }; - let write = BuiltinTool::environment(Op::WriteFile, surface); - let capture = BuiltinTool::environment(Op::Reference, surface); - let writer = context.clone().with_operation_id("write-binary"); - let output = write - .invoke_json( - BuiltinToolContext::Environment(&writer), - json!({key:"nested/source.bin","content_ref":content}), - ) - .await - .unwrap(); - assert_eq!(output.output_json["bytes_written"], size); - assert_eq!(output.output_json["receipt"]["phase"], "complete"); - assert_eq!( - std::fs::metadata(environment.path().join("nested/source.bin")) - .unwrap() - .len(), - size - ); - let capture_cas = tempfile::tempdir().unwrap(); - let capture_store = Arc::new( - store_fs::FsBlobStore::open(capture_cas.path()) - .await - .unwrap(), - ); - let mut capture_context = context.clone().with_operation_id("capture-binary"); - capture_context.blobs = capture_store.clone(); - let captured = capture - .invoke_json( - BuiltinToolContext::Environment(&capture_context), - json!({"path":"nested/source.bin"}), - ) - .await - .unwrap(); - assert_eq!(captured.output_json["content_ref"], content.to_string()); - assert_eq!(captured.output_json["byte_len"], size); - assert_eq!(captured.attachments.len(), 1); - assert_eq!(captured.output_json["source"]["id"], "test-env"); - assert_eq!( - capture_store.stat_blob(&content).await.unwrap().byte_len, - size - ); - assert_eq!( - capture_store - .read_blob_range(&content, size - 7, 7) - .await - .unwrap(), - vec![0x83; 7] - ); - std::fs::write( - environment.path().join("nested/source.bin"), - b"changed live file", + begin(&fs, "capture", capture(path(&root, "project"), filter)) + .await + .unwrap(); + let status = advance_until(&fs, "capture", TransferPhase::Inventory) + .await + .unwrap(); + let entries = inventory(&fs, "capture").await; + let paths: Vec<_> = entries.iter().map(|entry| entry.path.as_str()).collect(); + // The .git directory is always pruned and node_modules is ignored. + assert_eq!(paths, [".", ".gitignore", "alias.rs", "src", "src/lib.rs"]); + assert_eq!(status.entry_count, 5); + let content = digest(b"0123456789"); + assert!(entries.iter().any(|entry| matches!( + entry.content, + TreeContent::File { digest: Some(found), .. } if found == content + ))); + let first = fs + .transfer_read( + &context(), + ReadTransferContentParams { + operation_id: id("capture"), + digest: content, + offset: 0, + max_bytes: 4, + }, ) + .await .unwrap(); - let changed_source_retry = capture - .invoke_json( - BuiltinToolContext::Environment(&capture_context), - json!({"path":"nested/source.bin"}), - ) - .await - .unwrap(); - assert_eq!( - changed_source_retry.output_json, captured.output_json, - "same-operation capture retry must keep the original immutable version" - ); - std::fs::remove_file(environment.path().join("nested/source.bin")).unwrap(); - let retry = capture - .invoke_json( - BuiltinToolContext::Environment(&capture_context), - json!({"path":"nested/source.bin"}), - ) - .await - .unwrap(); - assert_eq!( - retry.output_json, captured.output_json, - "completed capture must not revisit the deleted live path" - ); - - // Materialization's receipt must make a lost commit response unambiguous. - std::fs::write(environment.path().join("result.bin"), b"original").unwrap(); - std::fs::set_permissions( - environment.path().join("result.bin"), - std::fs::Permissions::from_mode(0o751), + assert_eq!(first.data.as_slice(), b"0123"); + assert!(first.truncated); + let rest = fs + .transfer_read( + &context(), + ReadTransferContentParams { + operation_id: id("capture"), + digest: content, + offset: 4, + max_bytes: 100, + }, ) + .await .unwrap(); - let writer = context.clone().with_operation_id("lost-write-receipt"); - lose_commit.store(true, Ordering::Relaxed); - assert!( - write - .invoke_json( - BuiltinToolContext::Environment(&writer), - json!({key:"result.bin","content_ref":content}) - ) - .await - .is_err() - ); - assert_eq!( - std::fs::metadata(environment.path().join("result.bin")) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o751 - ); - std::fs::write(environment.path().join("result.bin"), b"later local edit").unwrap(); - let retry = write - .invoke_json( - BuiltinToolContext::Environment(&writer), - json!({key:"result.bin","content_ref":content}), - ) - .await - .unwrap(); - assert_eq!(retry.output_json["receipt"]["phase"], "complete"); - assert_eq!( - std::fs::read(environment.path().join("result.bin")).unwrap(), - b"later local edit" - ); - - std::fs::create_dir(environment.path().join("directory")).unwrap(); - std::fs::write(environment.path().join("directory/keep"), b"keep").unwrap(); - let writer = context.clone().with_operation_id("reject-directory-write"); - assert!( - write - .invoke_json( - BuiltinToolContext::Environment(&writer), - json!({key:"directory","content_ref":content}) - ) - .await - .is_err() - ); - assert_eq!( - std::fs::read(environment.path().join("directory/keep")).unwrap(), - b"keep" - ); - let reader = context - .clone() - .with_operation_id("reject-directory-capture"); - assert!( - capture - .invoke_json( - BuiltinToolContext::Environment(&reader), - json!({"path":"directory"}) - ) - .await - .is_err() - ); - assert!( - largest.load(Ordering::Relaxed) < 400_000, - "large binary transfers must use bounded chunks" - ); - } -} - -#[tokio::test(flavor = "current_thread")] -async fn rpc_vfs_roundtrip_streams_large_files_reuses_bytes_and_preserves_retry_receipt() { - let environment = tempfile::tempdir().unwrap(); - let cas = tempfile::tempdir().unwrap(); - let store = store_fs::FsBlobStore::open(cas.path()).await.unwrap(); - let size = 10 * 1024 * 1024 + 7; - let blob = repeated_ref(size); - store - .put_stream(&blob, size, &mut Repeated { remaining: size }) + assert_eq!(rest.data.as_slice(), b"456789"); + assert!(!rest.truncated); + let committed = fs + .transfer_commit(&context(), operation("capture")) .await .unwrap(); - let mut directory = vfs::VfsDirectory::default(); - directory.entries.insert( - "run".into(), - vfs::VfsEntry::File(vfs::VfsFile { - blob_ref: blob.clone(), - size_bytes: size, - executable: true, - media_type: None, - }), - ); - directory.entries.insert( - "empty".into(), - vfs::VfsEntry::Directory(vfs::VfsDirectory::default()), - ); - let selection = vfs::VfsEntry::Directory(directory.clone()); - let (remote, largest, lose) = remote(runtime(environment.path(), false)); - let first = transfer::materialize( - &remote, - &store, - "first", - &selection, - path("selected"), - TransferOnExisting::Replace, - ) - .await - .unwrap(); - assert_eq!(first.bytes, size); - assert_eq!(first.transferred_bytes, size); - assert_eq!(first.reused_bytes, 0); - assert!(environment.path().join("selected/empty").is_dir()); + assert_eq!(committed.phase, TransferPhase::Complete); + assert!(committed.receipt_expires_at_ms.is_some()); + assert!(committed.deadline_at_ms.is_none()); + // The receipt answers a retried commit and begin from the journal. assert_eq!( - std::fs::metadata(environment.path().join("selected/run")) + fs.transfer_commit(&context(), operation("capture")) + .await .unwrap() - .permissions() - .mode() - & 0o111, - 0o111 + .phase, + TransferPhase::Complete ); - use std::os::unix::fs::MetadataExt; - let inode = std::fs::metadata(environment.path().join("selected/run")) - .unwrap() - .ino(); - let unchanged = transfer::materialize( - &remote, - &store, - "unchanged", - &selection, - path("selected"), - TransferOnExisting::Replace, + let error = begin( + &fs, + "capture", + capture(path(&root, "other"), TreeFilter::default()), ) .await - .unwrap(); - assert_eq!(unchanged.transferred_bytes, 0); - assert_eq!(unchanged.reused_bytes, size); - assert_eq!( - std::fs::metadata(environment.path().join("selected/run")) - .unwrap() - .ino(), - inode, - "identical complete trees need no restaging" - ); - // Rename, mode-only edit and deletion reuse raw bytes while replacing the whole boundary. - let vfs::VfsEntry::File(mut file) = directory.entries.remove("run").unwrap() else { - unreachable!() - }; - file.executable = false; - directory.entries.remove("empty"); - directory - .entries - .insert("renamed".into(), vfs::VfsEntry::File(file)); - let selection = vfs::VfsEntry::Directory(directory); - std::fs::write(environment.path().join("sibling"), b"preserved").unwrap(); - lose.store(true, Ordering::Relaxed); - assert!( - transfer::materialize( - &remote, - &store, - "replace", - &selection, - path("selected"), - TransferOnExisting::Replace + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::OperationIdInUse)); + + // The completed capture's inventory stays pageable with its receipt, + // after the source changed and across a restart; its content does not. + std::fs::remove_file(root.join("project/src/lib.rs")).unwrap(); + assert_eq!(inventory(&fs, "capture").await, entries); + drop(fs); + let fs = filesystem(&root, false); + assert_eq!(inventory(&fs, "capture").await, entries); + let error = fs + .transfer_read( + &context(), + ReadTransferContentParams { + operation_id: id("capture"), + digest: content, + offset: 0, + max_bytes: 4, + }, ) .await - .is_err() - ); - assert!(!environment.path().join("selected/run").exists()); - assert!(!environment.path().join("selected/empty").exists()); - std::fs::write( - environment.path().join("selected/local-edit"), - b"keep on retry", - ) - .unwrap(); - let retry = transfer::materialize( - &remote, - &store, - "replace", - &selection, - path("selected"), - TransferOnExisting::Replace, + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::TransferPhase)); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_failed_capture_answers_its_first_error_and_never_commits() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("root"); + std::fs::create_dir_all(root.join("tree/a")).unwrap(); + std::fs::create_dir_all(base.join("outside")).unwrap(); + std::fs::write(root.join("tree/a/file"), b"a").unwrap(); + symlink(base.join("outside"), root.join("tree/escape")).unwrap(); + std::fs::write(root.join("tree/z"), b"z").unwrap(); + let fs = filesystem(&root, false); + + begin( + &fs, + "escape", + capture(path(&root, "tree"), TreeFilter::default()), ) .await .unwrap(); - assert_eq!(retry.phase, TransferPhase::Complete); - assert_eq!(retry.transferred_bytes, 0); - assert_eq!(retry.reused_bytes, size); - assert!(environment.path().join("selected/local-edit").exists()); - std::fs::remove_file(environment.path().join("selected/local-edit")).unwrap(); - let captured = transfer::capture(&remote, &store, None, "capture-existing", path("selected")) + let first = advance_until(&fs, "escape", TransferPhase::Inventory) + .await + .unwrap_err(); + assert_eq!(reason(&first), Some(PreconditionReason::SymlinkRejected)); + // Retries never continue the walk past the failure. + for _ in 0..3 { + let again = fs + .transfer_advance(&context(), operation("escape")) + .await + .unwrap_err(); + assert_eq!(again, first); + } + let error = fs + .transfer_commit(&context(), operation("escape")) + .await + .unwrap_err(); + assert_eq!(error, first); + let status = fs + .transfer_get_status(&context(), operation("escape")) .await .unwrap(); - assert_eq!(captured.entry, selection); - assert_eq!(captured.status.transferred_bytes, 0); - let empty_cas = tempfile::tempdir().unwrap(); - let empty_store = store_fs::FsBlobStore::open(empty_cas.path()).await.unwrap(); - let captured = transfer::capture( - &remote, - &empty_store, - None, - "capture-missing", - path("selected"), + assert_eq!(status.phase, TransferPhase::Failed); + assert_eq!(status.error.as_ref(), Some(&first)); + let aborted = fs + .transfer_abort(&context(), operation("escape")) + .await + .unwrap(); + assert_eq!(aborted.phase, TransferPhase::Failed); + + // A source that changes after hashing fails the operation at read time. + std::fs::remove_file(root.join("tree/escape")).unwrap(); + begin( + &fs, + "changed", + capture(path(&root, "tree"), TreeFilter::default()), ) .await .unwrap(); - assert_eq!(captured.entry, selection); - assert_eq!(captured.status.transferred_bytes, size); - assert!(empty_store.has_blob(&blob).await.unwrap()); + advance_until(&fs, "changed", TransferPhase::Inventory) + .await + .unwrap(); + std::fs::write(root.join("tree/z"), b"zz").unwrap(); + let error = fs + .transfer_read( + &context(), + ReadTransferContentParams { + operation_id: id("changed"), + digest: digest(b"z"), + offset: 0, + max_bytes: 10, + }, + ) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::Aborted { + reason: Some(AbortReason::SourceChanged) + } + )); + let error = fs + .transfer_commit(&context(), operation("changed")) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::Aborted); +} + +#[tokio::test(flavor = "current_thread")] +async fn entries_outside_the_selection_never_fail_a_capture() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("root"); + std::fs::create_dir_all(root.join("tree/src")).unwrap(); + std::fs::create_dir_all(root.join("tree/tmp")).unwrap(); + std::fs::create_dir_all(base.join("outside")).unwrap(); + std::fs::write(root.join("tree/src/lib.rs"), b"lib").unwrap(); + let _socket = std::os::unix::net::UnixListener::bind(root.join("tree/tmp/sock")).unwrap(); + symlink(base.join("outside"), root.join("tree/tmp/escape")).unwrap(); + symlink("missing", root.join("tree/tmp/dangling")).unwrap(); + let fs = filesystem(&root, false); + + let filter = TreeFilter { + include: vec!["src/**".to_owned()], + ..TreeFilter::default() + }; + begin(&fs, "selected", capture(path(&root, "tree"), filter)) + .await + .unwrap(); + let status = advance_until(&fs, "selected", TransferPhase::Inventory) + .await + .unwrap(); + assert_eq!(status.skipped_entry_count, 0, "{:?}", status.skipped); + let entries = inventory(&fs, "selected").await; assert!( - largest.load(Ordering::Relaxed) < 512 * 1024, - "every JSON message remains bounded" + entries + .iter() + .any(|entry| entry.path.as_str() == "src/lib.rs") ); - assert_eq!( - std::fs::read(environment.path().join("sibling")).unwrap(), - b"preserved" + assert!( + !entries + .iter() + .any(|entry| entry.path.as_str().starts_with("tmp/")) ); + + // The same entries fail a capture that selects them. + begin( + &fs, + "everything", + capture(path(&root, "tree"), TreeFilter::default()), + ) + .await + .unwrap(); + let error = advance_until(&fs, "everything", TransferPhase::Inventory) + .await + .unwrap_err(); + assert!(matches!( + reason(&error), + Some(PreconditionReason::SymlinkRejected | PreconditionReason::NotRegularFile) + )); } + #[tokio::test(flavor = "current_thread")] -async fn inventories_are_paged_and_twenty_gib_files_do_not_block_begin_or_one_advance() { +async fn skip_policies_report_what_a_capture_left_out() { let temp = tempfile::tempdir().unwrap(); - std::fs::create_dir(temp.path().join("many")).unwrap(); - for index in 0..300 { - std::fs::write(temp.path().join(format!("many/{index:04}")), b"same").unwrap(); - } - let runtime = runtime(temp.path(), false); - let fs = runtime.filesystem(); - fs.transfer(TransferRequest::Begin { - operation_id: "pages".into(), - selection: TransferSelection::Capture { - source: path("many"), + let root = temp.path().canonicalize().unwrap(); + std::fs::create_dir_all(root.join("tree")).unwrap(); + std::fs::write(root.join("tree/file"), b"f").unwrap(); + symlink("file", root.join("tree/link")).unwrap(); + let fs = filesystem(&root, false); + begin( + &fs, + "skips", + TransferSelection::Capture { + source: path(&root, "tree"), + filter: TreeFilter::default(), + symlinks: SymlinkPolicy::Skip, + special_files: SpecialFilePolicy::Skip, }, - limits: Default::default(), - }) + ) .await .unwrap(); - loop { - if let TransferResponse::Status(status) = fs - .transfer(TransferRequest::Advance { - operation_id: "pages".into(), - }) - .await - .unwrap() - && status.phase == TransferPhase::Ready - { - break; - } - } - let mut offset = 0; - let mut count = 0; - loop { - let TransferResponse::Inventory { - entries, - next_offset, - } = fs - .transfer(TransferRequest::Inventory { - operation_id: "pages".into(), - offset, - }) - .await - .unwrap() - else { - panic!() - }; - assert!(entries.len() <= MAX_INVENTORY_PAGE); - count += entries.len(); - if let Some(next) = next_offset { - offset = next; - } else { - break; - } - } - assert_eq!(count, 301); - let file = std::fs::File::create(temp.path().join("twenty-gib")).unwrap(); - file.set_len(20 * 1024 * 1024 * 1024).unwrap(); - let TransferResponse::Status(start) = fs - .transfer(TransferRequest::Begin { - operation_id: "large".into(), - selection: TransferSelection::Capture { - source: path("twenty-gib"), - }, - limits: Default::default(), - }) + let status = advance_until(&fs, "skips", TransferPhase::Inventory) .await - .unwrap() - else { - panic!() - }; - assert_eq!(start.phase, TransferPhase::Scanning); - let TransferResponse::Status(step) = fs - .transfer(TransferRequest::Advance { - operation_id: "large".into(), - }) - .await - .unwrap() - else { - panic!() - }; - assert_eq!(step.phase, TransferPhase::Scanning); - fs.transfer(TransferRequest::Abort { - operation_id: "large".into(), - }) - .await - .unwrap(); - assert_eq!(file.metadata().unwrap().len(), 20 * 1024 * 1024 * 1024); + .unwrap(); + assert_eq!(status.skipped_entry_count, 1); + assert_eq!(status.skipped[0].reason, TransferSkipReason::Symlink); + assert_eq!( + status.skipped[0].path.as_ref().map(TreePath::as_str), + Some("link") + ); } + #[tokio::test(flavor = "current_thread")] -async fn scan_fingerprints_content_and_filters_large_unrelated_files() { +async fn finished_operations_free_capacity_and_receipts_expire_from_completion() { let temp = tempfile::tempdir().unwrap(); - std::fs::create_dir(temp.path().join("skills")).unwrap(); - std::fs::write(temp.path().join("skills/SKILL.md"), b"first").unwrap(); - std::fs::File::create(temp.path().join("skills/large")) - .unwrap() - .set_len(20 * 1024 * 1024 * 1024) + let root = temp.path().canonicalize().unwrap(); + std::fs::write(root.join("file"), b"x").unwrap(); + let fs = filesystem(&root, false); + // Twice the active ceiling of eight, each finished before the next. + for index in 0..16 { + let name = format!("done-{index}"); + begin( + &fs, + &name, + capture(path(&root, "file"), TreeFilter::default()), + ) + .await .unwrap(); - let runtime = runtime(temp.path(), true); - let fs = runtime.filesystem(); - let mut query = ScanParams { - follow_symlinks: false, - roots: vec![path("skills")], - include_patterns: vec!["**/SKILL.md".into(), "SKILL.md".into()], - read_content: true, - digest_algorithm: None, - limits: Default::default(), - if_none_match: None, - }; - let first = fs.scan(query.clone()).await.unwrap(); - assert!(first.complete); - assert_eq!(first.entries.len(), 1); - query.if_none_match = first.fingerprint; - assert!(fs.scan(query.clone()).await.unwrap().unchanged); - std::fs::write(temp.path().join("skills/SKILL.md"), b"other").unwrap(); - assert!(!fs.scan(query.clone()).await.unwrap().unchanged); - query.roots.push(path("missing")); - let absent = fs.scan(query).await.unwrap(); - assert!(absent.complete); - assert!(!absent.unchanged); - assert!(absent.diagnostics.is_empty()); - let metadata = fs - .scan(ScanParams { - follow_symlinks: false, - roots: vec![path("skills/large")], - include_patterns: vec![], - read_content: false, - digest_algorithm: None, - limits: Default::default(), - if_none_match: None, - }) + advance_until(&fs, &name, TransferPhase::Inventory) + .await + .unwrap(); + fs.transfer_commit(&context(), operation(&name)) + .await + .unwrap(); + } + for index in 0..8 { + begin( + &fs, + &format!("active-{index}"), + capture(path(&root, "file"), TreeFilter::default()), + ) .await .unwrap(); - assert!(metadata.complete); + } + let error = begin( + &fs, + "one-more", + capture(path(&root, "file"), TreeFilter::default()), + ) + .await + .unwrap_err(); assert!(matches!( - metadata.entries[0].content, - ScanContent::File { - digest: None, - size_bytes: 21_474_836_480, + error.data, + ErrorData::Busy { + reason: Some(BusyReason::TransferCapacity), .. } )); - let hashed = fs - .scan(ScanParams { - follow_symlinks: false, - roots: vec![path("skills/SKILL.md")], - include_patterns: vec![], - read_content: false, - digest_algorithm: Some(ScanDigestAlgorithm::Sha256), - limits: Default::default(), - if_none_match: None, - }) + fs.transfer_abort(&context(), operation("active-0")) .await .unwrap(); - assert!(hashed.complete); - assert!( - matches!(&hashed.entries[0].content,ScanContent::File {digest:Some(digest),..} if digest==BlobRef::from_bytes(b"other").as_str()) - ); + begin( + &fs, + "one-more", + capture(path(&root, "file"), TreeFilter::default()), + ) + .await + .unwrap(); } -#[tokio::test(flavor = "current_thread")] -async fn rpc_materialize_creates_missing_parents_for_files_and_trees() { - let environment = tempfile::tempdir().unwrap(); - std::fs::write(environment.path().join("sibling"), b"preserved").unwrap(); - let (remote, _, _) = remote(runtime(environment.path(), false)); - let store = harness::storage::InMemoryBlobStore::new(); - let entry = vfs::VfsEntry::File(vfs::VfsFile { - blob_ref: store.put_bytes(b"content".to_vec()).await.unwrap(), - size_bytes: 7, - executable: true, - media_type: None, - }); - let mut directory = vfs::VfsDirectory::default(); - directory.entries.insert("file".into(), entry.clone()); - for (id, entry, on_existing) in [ - ("file", entry, TransferOnExisting::Error), - ( - "tree", - vfs::VfsEntry::Directory(directory), - TransferOnExisting::Replace, - ), - ] { - let destination = path(&format!("created/{id}/nested/selected")); - let status = transfer::materialize( - &remote, - &store, - id, - &entry, - destination.clone(), - on_existing, + +async fn append_all(fs: &LocalFileSystem, operation_id: &str, entries: Vec) { + let status = fs + .transfer_append( + &context(), + AppendTransferInventoryParams { + operation_id: id(operation_id), + index: 0, + entries, + last: true, + }, ) .await .unwrap(); - assert_eq!(status.phase, TransferPhase::Complete); - let captured = - transfer::capture(&remote, &store, None, &format!("capture-{id}"), destination) - .await - .unwrap(); - assert_eq!(captured.entry, entry); + assert_eq!(status.phase, TransferPhase::Scanning); +} + +fn file_entry(path: &str, bytes: &[u8], executable: bool) -> InventoryEntry { + InventoryEntry { + path: TreePath::new(path).unwrap(), + content: TreeContent::File { + size_bytes: bytes.len() as u64, + executable, + digest: Some(digest(bytes)), + }, + } +} + +fn directory_entry(path: &str) -> InventoryEntry { + InventoryEntry { + path: TreePath::new(path).unwrap(), + content: TreeContent::Directory {}, } - assert_eq!( - std::fs::read(environment.path().join("sibling")).unwrap(), - b"preserved" - ); } #[tokio::test(flavor = "current_thread")] -async fn secure_replacement_rejects_links_and_uses_private_staging_without_reading_write_only_targets() - { +async fn materialize_reuses_matching_destination_files_and_publishes_atomically() { let temp = tempfile::tempdir().unwrap(); - let outside = tempfile::tempdir().unwrap(); - std::fs::write(outside.path().join("secret"), b"untouched").unwrap(); - symlink(outside.path(), temp.path().join("escape")).unwrap(); - symlink("absent", temp.path().join("dangling")).unwrap(); - std::fs::write(temp.path().join("blocked"), b"not a directory").unwrap(); - let read_only = runtime(temp.path(), true); - let runtime = runtime(temp.path(), false); - let fs = runtime.filesystem(); - for (index, destination) in [ - "escape/secret", - "escape/missing/secret", - "dangling/missing/secret", - "blocked/missing/secret", - ] - .into_iter() - .enumerate() - { - assert_eq!( - fs.transfer(TransferRequest::Begin { - operation_id: format!("invalid-parent-{index}"), - selection: TransferSelection::Materialize { - destination: path(destination), - on_existing: TransferOnExisting::Replace - }, - limits: Default::default() - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Forbidden - ); - } - assert!(!outside.path().join("missing").exists()); - assert!(!temp.path().join("absent").exists()); - assert_eq!( - std::fs::read(outside.path().join("secret")).unwrap(), - b"untouched" - ); - assert_eq!( - std::fs::read(temp.path().join("blocked")).unwrap(), - b"not a directory" - ); - assert_eq!( - read_only - .filesystem() - .transfer(TransferRequest::Begin { - operation_id: "read-only-parents".into(), - selection: TransferSelection::Materialize { - destination: path("read-only/missing/target"), - on_existing: TransferOnExisting::Replace - }, - limits: Default::default() - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::CapabilityUnavailable - ); - assert!(!temp.path().join("read-only").exists()); - std::fs::write(temp.path().join("target"), b"old").unwrap(); - std::fs::set_permissions( - temp.path().join("target"), - std::fs::Permissions::from_mode(0o200), + let root = temp.path().canonicalize().unwrap(); + std::fs::create_dir_all(root.join("target/old")).unwrap(); + std::fs::write(root.join("target/old/kept"), b"reused bytes").unwrap(); + std::fs::write(root.join("target/unrelated"), vec![7u8; 100_000]).unwrap(); + let fs = filesystem(&root, true); + + let error = begin( + &fs, + "refused", + TransferSelection::Materialize { + destination: path(&root, "target"), + on_existing: TransferOnExisting::Error, + }, ) - .unwrap(); - fs.transfer(TransferRequest::Begin { - operation_id: "private".into(), - selection: TransferSelection::Materialize { - destination: path("target"), + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::AlreadyExists); + + begin( + &fs, + "tree", + TransferSelection::Materialize { + destination: path(&root, "target"), on_existing: TransferOnExisting::Replace, }, - limits: Default::default(), - }) + ) .await .unwrap(); - for entry in std::fs::read_dir(temp.path()).unwrap() { - let entry = entry.unwrap(); - if entry - .file_name() - .to_string_lossy() - .starts_with(".env-transfer-") - { - assert_eq!( - entry.metadata().unwrap().permissions().mode() & 0o777, - 0o700 - ); - } - } - assert_eq!( - fs.transfer(TransferRequest::Begin { - operation_id: "overlap".into(), - selection: TransferSelection::Materialize { - destination: path("target/nested"), - on_existing: TransferOnExisting::Replace + // A file entry without its digest is refused and changes nothing. + let error = fs + .transfer_append( + &context(), + AppendTransferInventoryParams { + operation_id: id("tree"), + index: 0, + entries: vec![ + directory_entry("."), + InventoryEntry { + path: TreePath::new("bad").unwrap(), + content: TreeContent::File { + size_bytes: 1, + executable: false, + digest: None, + }, + }, + ], + last: true, }, - limits: Default::default() - }) + ) .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Conflict - ); - fs.transfer(TransferRequest::Abort { - operation_id: "private".into(), - }) - .await - .unwrap(); - let (remote, _, _) = remote(runtime.clone()); - let store = harness::storage::InMemoryBlobStore::new(); - let blob = store.put_bytes(b"new".to_vec()).await.unwrap(); - let entry = vfs::VfsEntry::File(vfs::VfsFile { - blob_ref: blob, - size_bytes: 3, - executable: false, - media_type: None, - }); - transfer::materialize( - &remote, - &store, - "write-only", - &entry, - path("target"), - TransferOnExisting::Replace, + .unwrap_err(); + assert!(matches!( + &error.data, + ErrorData::InvalidParams { field: Some(field), .. } if field == "entries[1].content.digest" + )); + let uploaded = b"uploaded content".to_vec(); + append_all( + &fs, + "tree", + vec![ + directory_entry("."), + directory_entry("bin"), + file_entry("bin/run", &uploaded, true), + file_entry("copy", b"reused bytes", false), + file_entry("empty", b"", false), + ], ) - .await - .unwrap(); - assert_eq!(std::fs::read(temp.path().join("target")).unwrap(), b"new"); + .await; + advance_until(&fs, "tree", TransferPhase::Content) + .await + .unwrap(); + let missing = fs + .transfer_list_missing( + &context(), + ListTransferMissingParams { + operation_id: id("tree"), + limit: 10, + }, + ) + .await + .unwrap(); + assert_eq!(missing.digests, [digest(&uploaded)]); + assert_eq!(missing.remaining_count, 0); + let write = |offset: u64, data: &[u8]| WriteTransferContentParams { + operation_id: id("tree"), + digest: digest(&uploaded), + offset, + data: data.into(), + }; + let next = fs + .transfer_write(&context(), write(0, &uploaded[..8])) + .await + .unwrap() + .next_offset; + assert_eq!(next, 8); + // An identical repeat is accepted; a gap is refused without failing. assert_eq!( - std::fs::read(outside.path().join("secret")).unwrap(), - b"untouched" + fs.transfer_write(&context(), write(0, &uploaded[..8])) + .await + .unwrap() + .next_offset, + 8 ); - transfer::materialize( - &remote, - &store, - "to-directory", - &vfs::VfsEntry::Directory(vfs::VfsDirectory::default()), - path("target"), - TransferOnExisting::Replace, - ) - .await - .unwrap(); - assert!(temp.path().join("target").is_dir()); - transfer::materialize( - &remote, - &store, - "back-to-file", - &entry, - path("target"), - TransferOnExisting::Replace, + let error = fs + .transfer_write(&context(), write(10, &uploaded[10..])) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::TransferOffset)); + fs.transfer_write(&context(), write(8, &uploaded[8..])) + .await + .unwrap(); + let ready = advance_until(&fs, "tree", TransferPhase::Ready) + .await + .unwrap(); + assert_eq!(ready.reused_bytes, b"reused bytes".len() as u64); + fs.transfer_commit(&context(), operation("tree")) + .await + .unwrap(); + assert_eq!( + std::fs::read(root.join("target/bin/run")).unwrap(), + uploaded + ); + assert_eq!( + std::fs::read(root.join("target/copy")).unwrap(), + b"reused bytes" + ); + assert!(root.join("target/empty").exists()); + assert!(!root.join("target/unrelated").exists()); + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(root.join("target/bin/run")) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o111, 0o111); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_digest_mismatch_fails_the_operation() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + begin( + &fs, + "mismatch", + TransferSelection::Materialize { + destination: path(&root, "out"), + on_existing: TransferOnExisting::Replace, + }, ) .await .unwrap(); - - let capture = fs - .transfer(TransferRequest::Begin { - operation_id: "observe".into(), - selection: TransferSelection::Capture { - source: path("target"), - }, - limits: Default::default(), - }) + append_all(&fs, "mismatch", vec![file_entry(".", b"expected", false)]).await; + advance_until(&fs, "mismatch", TransferPhase::Content) .await .unwrap(); - assert!(matches!(capture, TransferResponse::Status(_))); - loop { - if let TransferResponse::Status(status) = fs - .transfer(TransferRequest::Advance { - operation_id: "observe".into(), - }) - .await - .unwrap() - && status.phase == TransferPhase::Ready - { - break; - } - } - std::fs::write(temp.path().join("target"), b"changed").unwrap(); - assert_eq!( - fs.transfer(TransferRequest::Commit { - operation_id: "observe".into() - }) + let error = fs + .transfer_write( + &context(), + WriteTransferContentParams { + operation_id: id("mismatch"), + digest: digest(b"expected"), + offset: 0, + data: b"tampered".as_slice().into(), + }, + ) .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Conflict - ); + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::DigestMismatch)); + let status = fs + .transfer_get_status(&context(), operation("mismatch")) + .await + .unwrap(); + assert_eq!(status.phase, TransferPhase::Failed); + assert!(!root.join("out").exists()); } #[tokio::test(flavor = "current_thread")] -async fn persisted_receipts_survive_restart_and_interrupted_operations_fail_closed() { +async fn transfer_write_file_follows_fs_write_file_semantics() { let temp = tempfile::tempdir().unwrap(); - let cas = tempfile::tempdir().unwrap(); - let store = store_fs::FsBlobStore::open(cas.path()).await.unwrap(); - let blob = store.put_bytes(b"content".to_vec()).await.unwrap(); - let entry = vfs::VfsEntry::File(vfs::VfsFile { - blob_ref: blob, - size_bytes: 7, + let root = temp.path().canonicalize().unwrap(); + std::fs::write(root.join("target"), b"old").unwrap(); + symlink("target", root.join("link")).unwrap(); + let fs = filesystem(&root, true); + let selection = |create: WriteCondition| TransferSelection::WriteFile { + destination: path(&root, "link"), + create, + if_match: None, executable: false, - media_type: None, - }); - { - let (remote, _, _) = remote(runtime(temp.path(), false)); - transfer::materialize( - &remote, - &store, - "completed", - &entry, - path("target"), - TransferOnExisting::Replace, - ) + }; + let error = begin(&fs, "absent", selection(WriteCondition::IfAbsent)) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::CreateCondition)); + begin(&fs, "write", selection(WriteCondition::IfPresent)) .await .unwrap(); - transfer::capture(&remote, &store, None, "captured", path("target")) - .await - .unwrap(); - use tools::transfer::EnvironmentTransfer; - remote - .request(TransferRequest::Begin { - operation_id: "interrupted".into(), - selection: TransferSelection::Materialize { - destination: path("target"), - on_existing: TransferOnExisting::Replace, - }, - limits: Default::default(), - }) - .await - .unwrap(); - } - std::fs::write(temp.path().join("target"), b"later edit").unwrap(); - let (remote, _, _) = remote(runtime(temp.path(), false)); - let receipt = transfer::materialize( - &remote, - &store, - "completed", - &entry, - path("target"), - TransferOnExisting::Replace, + append_all(&fs, "write", vec![file_entry(".", b"new content", false)]).await; + advance_until(&fs, "write", TransferPhase::Content) + .await + .unwrap(); + fs.transfer_write( + &context(), + WriteTransferContentParams { + operation_id: id("write"), + digest: digest(b"new content"), + offset: 0, + data: b"new content".as_slice().into(), + }, ) .await .unwrap(); - assert_eq!(receipt.phase, TransferPhase::Complete); - assert_eq!( - std::fs::read(temp.path().join("target")).unwrap(), - b"later edit" - ); - let capture = transfer::capture(&remote, &store, None, "captured", path("target")) + advance_until(&fs, "write", TransferPhase::Ready) .await .unwrap(); - assert_eq!(capture.entry, entry); - assert!( - transfer::materialize( - &remote, - &store, - "interrupted", - &entry, - path("target"), - TransferOnExisting::Replace - ) + fs.transfer_commit(&context(), operation("write")) .await - .is_err() - ); - assert_eq!( - std::fs::read(temp.path().join("target")).unwrap(), - b"later edit" + .unwrap(); + assert_eq!(std::fs::read(root.join("target")).unwrap(), b"new content"); + assert!( + std::fs::symlink_metadata(root.join("link")) + .unwrap() + .file_type() + .is_symlink() ); } + #[tokio::test(flavor = "current_thread")] -async fn bad_chunks_invalid_manifests_and_read_only_requests_never_publish() { +async fn read_only_daemons_refuse_publishes() { let temp = tempfile::tempdir().unwrap(); - std::fs::write(temp.path().join("old"), b"keep").unwrap(); - let runtime = runtime(temp.path(), false); - let fs = runtime.filesystem(); - fs.transfer(TransferRequest::Begin { - operation_id: "invalid".into(), - selection: TransferSelection::Materialize { - destination: path("old"), + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + let error = begin( + &fs, + "denied", + TransferSelection::Materialize { + destination: path(&root, "out"), on_existing: TransferOnExisting::Replace, }, - limits: Default::default(), - }) + ) .await - .unwrap(); - loop { - if let TransferResponse::Status(status) = fs - .transfer(TransferRequest::Advance { - operation_id: "invalid".into(), - }) - .await - .unwrap() - && status.phase == TransferPhase::Inventory - { - break; - } - } - let digest = BlobRef::from_bytes(b"new").to_string(); - let file = |path: &str| InventoryEntry { - path: path.into(), - content: InventoryContent::File { - size_bytes: 3, - executable: false, - digest: digest.clone(), - }, + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::CapabilityUnavailable); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_restart_fails_active_operations_and_discards_their_staging() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let selection = TransferSelection::Materialize { + destination: path(&root, "out"), + on_existing: TransferOnExisting::Replace, }; - for bad in ["../escape", "/absolute", "a//b", "a/../b", "a\\b"] { - assert_eq!( - fs.transfer(TransferRequest::Append { - operation_id: "invalid".into(), - offset: 0, - entries: vec![file(bad)], - last: true - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::InvalidRequest - ); + { + let fs = filesystem(&root, true); + begin(&fs, "interrupted", selection.clone()).await.unwrap(); + let staged = std::fs::read_dir(&root).unwrap().flatten().any(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".lightspeed-transfer-") + }); + assert!(staged); } - let page = TransferRequest::Append { - operation_id: "invalid".into(), - offset: 0, - entries: vec![file("")], - last: true, - }; - fs.transfer(page.clone()).await.unwrap(); - fs.transfer(page).await.unwrap(); - assert_eq!( - fs.transfer(TransferRequest::Write { - operation_id: "invalid".into(), - digest: digest.clone(), - offset: 0, - data: vec![0; MAX_CONTENT_CHUNK + 1].into() - }) + let fs = filesystem(&root, true); + let status = fs + .transfer_get_status(&context(), operation("interrupted")) .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::InvalidRequest - ); + .unwrap(); + assert_eq!(status.phase, TransferPhase::Failed); assert_eq!( - fs.transfer(TransferRequest::Write { - operation_id: "invalid".into(), - digest: digest.clone(), - offset: 0, - data: b"bad".as_slice().into() - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Conflict - ); - assert_eq!(std::fs::read(temp.path().join("old")).unwrap(), b"keep"); - let chunk = TransferRequest::Write { - operation_id: "invalid".into(), - digest, - offset: 0, - data: b"new".as_slice().into(), - }; - fs.transfer(chunk.clone()).await.unwrap(); - fs.transfer(chunk).await.unwrap(); - fs.transfer(TransferRequest::Abort { - operation_id: "invalid".into(), - }) + status.error.as_ref().map(ProtocolError::kind), + Some(ProtocolErrorKind::Aborted) + ); + let error = begin(&fs, "interrupted", selection).await.unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::Aborted); + // Staging cleanup of the dropped instance and recovery both remove it. + for _ in 0..100 { + let staged = std::fs::read_dir(&root).unwrap().flatten().any(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".lightspeed-transfer-") + }); + if !staged { + return; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + panic!("staging survived the restart"); +} + +#[tokio::test(flavor = "current_thread")] +async fn operations_fail_when_their_duration_runs_out() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + std::fs::write(root.join("file"), b"x").unwrap(); + let fs = filesystem(&root, false); + fs.transfer_begin( + &context(), + BeginTransferParams { + operation_id: id("short"), + selection: capture(path(&root, "file"), TreeFilter::default()), + limits: TransferLimits { + max_duration_ms: 1, + ..limits() + }, + }, + ) .await .unwrap(); - assert_eq!( - fs.transfer(TransferRequest::Commit { - operation_id: "invalid".into() - }) + std::thread::sleep(std::time::Duration::from_millis(5)); + let error = fs + .transfer_advance(&context(), operation("short")) .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Conflict - ); - let read_only = crate::server::streaming_transfer_tests::runtime(temp.path(), true); - assert_eq!( - read_only - .filesystem() - .transfer(TransferRequest::Begin { - operation_id: "denied".into(), - selection: TransferSelection::Materialize { - destination: path("old"), - on_existing: TransferOnExisting::Replace - }, - limits: Default::default() - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::CapabilityUnavailable - ); - assert_eq!(std::fs::read(temp.path().join("old")).unwrap(), b"keep"); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::DeadlineExceeded); + let status = fs + .transfer_get_status(&context(), operation("short")) + .await + .unwrap(); + assert_eq!(status.phase, TransferPhase::Failed); } #[tokio::test(flavor = "current_thread")] -async fn configured_cwd_aliases_work_under_the_host_root() { +async fn a_retried_sealing_append_answers_the_status() { let temp = tempfile::tempdir().unwrap(); - let alias = temp.path().join("cwd-alias"); - let actual = temp.path().join("actual"); - std::fs::create_dir(&actual).unwrap(); - symlink(&actual, &alias).unwrap(); - std::fs::write(actual.join("file"), b"bytes").unwrap(); - let fs = crate::filesystem::LocalFileSystem::new(Path::new("/").into(), alias, true); - let result = fs - .scan(ScanParams { - follow_symlinks: false, - roots: vec![path("file")], - include_patterns: vec![], - read_content: true, - digest_algorithm: None, - limits: Default::default(), - if_none_match: None, - }) + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + begin( + &fs, + "sealed", + TransferSelection::Materialize { + destination: path(&root, "out"), + on_existing: TransferOnExisting::Replace, + }, + ) + .await + .unwrap(); + let entries = vec![directory_entry("."), file_entry("file", b"bytes", false)]; + append_all(&fs, "sealed", entries.clone()).await; + // The response to the sealing page was lost; the client sends it again. + let retried = fs + .transfer_append( + &context(), + AppendTransferInventoryParams { + operation_id: id("sealed"), + index: 0, + entries, + last: true, + }, + ) .await .unwrap(); - assert!(result.complete, "{:?}", result.diagnostics); - assert_eq!( - result.entries[0].data.as_ref().unwrap().as_slice(), - b"bytes" - ); - symlink(&actual, actual.join("user-link")).unwrap(); - let result = fs - .scan(ScanParams { - follow_symlinks: false, - roots: vec![path("user-link/file")], - include_patterns: vec![], - read_content: true, - digest_algorithm: None, - limits: Default::default(), - if_none_match: None, - }) + assert_ne!(retried.phase, TransferPhase::Inventory); + assert_eq!(retried.entry_count, 2); + let error = fs + .transfer_append( + &context(), + AppendTransferInventoryParams { + operation_id: id("sealed"), + index: 0, + entries: vec![directory_entry(".")], + last: true, + }, + ) .await - .unwrap(); - assert!(!result.complete, "user-selected symlinks remain rejected"); + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::TransferPhase)); } #[tokio::test(flavor = "current_thread")] -async fn scan_installer_layouts_aliases_collisions_and_semantic_publication() { - use tools::skills::environment::*; +async fn a_refused_begin_and_an_abort_leave_no_created_directories() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); - let home = root.join("home"); - let cwd = root.join("project/subdir"); - let canonical = root.join("installed/review"); - for directory in [ - &home, - &cwd, - &canonical, - &root.join("project/.agents/skills"), - &home.join(".codex/skills/copy"), - ] { - std::fs::create_dir_all(directory).unwrap(); - } - let doc = "---\nname: review\ndescription: >-\n Review code\n carefully.\nmetadata:\n hooks: [ignored]\n---\nbody one"; - std::fs::write(canonical.join("SKILL.md"), doc).unwrap(); - std::fs::write(home.join(".codex/skills/copy/SKILL.md"), doc).unwrap(); - symlink(&canonical, root.join("project/.agents/skills/review")).unwrap(); - symlink(&canonical, root.join("project/.agents/skills/alias")).unwrap(); - let config = harness::EnvironmentSkillsConfig { - roots: Some(vec![ - root.join("project/.agents/skills") - .to_string_lossy() - .into_owned(), - home.join(".codex/skills").to_string_lossy().into_owned(), - ]), + let fs = filesystem(&root, true); + let materialize = |destination: &str| TransferSelection::Materialize { + destination: path(&root, destination), + on_existing: TransferOnExisting::Replace, }; - let runtime = runtime(&root, true); - let mut query = environment_skill_scan_query(&config, cwd.to_str(), home.to_str()).unwrap(); - let first = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(first.complete, "{:?}", first.diagnostics); - let catalog = environment_skill_catalog("machine-a", &first).unwrap(); - assert_eq!( - catalog.skills.len(), - 2, - "aliases collapse but independent copies remain" - ); - assert!( - catalog - .skills - .iter() - .all(|skill| skill.description == "Review code carefully.") - ); - assert_ne!(catalog.skills[0].skill_id, catalog.skills[1].skill_id); - let other = environment_skill_catalog("machine-b", &first).unwrap(); - assert_ne!(catalog.skills[0].skill_id, other.skills[0].skill_id); - query.if_none_match = first.fingerprint; - assert!( - runtime - .filesystem() - .scan(query.clone()) - .await - .unwrap() - .unchanged - ); - std::fs::write(canonical.join("SKILL.md"), doc.replace("one", "two")).unwrap(); - let body_edit = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(!body_edit.unchanged); - assert_eq!( - catalog, - environment_skill_catalog("machine-a", &body_edit).unwrap() - ); - assert!( - String::from_utf8(std::fs::read(canonical.join("SKILL.md")).unwrap()) - .unwrap() - .ends_with("body two") - ); - let blobs = harness::storage::InMemoryBlobStore::new(); - let Some(harness::CoreAgentCommand::UpsertContext { entry, .. }) = - publish_environment_skill_catalog(&blobs, None, &catalog) - .await - .unwrap() - else { - panic!("publication"); - }; - assert!( - publish_environment_skill_catalog( - &blobs, - Some(&entry), - &environment_skill_catalog("machine-a", &body_edit).unwrap() - ) + begin(&fs, "outer", materialize("a/b")).await.unwrap(); + assert!(root.join("a").is_dir(), "begin creates missing parents"); + let error = begin(&fs, "inner", materialize("a/b/c/d")) .await - .unwrap() - .is_none() - ); - let prefix = blobs.read_bytes(&entry.content.content_ref).await.unwrap(); - std::fs::write( - canonical.join("SKILL.md"), - doc.replace("carefully", "thoroughly"), - ) - .unwrap(); - let edited = runtime.filesystem().scan(query.clone()).await.unwrap(); + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::Busy { + reason: Some(BusyReason::TransferCapacity), + .. + } + )); assert!( - publish_environment_skill_catalog( - &blobs, - Some(&entry), - &environment_skill_catalog("machine-a", &edited).unwrap() - ) - .await - .unwrap() - .is_some() - ); - assert_eq!( - blobs.read_bytes(&entry.content.content_ref).await.unwrap(), - prefix - ); - std::fs::remove_file(canonical.join("SKILL.md")).unwrap(); - let removed = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert_eq!( - environment_skill_catalog("machine-a", &removed) - .unwrap() - .skills - .len(), - 1 + !root.join("a/b").exists(), + "a refused begin changes nothing" ); - std::fs::write( - canonical.join("SKILL.md"), - "---\nname: broken\ndescription: [invalid]\n---", - ) - .unwrap(); - let invalid = environment_skill_catalog( - "machine-a", - &runtime.filesystem().scan(query).await.unwrap(), - ) - .unwrap(); - assert_eq!(invalid.skills.len(), 1); - assert_eq!(invalid.warnings.len(), 1); + + fs.transfer_abort(&context(), operation("outer")) + .await + .unwrap(); + for _ in 0..100 { + if !root.join("a").exists() { + return; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + panic!("the aborted begin's parent directory survived"); } #[tokio::test(flavor = "current_thread")] -async fn scan_retarget_scope_loops_and_aggregate_limits_are_explicit() { +async fn a_receipt_names_its_staging_until_removal_is_confirmed() { + use super::journal::{self, Record, StageRecord}; + let temp = tempfile::tempdir().unwrap(); - let outside = tempfile::tempdir().unwrap(); - for name in ["one", "two"] { - std::fs::create_dir(temp.path().join(name)).unwrap(); - std::fs::write(temp.path().join(name).join("SKILL.md"), "same").unwrap(); - } - symlink(temp.path().join("one"), temp.path().join("alias")).unwrap(); - let runtime = runtime(temp.path(), true); - let mut query = ScanParams { - roots: vec![path("alias")], - include_patterns: vec!["**/SKILL.md".into(), "SKILL.md".into()], - read_content: true, - follow_symlinks: true, - digest_algorithm: Some(ScanDigestAlgorithm::Sha256), - limits: InventoryLimits::default(), - if_none_match: None, + let root = temp.path().canonicalize().unwrap(); + let selection = TransferSelection::Materialize { + destination: path(&root, "out"), + on_existing: TransferOnExisting::Replace, }; - let first = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(first.complete); - query.if_none_match = first.fingerprint; - let parent_scope = environment_daemon_scope_for_test(temp.path()); - assert!( - !parent_scope.scan(query.clone()).await.unwrap().unchanged, - "changed access scope cannot authorize reuse" - ); - let mut shallow = query.clone(); - shallow.limits.max_depth = 0; - let truncated = runtime.filesystem().scan(shallow).await.unwrap(); - assert!(!truncated.complete && !truncated.unchanged); - let mut directories = query.clone(); - directories.include_patterns.clear(); - directories.read_content = false; - directories.digest_algorithm = None; - assert!( - runtime - .filesystem() - .scan(directories) - .await - .unwrap() - .entries - .iter() - .any(|entry| matches!(entry.content, ScanContent::Directory)) - ); - std::fs::remove_file(temp.path().join("alias")).unwrap(); - symlink(temp.path().join("two"), temp.path().join("alias")).unwrap(); - let retargeted = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(retargeted.complete && !retargeted.unchanged); - assert_ne!( - first.entries[0].canonical_path, - retargeted.entries[0].canonical_path - ); - assert!( - matches!(&retargeted.entries[0].content, ScanContent::File { digest: Some(digest), .. } if digest == BlobRef::from_bytes(b"same").as_str()) - ); - query.if_none_match = retargeted.fingerprint; - query.read_content = false; - assert!( - !runtime - .filesystem() - .scan(query.clone()) - .await - .unwrap() - .unchanged - ); - query.roots = vec![path("one"), path("two")]; - query.limits.max_total_bytes = 7; - let limited = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(!limited.complete && !limited.unchanged && limited.fingerprint.is_none()); - assert!(!limited.diagnostics.is_empty()); - query.limits = InventoryLimits::default(); - query.roots = vec![path("alias")]; - symlink(temp.path().join("two"), temp.path().join("two/loop")).unwrap(); - assert!( - !runtime - .filesystem() - .scan(query.clone()) + let status = { + let fs = filesystem(&root, true); + begin(&fs, "done", selection.clone()).await.unwrap(); + fs.transfer_abort(&context(), operation("done")) .await .unwrap() - .complete - ); - std::fs::remove_file(temp.path().join("two/loop")).unwrap(); - std::fs::remove_file(temp.path().join("alias")).unwrap(); - symlink(outside.path(), temp.path().join("alias")).unwrap(); - let escaped = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(!escaped.complete); - assert_eq!( - escaped.diagnostics[0].error.code, - EnvironmentProtocolErrorCode::Forbidden - ); - std::fs::remove_file(temp.path().join("alias")).unwrap(); - symlink(temp.path().join("missing"), temp.path().join("alias")).unwrap(); - let dangling = runtime.filesystem().scan(query).await.unwrap(); - assert!(!dangling.complete && !dangling.unchanged); -} - -fn environment_daemon_scope_for_test(cwd: &std::path::Path) -> crate::filesystem::LocalFileSystem { - crate::filesystem::LocalFileSystem::new( - cwd.parent().unwrap().to_path_buf(), - cwd.to_path_buf(), - false, + }; + assert_eq!(status.phase, TransferPhase::Aborted); + // An exit before the background removal finished: the stage is still + // there and the receipt names it. + let stage = ".lightspeed-transfer-0123456789abcdef0123456789abcdef"; + std::fs::create_dir_all(root.join(stage).join("tree/replaced")).unwrap(); + let journal = root.join(".state/transfers"); + journal::write( + &journal, + &id("done"), + &Record { + params: BeginTransferParams { + operation_id: id("done"), + selection, + limits: limits(), + }, + status: status.clone(), + stage: Some(StageRecord::new(&[], stage)), + spool: None, + committing: false, + }, ) + .unwrap(); + + let fs = filesystem(&root, true); + assert!(!root.join(stage).exists(), "recovery removes the stage"); + let record = journal::read(&journal, &id("done")).unwrap().unwrap(); + assert!(record.stage.is_none()); + let answered = fs + .transfer_get_status(&context(), operation("done")) + .await + .unwrap(); + assert_eq!(answered.phase, TransferPhase::Aborted); } diff --git a/crates/environment-daemon/src/filesystem/walk.rs b/crates/environment-daemon/src/filesystem/walk.rs new file mode 100644 index 000000000..a56bf5871 --- /dev/null +++ b/crates/environment-daemon/src/filesystem/walk.rs @@ -0,0 +1,754 @@ +//! One traversal for `fs/searchText`, `fs/globFiles`, `fs/scan` and transfer +//! captures: the [`TreeFilter`] pattern grammar and a resumable walk beneath +//! the root descriptor. +//! +//! Patterns match paths relative to the walk's root, gitignore-style: `*` +//! and `?` within one segment, `**` across segments, `[...]` and `{a,b}`. A +//! pattern without `/` matches the entry's name at any depth, a `/` anywhere +//! but at the end anchors it to the root, and a trailing `/` matches +//! directories only. The backslash is an ordinary character, as in names. +//! +//! The walk visits children in byte order of their names, depth first, so +//! results are sorted by path. It always prunes VCS metadata directories and +//! the daemon state directory; by default it respects `.gitignore`, +//! `.ignore` and `.git/info/exclude` (in that ascending precedence, each +//! level overriding its ancestors) and skips names starting with `.`. + +use std::{ + collections::HashSet, + ffi::{OsStr, OsString}, + os::unix::ffi::OsStrExt, + path::PathBuf, + sync::Arc, +}; + +use environment_protocol::{ + data::fs::{FileKind, TreeFilter}, + error::{PermissionReason, PreconditionReason, ProtocolError}, + paths::TreePath, +}; +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use ignore::gitignore::{Gitignore, GitignoreBuilder}; + +use super::{ + confinement::{Confinement, Resolved}, + io_error, + os::{self, Directory, EntryStat}, +}; + +/// Directories every walk prunes, whatever the filter says. +const VCS_DIRECTORIES: [&str; 4] = [".git", ".hg", ".svn", ".jj"]; +/// Steps (pruned entries, listings) a walk takes without an event before it +/// yields, so its caller can check its deadline and cancellation. +const STEPS_PER_YIELD: u32 = 1024; +/// Largest ignore file read; longer files are cut. +const MAX_IGNORE_FILE_BYTES: u64 = 1024 * 1024; + +/// A compiled pattern list. +#[derive(Clone, Debug, Default)] +pub(crate) struct PatternSet { + any: Option, + directories: Option, + count: usize, +} + +impl PatternSet { + /// Compile `patterns`; `field` names the list in errors + /// (`filter.include`). + pub fn compile( + patterns: &[String], + ignore_case: bool, + field: &str, + ) -> Result { + let mut any = GlobSetBuilder::new(); + let mut directories = GlobSetBuilder::new(); + let (mut any_count, mut directory_count) = (0, 0); + for (index, pattern) in patterns.iter().enumerate() { + let field = format!("{field}[{index}]"); + let (glob, directory_only) = translate(pattern) + .ok_or_else(|| ProtocolError::invalid_field(&field, "empty pattern"))?; + let glob = GlobBuilder::new(&glob) + .literal_separator(true) + .backslash_escape(false) + .case_insensitive(ignore_case) + .build() + .map_err(|error| { + ProtocolError::invalid_field(&field, format!("invalid pattern: {error}")) + })?; + if directory_only { + directories.add(glob); + directory_count += 1; + } else { + any.add(glob); + any_count += 1; + } + } + let build = |builder: GlobSetBuilder, count: usize| -> Result<_, ProtocolError> { + if count == 0 { + return Ok(None); + } + builder + .build() + .map(Some) + .map_err(|error| ProtocolError::invalid_field(field, error.to_string())) + }; + Ok(Self { + any: build(any, any_count)?, + directories: build(directories, directory_count)?, + count: patterns.len(), + }) + } + + pub fn is_empty(&self) -> bool { + self.count == 0 + } + + pub fn matches(&self, path: &TreePath, is_dir: bool) -> bool { + let path = path.as_str(); + self.any.as_ref().is_some_and(|set| set.is_match(path)) + || (is_dir + && self + .directories + .as_ref() + .is_some_and(|set| set.is_match(path))) + } +} + +/// Gitignore-style anchoring into a globset pattern: `(glob, directory +/// only)`, or `None` for an empty pattern. +fn translate(pattern: &str) -> Option<(String, bool)> { + let (body, directory_only) = match pattern.strip_suffix('/') { + Some(body) if !body.is_empty() => (body, true), + _ => (pattern, false), + }; + if body.is_empty() || body == "/" { + return None; + } + let glob = if let Some(anchored) = body.strip_prefix('/') { + anchored.to_owned() + } else if body.contains('/') { + body.to_owned() + } else { + format!("**/{body}") + }; + Some((glob, directory_only)) +} + +/// A [`TreeFilter`] ready to walk. +#[derive(Clone, Debug)] +pub(crate) struct CompiledFilter { + pub include: PatternSet, + pub exclude: PatternSet, + pub include_hidden: bool, + pub include_ignored: bool, + pub max_depth: Option, +} + +impl CompiledFilter { + /// `prefix` is the wire field holding the filter (`filter`). + pub fn compile(filter: &TreeFilter, prefix: &str) -> Result { + Ok(Self { + include: PatternSet::compile( + &filter.include, + filter.ignore_case, + &format!("{prefix}.include"), + )?, + exclude: PatternSet::compile( + &filter.exclude, + filter.ignore_case, + &format!("{prefix}.exclude"), + )?, + include_hidden: filter.include_hidden, + include_ignored: filter.include_ignored, + max_depth: filter.max_depth, + }) + } + + /// Visit everything: no patterns, hidden and ignored entries included. + pub fn everything() -> Self { + Self { + include: PatternSet::default(), + exclude: PatternSet::default(), + include_hidden: true, + include_ignored: true, + max_depth: None, + } + } + + pub fn selects(&self, path: &TreePath, is_dir: bool) -> bool { + self.include.is_empty() || self.include.matches(path, is_dir) + } +} + +/// What a walk does with a symlink it meets below its root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LinkMode { + /// Report the link itself as an entry of kind symlink. + Report, + /// Follow it when its target stays inside the filesystem root; report + /// it as skipped otherwise. + Follow, +} + +/// One step of a walk. +#[derive(Debug)] +pub(crate) enum Event { + Entry(Entry), + /// Nothing to report yet: the walk pruned or listed for a while. The + /// caller checks its bounds and steps again. + Yield, + Skip { + /// Absent when the name is not UTF-8. + path: Option, + reason: Skip, + /// The skipped entry matters to the filter: its path, taken as a + /// non-directory, matches `include` (always without includes), or + /// it is a directory the walk had to descend. Always true without a + /// path. + selected: bool, + }, +} + +/// Why a walk left an entry out. +#[derive(Debug)] +pub(crate) enum Skip { + NonUtf8Name, + /// The entry vanished, could not be read, or (a directory) could not be + /// listed. + Unreadable(ProtocolError), + /// A followed link left the root, addressed the state directory, looped + /// or closed a directory cycle. + LinkOutsideRoot(ProtocolError), + /// A followed link whose target does not exist. + DanglingLink, +} + +/// An entry the walk visited. +#[derive(Debug)] +pub(crate) struct Entry { + /// Relative to the walk's root, as walked. + pub path: TreePath, + pub depth: u32, + /// The directory holding the entry: for a followed link, its target. + pub parent: Arc, + pub name: OsString, + /// Canonical segments below the filesystem root. + pub canonical: Vec, + /// `lstat` of the entry, or of the link's target when it was followed. + pub stat: EntryStat, + /// Reached through a followed symlink. + pub via_link: bool, + /// Matches the filter's `include` (always true without includes). + pub selected: bool, +} + +struct Frame { + directory: Arc, + canonical: Vec, + path: TreePath, + depth: u32, + identity: (u64, u64), + /// Children left to visit, in reverse order; `None` until listed. + names: Option>, + /// Ignore levels pushed when this frame was listed. + ignore_levels: usize, +} + +struct IgnoreLevel { + matcher: Gitignore, +} + +/// A depth-first walk beneath a resolved directory. Resumable: callers step +/// it with [`Walker::next_event`] and may stop at any time. +pub(crate) struct Walker { + confinement: Arc, + filter: CompiledFilter, + links: LinkMode, + max_depth: Option, + /// Coordinates of ignore matching: the walk root's canonical segments. + base: PathBuf, + stack: Vec, + ignores: Vec, + identities: HashSet<(u64, u64)>, +} + +impl Walker { + /// Walk below `root`, which must be a resolved directory. `max_depth` + /// overrides the filter's. + pub fn new( + confinement: Arc, + root: &Resolved, + filter: CompiledFilter, + links: LinkMode, + max_depth: Option, + subject: &dyn std::fmt::Display, + ) -> Result { + let directory = root + .open_directory() + .map_err(|error| io_error(error, subject))?; + let stat = EntryStat::of_directory(&directory).map_err(|error| io_error(error, subject))?; + let base = segments_path(&root.segments); + let mut walker = Self { + confinement, + max_depth: max_depth.or(filter.max_depth), + filter, + links, + base, + stack: Vec::new(), + ignores: Vec::new(), + identities: HashSet::new(), + }; + if !walker.filter.include_ignored { + walker.load_ancestor_ignores(&root.segments); + } + walker.identities.insert((stat.dev, stat.ino)); + walker.stack.push(Frame { + directory: Arc::new(directory), + canonical: root.segments.clone(), + path: TreePath::root(), + depth: 0, + identity: (stat.dev, stat.ino), + names: None, + ignore_levels: 0, + }); + Ok(walker) + } + + /// The next entry or skip, [`Event::Yield`] after a stretch without + /// one, or `None` when the walk is done. + pub fn next_event(&mut self) -> Option { + for _ in 0..STEPS_PER_YIELD { + let frame = self.stack.last_mut()?; + if frame.names.is_none() { + if let Some(event) = self.list_top() { + return Some(event); + } + continue; + } + let frame = self.stack.last_mut().expect("frame listed above"); + let Some(name) = frame.names.as_mut().and_then(Vec::pop) else { + self.pop_frame(); + continue; + }; + if let Some(event) = self.visit(name) { + return Some(event); + } + } + Some(Event::Yield) + } + + /// List the top frame; an unlistable directory is skipped and popped. + fn list_top(&mut self) -> Option { + let frame = self.stack.last_mut().expect("caller checked"); + if self.max_depth.is_some_and(|max| frame.depth >= max) { + frame.names = Some(Vec::new()); + return None; + } + match frame.directory.names() { + Ok(mut names) => { + names.sort_by(|left, right| right.as_bytes().cmp(left.as_bytes())); + frame.names = Some(names); + if !self.filter.include_ignored { + let directory = frame.directory.clone(); + let at = self.base.join(segments_path_tree(&frame.path)); + let pushed = self.load_ignores(&directory, at); + self.stack.last_mut().expect("frame").ignore_levels = pushed; + } + None + } + Err(error) => { + let path = frame.path.clone(); + let error = io_error(error, &path); + self.pop_frame(); + Some(Event::Skip { + path: (!path.is_root()).then_some(path), + reason: Skip::Unreadable(error), + selected: true, + }) + } + } + } + + fn pop_frame(&mut self) { + if let Some(frame) = self.stack.pop() { + self.ignores + .truncate(self.ignores.len().saturating_sub(frame.ignore_levels)); + self.identities.remove(&frame.identity); + } + } + + fn visit(&mut self, name: OsString) -> Option { + let frame = self.stack.last().expect("visiting a listed frame"); + let depth = frame.depth + 1; + let Some(utf8) = os::utf8_name(&name) else { + return Some(Event::Skip { + path: None, + reason: Skip::NonUtf8Name, + selected: true, + }); + }; + if VCS_DIRECTORIES.contains(&utf8) { + return None; + } + let mut canonical = frame.canonical.clone(); + canonical.push(name.clone()); + if !self.filter.include_hidden && utf8.starts_with('.') { + return None; + } + let path = frame + .path + .join(utf8) + .expect("a UTF-8 name without '/' joins"); + let parent = frame.directory.clone(); + let parent_canonical = frame.canonical.clone(); + let stat = match parent.stat(&name) { + Ok(stat) => stat, + Err(error) => { + let error = io_error(error, &path); + return self.skip_entry(path, Skip::Unreadable(error)); + } + }; + if self.confinement.is_state(&stat) { + return None; + } + let mut entry = Entry { + path, + depth, + parent, + name, + canonical, + stat, + via_link: false, + selected: false, + }; + if stat.kind == FileKind::Symlink + && self.links == LinkMode::Follow + && let Err(skip) = self.follow(&parent_canonical, &mut entry) + { + return self.skip_entry(entry.path, skip); + } + let is_dir = entry.stat.is_dir(); + if self.ignored(&entry.path, is_dir) || self.filter.exclude.matches(&entry.path, is_dir) { + return None; + } + entry.selected = self.filter.selects(&entry.path, is_dir); + if is_dir { + if entry.via_link && self.identities.contains(&(entry.stat.dev, entry.stat.ino)) { + return Some(Event::Skip { + reason: Skip::LinkOutsideRoot(ProtocolError::failed_precondition( + PreconditionReason::SymlinkLoop, + format!("{}: the link closes a directory cycle", entry.path), + )), + path: Some(entry.path), + selected: true, + }); + } + match entry.parent.child(&entry.name) { + Ok(directory) => { + let identity = (entry.stat.dev, entry.stat.ino); + self.identities.insert(identity); + self.stack.push(Frame { + directory: Arc::new(directory), + canonical: entry.canonical.clone(), + path: entry.path.clone(), + depth, + identity, + names: None, + ignore_levels: 0, + }); + } + Err(error) => { + return Some(Event::Skip { + reason: Skip::Unreadable(io_error(error, &entry.path)), + path: Some(entry.path), + selected: true, + }); + } + } + } + Some(Event::Entry(entry)) + } + + /// A skip of an entry whose kind is unknown (it vanished, or a link + /// that could not be followed), taken as a non-directory: pruned like + /// one when ignored or excluded, selected like one otherwise. + fn skip_entry(&self, path: TreePath, reason: Skip) -> Option { + if self.ignored(&path, false) || self.filter.exclude.matches(&path, false) { + return None; + } + Some(Event::Skip { + selected: self.filter.selects(&path, false), + path: Some(path), + reason, + }) + } + + /// Replace a link entry by its in-root target. + fn follow(&self, parent: &[OsString], entry: &mut Entry) -> Result<(), Skip> { + let resolved = self + .confinement + .lookup_link(parent, &entry.name, &entry.path) + .map_err(|error| match &error.data { + environment_protocol::error::ErrorData::PermissionDenied { + reason: Some(PermissionReason::OutsideRoot | PermissionReason::DaemonState), + } + | environment_protocol::error::ErrorData::FailedPrecondition { + reason: PreconditionReason::SymlinkLoop, + } => Skip::LinkOutsideRoot(error), + environment_protocol::error::ErrorData::NotFound { .. } => Skip::DanglingLink, + _ => Skip::Unreadable(error), + })?; + let stat = resolved.stat.expect("a followed lookup reports its target"); + entry.parent = Arc::new(resolved.parent); + entry.name = resolved.name; + entry.canonical = resolved.segments; + entry.stat = stat; + entry.via_link = true; + Ok(()) + } + + fn ignored(&self, path: &TreePath, is_dir: bool) -> bool { + if self.filter.include_ignored { + return false; + } + let candidate = self.base.join(segments_path_tree(path)); + for level in self.ignores.iter().rev() { + let matched = level.matcher.matched(&candidate, is_dir); + if matched.is_ignore() { + return true; + } + if matched.is_whitelist() { + return false; + } + } + false + } + + fn load_ancestor_ignores(&mut self, root: &[OsString]) { + for depth in 0..root.len() { + let segments = &root[..depth]; + let Ok(resolved) = self.confinement.reopen(segments, &"ignore file ancestor") else { + continue; + }; + let Ok(directory) = resolved.open_directory() else { + continue; + }; + self.load_ignores(&directory, segments_path(segments)); + } + } + + /// Push the ignore level of one directory; returns how many levels were + /// pushed (0 or 1). + fn load_ignores(&mut self, directory: &Directory, at: PathBuf) -> usize { + let mut builder = GitignoreBuilder::new(&at); + let mut lines = 0usize; + let mut add = |content: Vec| { + for line in String::from_utf8_lossy(&content).lines() { + if builder.add_line(None, line).is_ok() { + lines += 1; + } + } + }; + if let Ok(git) = directory.child(OsStr::new(".git")) + && let Ok(info) = git.child(OsStr::new("info")) + && let Some(content) = read_small(&info, "exclude") + { + add(content); + } + for name in [".gitignore", ".ignore"] { + if let Some(content) = read_small(directory, name) { + add(content); + } + } + if lines == 0 { + return 0; + } + match builder.build() { + Ok(matcher) => { + self.ignores.push(IgnoreLevel { matcher }); + 1 + } + Err(_) => 0, + } + } +} + +fn read_small(directory: &Directory, name: &str) -> Option> { + let mut file = directory.open_read(OsStr::new(name)).ok()?; + if !EntryStat::of_file(&file).ok()?.is_file() { + return None; + } + os::read_up_to(&mut file, MAX_IGNORE_FILE_BYTES).ok() +} + +/// `/` joined with segments: the coordinate space of ignore matching. +fn segments_path(segments: &[OsString]) -> PathBuf { + let mut path = PathBuf::from("/"); + path.extend(segments); + path +} + +fn segments_path_tree(path: &TreePath) -> PathBuf { + path.segments().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filesystem::confinement::Lookup; + use environment_protocol::paths::EnvironmentPath; + + fn set(patterns: &[&str]) -> PatternSet { + PatternSet::compile( + &patterns.iter().map(|p| (*p).to_owned()).collect::>(), + false, + "filter.include", + ) + .unwrap() + } + + fn tree(path: &str) -> TreePath { + TreePath::new(path).unwrap() + } + + #[test] + fn patterns_follow_gitignore_anchoring() { + let star = set(&["src/*.rs"]); + assert!(star.matches(&tree("src/lib.rs"), false)); + assert!(!star.matches(&tree("src/deep/nested.rs"), false)); + + let name = set(&["*.md"]); + assert!(name.matches(&tree("readme.md"), false)); + assert!(name.matches(&tree("docs/deep/guide.md"), false)); + + let anchored = set(&["/build"]); + assert!(anchored.matches(&tree("build"), true)); + assert!(!anchored.matches(&tree("sub/build"), true)); + + let spanning = set(&["**/SKILL.md"]); + assert!(spanning.matches(&tree("SKILL.md"), false)); + assert!(spanning.matches(&tree("a/b/SKILL.md"), false)); + + let directories = set(&["target/"]); + assert!(directories.matches(&tree("crates/target"), true)); + assert!(!directories.matches(&tree("crates/target"), false)); + + let alternation = set(&["*.{rs,toml}"]); + assert!(alternation.matches(&tree("Cargo.toml"), false)); + + assert!(PatternSet::compile(&["".into()], false, "filter.include").is_err()); + assert!(PatternSet::compile(&["[".into()], false, "filter.include").is_err()); + } + + fn walk(root: &std::path::Path, filter: TreeFilter, links: LinkMode) -> Vec { + let confinement = Arc::new(Confinement::new(root.into(), root.join(".state")).unwrap()); + let resolved = confinement + .lookup( + &EnvironmentPath::new(root.to_str().unwrap()).unwrap(), + Lookup::FOLLOW, + ) + .unwrap(); + let mut walker = Walker::new( + confinement, + &resolved, + CompiledFilter::compile(&filter, "filter").unwrap(), + links, + None, + &"root", + ) + .unwrap(); + let mut seen = Vec::new(); + while let Some(event) = walker.next_event() { + match event { + Event::Entry(entry) if entry.selected => seen.push(entry.path.to_string()), + Event::Entry(_) | Event::Yield => {} + Event::Skip { path, .. } => seen.push(format!( + "skip:{}", + path.map(|path| path.to_string()).unwrap_or_default() + )), + } + } + seen + } + + #[test] + fn walks_prune_vcs_state_hidden_and_ignored_entries_by_default() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + for directory in [ + ".git/objects", + "src", + "node_modules/pkg", + ".hidden", + ".state", + ] { + std::fs::create_dir_all(root.join(directory)).unwrap(); + } + std::fs::write(root.join(".git/objects/pack"), "pack").unwrap(); + std::fs::write(root.join(".gitignore"), "node_modules/\n*.log\n!keep.log\n").unwrap(); + std::fs::write(root.join("src/lib.rs"), "lib").unwrap(); + std::fs::write(root.join("src/debug.log"), "log").unwrap(); + std::fs::write(root.join("src/keep.log"), "log").unwrap(); + std::fs::write(root.join("node_modules/pkg/index.js"), "js").unwrap(); + std::fs::write(root.join(".hidden/file"), "hidden").unwrap(); + std::fs::write(root.join(".state/daemon-key"), "key").unwrap(); + + assert_eq!( + walk(&root, TreeFilter::default(), LinkMode::Report), + ["src", "src/keep.log", "src/lib.rs"] + ); + let everything = TreeFilter { + include_hidden: true, + include_ignored: true, + ..TreeFilter::default() + }; + // VCS metadata and the state directory stay pruned. + assert_eq!( + walk(&root, everything, LinkMode::Report), + [ + ".gitignore", + ".hidden", + ".hidden/file", + "node_modules", + "node_modules/pkg", + "node_modules/pkg/index.js", + "src", + "src/debug.log", + "src/keep.log", + "src/lib.rs", + ] + ); + } + + #[test] + fn followed_links_stay_inside_the_root_and_cycles_are_skipped() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("root"); + std::fs::create_dir_all(root.join("real")).unwrap(); + std::fs::create_dir_all(base.join("outside")).unwrap(); + std::fs::write(root.join("real/file"), "x").unwrap(); + symlink("real", root.join("alias")).unwrap(); + symlink("..", root.join("real/up")).unwrap(); + symlink(base.join("outside"), root.join("escape")).unwrap(); + + let filter = TreeFilter { + include_hidden: true, + ..TreeFilter::default() + }; + assert_eq!( + walk(&root, filter, LinkMode::Follow), + [ + "alias", + "alias/file", + "skip:alias/up", + "skip:escape", + "real", + "real/file", + "skip:real/up", + ] + ); + } +} diff --git a/crates/environment-daemon/src/identity.rs b/crates/environment-daemon/src/identity.rs index bfe1ed4e5..8fa9d2d1c 100644 --- a/crates/environment-daemon/src/identity.rs +++ b/crates/environment-daemon/src/identity.rs @@ -10,7 +10,13 @@ use std::path::{Path, PathBuf}; use anyhow::{Context as _, Result, bail}; use ed25519_dalek::{Signer as _, SigningKey, VerifyingKey}; -use environment_protocol::registration::{decode_hex, encode_hex, signed_registration_message}; +use environment_protocol::{ + bytes::{decode_hex_array, encode_hex}, + registration::{ + ChallengeNonce, DaemonPublicKey, Ed25519Signature, signed_registration_message, + }, + routes::CanonicalOrigin, +}; /// Name of the key file inside the daemon state directory. pub const DAEMON_KEY_FILE: &str = "daemon-key"; @@ -24,7 +30,7 @@ pub struct DaemonIdentity { impl std::fmt::Debug for DaemonIdentity { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DaemonIdentity") - .field("public_key", &self.public_key_hex()) + .field("public_key", &self.public_key()) .field("path", &self.path) .finish() } @@ -32,60 +38,73 @@ impl std::fmt::Debug for DaemonIdentity { impl DaemonIdentity { /// Load the key from `/daemon-key`, or generate one there. - /// The directory is created `0700` and the file `0600`; generation is - /// atomic so a crash cannot leave a half-written key behind. + /// The file is created `0600` and installed without ever replacing an + /// existing key: when two first starts race, the key installed first + /// wins and both processes load it. pub fn load_or_create(state_dir: &Path) -> Result { let path = state_dir.join(DAEMON_KEY_FILE); - if path.exists() { - let contents = std::fs::read_to_string(&path) - .with_context(|| format!("read daemon key {}", path.display()))?; - let seed = decode_hex(contents.trim()) - .filter(|seed| seed.len() == 32) - .with_context(|| { - format!("daemon key {} is not a 32-byte hex seed", path.display()) - })?; - let seed: [u8; 32] = seed.try_into().expect("length checked"); - return Ok(Self { - signing_key: SigningKey::from_bytes(&seed), - path, - }); + if let Some(identity) = Self::load(&path)? { + return Ok(identity); } - std::fs::create_dir_all(state_dir) - .with_context(|| format!("create state dir {}", state_dir.display()))?; - restrict_dir(state_dir)?; let mut seed = [0u8; 32]; rand::TryRng::try_fill_bytes(&mut rand::rngs::SysRng, &mut seed) .context("generate daemon identity seed")?; let signing_key = SigningKey::from_bytes(&seed); - let temp = state_dir.join(format!(".{DAEMON_KEY_FILE}.{}", std::process::id())); + let mut suffix = [0u8; 8]; + rand::TryRng::try_fill_bytes(&mut rand::rngs::SysRng, &mut suffix) + .context("generate temporary key name")?; + let temp = state_dir.join(format!(".{DAEMON_KEY_FILE}.{}", encode_hex(&suffix))); write_private(&temp, &encode_hex(&signing_key.to_bytes()))?; - if let Err(error) = std::fs::rename(&temp, &path) { - let _ = std::fs::remove_file(&temp); - if path.exists() { - // Lost a race with a concurrent start of the same daemon: - // the other key wins so both processes share one identity. - return Self::load_or_create(state_dir); + // A hard link fails when the destination exists, unlike rename, + // which would silently replace a key a concurrent start installed. + let installed = std::fs::hard_link(&temp, &path); + let _ = std::fs::remove_file(&temp); + match installed { + Ok(()) => { + sync_dir(state_dir); + Ok(Self { signing_key, path }) } - bail!("install daemon key {}: {error}", path.display()); + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Self::load(&path)? + .with_context(|| format!("daemon key {} vanished during install", path.display())), + Err(error) => bail!("install daemon key {}: {error}", path.display()), } - Ok(Self { signing_key, path }) } - /// Lowercase hex of the raw 32-byte public key. - pub fn public_key_hex(&self) -> String { - encode_hex(self.signing_key.verifying_key().as_bytes()) + fn load(path: &Path) -> Result> { + let contents = match std::fs::read_to_string(path) { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| format!("read daemon key {}", path.display())); + } + }; + let seed = decode_hex_array::<32>(contents.trim()) + .with_context(|| format!("daemon key {} is not a 32-byte hex seed", path.display()))?; + Ok(Some(Self { + signing_key: SigningKey::from_bytes(&seed), + path: path.to_path_buf(), + })) + } + + /// `daemonPublicKey`. + pub fn public_key(&self) -> DaemonPublicKey { + DaemonPublicKey::from_bytes(self.signing_key.verifying_key().to_bytes()) } pub fn verifying_key(&self) -> VerifyingKey { self.signing_key.verifying_key() } - /// Hex signature over the domain-separated challenge. - pub fn sign_challenge(&self, nonce: &[u8]) -> String { - encode_hex( - &self - .signing_key - .sign(&signed_registration_message(nonce)) + /// `signature` over the challenge nonce bound to the canonical origin of + /// the gateway URL this daemon dialed. + pub fn sign_challenge( + &self, + nonce: &ChallengeNonce, + origin: &CanonicalOrigin, + ) -> Ed25519Signature { + Ed25519Signature::from_bytes( + self.signing_key + .sign(&signed_registration_message(nonce, origin)) .to_bytes(), ) } @@ -113,30 +132,27 @@ fn write_private(path: &Path, contents: &str) -> Result<()> { Ok(()) } -fn restrict_dir(path: &Path) -> Result<()> { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) - .with_context(|| format!("restrict state dir {}", path.display()))?; +/// Make the new directory entry durable; best effort. +fn sync_dir(path: &Path) { + if let Ok(directory) = std::fs::File::open(path) { + let _ = directory.sync_all(); } - Ok(()) } #[cfg(test)] mod tests { use ed25519_dalek::Verifier as _; + use environment_protocol::routes::canonical_origin; use super::*; #[test] fn identity_is_generated_once_and_reloaded_from_the_state_dir() { let temp = tempfile::tempdir().expect("tempdir"); - let state = temp.path().join("state"); - let first = DaemonIdentity::load_or_create(&state).expect("create"); - let second = DaemonIdentity::load_or_create(&state).expect("reload"); - assert_eq!(first.public_key_hex(), second.public_key_hex()); - assert_eq!(first.public_key_hex().len(), 64); + let state = temp.path(); + let first = DaemonIdentity::load_or_create(state).expect("create"); + let second = DaemonIdentity::load_or_create(state).expect("reload"); + assert_eq!(first.public_key(), second.public_key()); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt as _; @@ -146,37 +162,66 @@ mod tests { .mode() & 0o777; assert_eq!(mode, 0o600); - let dir_mode = std::fs::metadata(&state) - .expect("meta") - .permissions() - .mode() - & 0o777; - assert_eq!(dir_mode, 0o700); } + assert_eq!( + std::fs::read_dir(state).unwrap().count(), + 1, + "no temporary file is left behind" + ); std::fs::remove_file(first.path()).expect("reset identity"); - let reset = DaemonIdentity::load_or_create(&state).expect("regenerate"); - assert_ne!(reset.public_key_hex(), first.public_key_hex()); + let reset = DaemonIdentity::load_or_create(state).expect("regenerate"); + assert_ne!(reset.public_key(), first.public_key()); + } + + #[test] + fn concurrent_first_starts_share_the_key_installed_first() { + let temp = tempfile::tempdir().expect("tempdir"); + let state = temp.path().to_path_buf(); + let handles: Vec<_> = (0..8) + .map(|_| { + let state = state.clone(); + std::thread::spawn(move || DaemonIdentity::load_or_create(&state).unwrap()) + }) + .collect(); + let keys: std::collections::BTreeSet = handles + .into_iter() + .map(|handle| handle.join().unwrap().public_key().to_hex()) + .collect(); + assert_eq!(keys.len(), 1, "every racing start holds the installed key"); + let on_disk = DaemonIdentity::load_or_create(&state).unwrap(); + assert_eq!( + keys.into_iter().next().unwrap(), + on_disk.public_key().to_hex() + ); } #[test] - fn challenge_signatures_verify_against_the_public_key_and_domain() { + fn challenge_signatures_bind_the_nonce_and_the_dialed_origin() { let temp = tempfile::tempdir().expect("tempdir"); let identity = DaemonIdentity::load_or_create(temp.path()).expect("create"); - let nonce = [7u8; 32]; - let signature = decode_hex(&identity.sign_challenge(&nonce)).expect("hex"); - let signature = ed25519_dalek::Signature::from_slice(&signature).expect("signature"); - identity - .verifying_key() - .verify(&signed_registration_message(&nonce), &signature) + let nonce = ChallengeNonce::from_bytes([7u8; 32]); + let origin = canonical_origin("wss://gateway.example/environment-gateway/connect").unwrap(); + let signature = identity.sign_challenge(&nonce, &origin); + let signature = ed25519_dalek::Signature::from_bytes(signature.as_bytes()); + let verifying = identity.verifying_key(); + verifying + .verify_strict(&signed_registration_message(&nonce, &origin), &signature) .expect("verifies"); + assert!(verifying.verify(nonce.as_bytes(), &signature).is_err()); + let elsewhere = canonical_origin("wss://other.example").unwrap(); assert!( - identity.verifying_key().verify(&nonce, &signature).is_err(), - "signature must not verify without the domain separator" + verifying + .verify(&signed_registration_message(&nonce, &elsewhere), &signature) + .is_err(), + "a signature for one gateway does not verify at another" ); + let other_nonce = ChallengeNonce::from_bytes([8u8; 32]); assert!( - identity - .verifying_key() - .verify(&signed_registration_message(&[8u8; 32]), &signature) + verifying + .verify( + &signed_registration_message(&other_nonce, &origin), + &signature + ) .is_err() ); } @@ -184,7 +229,9 @@ mod tests { #[test] fn corrupt_key_files_are_rejected_rather_than_replaced() { let temp = tempfile::tempdir().expect("tempdir"); - std::fs::write(temp.path().join(DAEMON_KEY_FILE), "not-hex").expect("write"); + let path = temp.path().join(DAEMON_KEY_FILE); + std::fs::write(&path, "not-hex").expect("write"); assert!(DaemonIdentity::load_or_create(temp.path()).is_err()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "not-hex"); } } diff --git a/crates/environment-daemon/src/jobs.rs b/crates/environment-daemon/src/jobs.rs index 19fdd3553..b34b1d28e 100644 --- a/crates/environment-daemon/src/jobs.rs +++ b/crates/environment-daemon/src/jobs.rs @@ -1,2051 +1,2082 @@ +//! `job/*`: durable batch commands with dependencies and queues, on the +//! shared execution primitive. +//! +//! A job is identified by its `jobId` together with the `requestId` of the +//! `job/start` that created it and a hash of its spec: an identical retry +//! replays the job, any other reuse of the id is `jobIdInUse`. A job runs +//! once its dependencies satisfy its policy and its queue key is free. When +//! its root exits the daemon sweeps its process group (`SIGTERM`, the +//! cancel grace, `SIGKILL`) and reports what it swept. +//! +//! Records and output persist under `/jobs/` through the job store, +//! off the runtime thread; output keeps a head and a tail with the middle +//! as an overflow gap. Startup loads summaries only, quarantines unreadable +//! records instead of failing, marks unfinished jobs `interrupted` and kills +//! the groups an earlier instance left running in this boot. + use std::{ - collections::{BTreeMap, BTreeSet}, - path::{Component, PathBuf}, - process::Stdio, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::{Duration, SystemTime, UNIX_EPOCH}, + collections::{BTreeMap, BTreeSet, VecDeque}, + path::{Path, PathBuf}, + sync::{Arc, Mutex, MutexGuard}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use environment_protocol::{ - data::jobs::{ - CancelJobsParams, CancelJobsResponse, JobCancelScope, JobDependencyPolicy, JobOutputChunk, - JobOutputStream, JobReadResult, JobStartSpec, JobStatus, JobSummary, ListJobsParams, - ListJobsResponse, ReadJobsParams, ReadJobsResponse, StartJobsParams, StartJobsResponse, + bytes::encode_hex, + data::{ + exec::{ + CommandSpec, ExecInput, ExitStatus, LeftoverDisposition, LeftoverReport, OutputPage, + OutputStream, ReturnOn, StoppedBy, + }, + handshake::JobLimits, + job::{ + CancelJobsParams, CancelJobsResponse, JobCancelScope, JobDependency, + JobDependencyPolicy, JobReadEntry, JobReadRequest, JobStatus, JobSummary, + JobSummaryEntry, ReadJobsParams, ReadJobsResponse, StartJobsParams, StartJobsResponse, + }, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::{ByteChunk, EnvironmentPath, JobId, SecretString}, + error::{PreconditionReason, ProtocolError}, + ids::{DaemonInstanceId, JobId, JobName, JobRequestId, QueueKey}, + limits::MAX_PAGE_CHUNKS, + paths::EnvironmentPath, }; use serde::{Deserialize, Serialize}; -use tokio::{ - io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, - process::Command, - sync::{Mutex, Notify}, - task::JoinHandle, -}; +use sha2::{Digest as _, Sha256}; +use tokio::sync::watch; use crate::{ + dispatch::RequestContext, + exec::{ + AfterExit, ExecOptions, Execution, OutputSink, PreparedCommand, + job_store::{JobStore, load_output}, + log::OutputLog, + outcome::describe_exit, + spawn_record::{SpawnRecord, write_json_atomic}, + validate_command, + }, + filesystem::Confinement, process_group, - redaction::{redact_bytes, redactions_for_secret_env}, }; +/// Version of the persisted job record; a record of another version is +/// quarantined. +const JOB_RECORD_FORMAT: u32 = 1; + +/// Terminal jobs whose output stays in memory after completion or a load. +const MAX_CACHED_OUTPUTS: usize = 16; + +/// Everything a [`JobManager`] needs from the daemon. +pub struct JobConfig { + /// Canonical default `cwd`. + pub cwd: PathBuf, + /// Resolves every `cwd` when the job starts running. + pub confinement: Arc, + /// Records and output live in `/jobs`. + pub state_dir: PathBuf, + /// Daemon variables no child inherits. + pub scrubbed_env: Vec, + pub instance_id: DaemonInstanceId, + pub limits: JobLimits, + /// `limits.maxWaitMs`, the `waitMs` ceiling. + pub max_wait_ms: u64, +} + +/// The daemon's jobs. Cheap to clone. #[derive(Clone)] pub struct JobManager { - cwd: PathBuf, - fs_root: PathBuf, - jobs_root: PathBuf, - state: Arc>, - notify: Arc, - /// Daemon configuration variables no job may inherit. - scrubbed_env: Arc>, + inner: Arc, +} + +struct JobsInner { + config: JobConfig, + store: JobStore, + state: Mutex, + /// Bumped on every status change and every output append. + changed: watch::Sender, } #[derive(Default)] -struct JobManagerState { - accepting: bool, - jobs: BTreeMap, - running: BTreeMap>, - secret_envs: BTreeMap>, - next_accept_seq: u64, +struct JobsState { + jobs: BTreeMap, + next_seq: u64, + /// Terminal jobs holding their output in memory, oldest first. + cached: VecDeque, } -type JobKey = (String, JobId); +struct JobEntry { + record: StoredJob, + /// The command of a queued job, secrets included. Memory only. + pending: Option, + run: Option, + /// The output of a terminal job, while cached. + output: Option>, +} -struct RunningJob { - cancel: Notify, - interrupted: AtomicBool, +struct Run { + execution: Execution, + started: Instant, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -struct JobRecord { - namespace: String, +/// A running job's output: retained head and tail in memory, every byte +/// forwarded to the store. +struct JobOutput { job_id: JobId, - name: Option, - argv: Vec, - cwd: Option, - env: BTreeMap, - #[serde(default)] - secret_env_names: Vec, - stdin: Option, - timeout_ms: Option, - dependency_policy: JobDependencyPolicy, - dependencies: Vec, - queue_key: Option, + log: OutputLog, + cap: u64, + store: JobStore, + changed: watch::Sender, +} + +impl OutputSink for JobOutput { + fn append(&mut self, stream: OutputStream, data: &[u8]) { + self.store + .append(&self.job_id, self.log.end(), stream, data); + self.log.append(stream, data); + self.log.enforce_cap(self.cap, 0); + self.changed + .send_modify(|version| *version = version.wrapping_add(1)); + } +} + +/// The persisted record of one job. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct StoredJob { + format: u32, + job_id: JobId, + request_id: JobRequestId, + #[serde(default, skip_serializing_if = "Option::is_none")] + name: Option, spec_hash: String, - #[serde(default)] - accept_seq: u64, + seq: u64, status: JobStatus, - created_at_ms: u64, - queued_at_ms: Option, - started_at_ms: Option, - finished_at_ms: Option, - exit_code: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + depends_on: Vec, + dependency_policy: JobDependencyPolicy, + #[serde(default, skip_serializing_if = "Option::is_none")] + queue_key: Option, + created_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + queued_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + started_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + finished_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cancel_requested_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + exit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + stopped_by: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + leftovers: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + error: Option, + /// The running job's process group, for restart reconciliation. + #[serde(default, skip_serializing_if = "Option::is_none")] + spawn: Option, + /// Output bytes produced, final once the job is terminal. #[serde(default)] - orphaned_descendants: bool, - failure: Option, - output_chunks: Vec, - output_next_seq: u64, + output_end_bytes: u64, } -#[derive(Clone)] -struct ResolvedJob { - spec: JobStartSpec, - dependencies: Vec, - secret_env: BTreeMap, - spec_hash: String, -} +impl StoredJob { + fn summary(&self) -> JobSummary { + JobSummary { + job_id: self.job_id.clone(), + request_id: self.request_id.clone(), + name: self.name.clone(), + status: self.status, + depends_on: self.depends_on.clone(), + queue_key: self.queue_key.clone(), + created_at_ms: self.created_at_ms, + queued_at_ms: self.queued_at_ms, + started_at_ms: self.started_at_ms, + finished_at_ms: self.finished_at_ms, + cancel_requested_at_ms: self.cancel_requested_at_ms, + duration_ms: self.duration_ms, + exit: self.exit.clone(), + stopped_by: self.stopped_by, + leftovers: self.leftovers.clone(), + error: self.error.clone(), + } + } -#[derive(Clone, Copy)] -enum FinishKind { - Exit, - Cancelled, - Interrupted, - TimedOut, + fn encode(&self) -> Vec { + serde_json::to_vec(self).expect("job records encode") + } } -struct SpawnOutcome { - finish: FinishKind, - exit_code: Option, - orphaned_descendants: bool, +/// A validated `job/start` entry with its dependencies resolved. +struct ResolvedJob { + job_id: JobId, + name: Option, + command: CommandSpec, + depends_on: Vec, + dependency_policy: JobDependencyPolicy, + queue_key: Option, + spec_hash: String, } impl JobManager { - pub fn new(cwd: PathBuf, fs_root: PathBuf, state_dir: PathBuf) -> anyhow::Result { - let cwd = normalize_path(cwd); - let fs_root = normalize_path(fs_root); - let jobs_root = normalize_path(state_dir).join("jobs"); - std::fs::create_dir_all(&jobs_root)?; - let mut state = JobManagerState { - accepting: true, - ..JobManagerState::default() - }; + /// Loads job summaries and reconciles what an earlier instance left. + /// Never fails on a record: unreadable ones are quarantined. + pub fn new(config: JobConfig) -> anyhow::Result { + let jobs_dir = config.state_dir.join("jobs"); + std::fs::create_dir_all(&jobs_dir)?; + let quarantine_dir = config.state_dir.join("jobs-quarantine"); + let boot = process_group::boot_id(); let now = now_ms(); - - for entry in std::fs::read_dir(&jobs_root)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|extension| extension.to_str()) != Some("json") { - continue; - } - let bytes = std::fs::read(&path)?; - let mut record: JobRecord = serde_json::from_slice(&bytes)?; + let mut state = JobsState::default(); + for entry in std::fs::read_dir(&jobs_dir)? { + let path = entry?.path(); + let mut record = match load_record(&path) { + Ok(record) => record, + Err(reason) => { + quarantine(&path, &quarantine_dir, &reason); + continue; + } + }; if !record.status.is_terminal() { + if record.status == JobStatus::Running + && let Some(spawn) = &record.spawn + && spawn.same_boot(boot.as_deref()) + { + spawn.group().signal(process_group::GroupSignal::Kill); + } + record.error = Some(ProtocolError::aborted( + None, + format!( + "the daemon stopped while the job was {}", + status_name(record.status) + ), + )); record.status = JobStatus::Interrupted; record.finished_at_ms = Some(now); - record.failure = - Some("lightspeed-envd restarted before this job could be recovered".to_owned()); - persist_record_at(&jobs_root, &record)?; + // The last record predates the output the job produced. + let cap = config.limits.max_retained_output_bytes; + if let Ok(log) = load_output(&path, record.output_end_bytes, cap) { + record.output_end_bytes = log.end(); + } + if let Err(error) = write_json_atomic(&path.join("record.json"), &record) { + eprintln!( + "lightspeed-envd: could not persist interrupted job {}: {error}", + record.job_id + ); + } } - state.next_accept_seq = state - .next_accept_seq - .max(record.accept_seq.saturating_add(1)); - state - .jobs - .insert(job_key(&record.namespace, &record.job_id), record); + state.next_seq = state.next_seq.max(record.seq.saturating_add(1)); + state.jobs.insert( + record.job_id.clone(), + JobEntry { + record, + pending: None, + run: None, + output: None, + }, + ); } - + let cap = config.limits.max_retained_output_bytes; + let store = JobStore::start(jobs_dir, cap / 2, cap - cap / 2)?; Ok(Self { - cwd, - fs_root, - jobs_root, - state: Arc::new(Mutex::new(state)), - notify: Arc::new(Notify::new()), - scrubbed_env: Arc::new(Vec::new()), + inner: Arc::new(JobsInner { + config, + store, + state: Mutex::new(state), + changed: watch::Sender::new(0), + }), }) } - pub fn with_scrubbed_env(mut self, names: Vec) -> Self { - self.scrubbed_env = Arc::new(names); - self - } - - /// Jobs currently executing (running or cancelling). + /// Jobs that are `running`. pub async fn running_count(&self) -> u32 { - u32::try_from(self.state.lock().await.running.len()).unwrap_or(u32::MAX) + self.maintain(); + let running = self + .state() + .jobs + .values() + .filter(|entry| entry.record.status == JobStatus::Running) + .count(); + u32::try_from(running).unwrap_or(u32::MAX) } - pub async fn start_jobs( + pub async fn start( &self, + _ctx: &RequestContext, params: StartJobsParams, - ) -> Result { - let now = now_ms(); - let resolved = { - let state = self.state.lock().await; - require_accepting(&state)?; - validate_and_resolve_start(&state, ¶ms)? - }; - let mut accepted = Vec::new(); - { - let mut state = self.state.lock().await; - require_accepting(&state)?; - for job in &resolved { - let key = job_key(¶ms.namespace, &job.spec.job_id); - if state.jobs.contains_key(&key) { - state - .secret_envs - .entry(key.clone()) - .or_insert_with(|| job.secret_env.clone()); - accepted.push(key); - continue; + ) -> Result { + let resolved = self.resolve_start(¶ms)?; + self.maintain(); + let records = { + let mut state = self.state(); + if let Some(replay) = replay_or_conflict(&state, ¶ms.request_id, &resolved)? { + return Ok(StartJobsResponse { jobs: replay }); + } + for (index, job) in resolved.iter().enumerate() { + for dependency in &job.depends_on { + let known = state.jobs.contains_key(dependency) + || resolved.iter().any(|other| &other.job_id == dependency); + if !known { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].dependsOn"), + format!("unknown job id {dependency}"), + )); + } } - let accept_seq = state.next_accept_seq; - state.next_accept_seq = state.next_accept_seq.saturating_add(1); - - let record = JobRecord { - namespace: params.namespace.clone(), - job_id: job.spec.job_id.clone(), - name: job.spec.name.clone(), - argv: job.spec.argv.clone(), - cwd: job.spec.cwd.clone(), - env: job.spec.env.clone(), - secret_env_names: job.secret_env.keys().cloned().collect(), - stdin: job.spec.stdin.clone(), - timeout_ms: job.spec.timeout_ms, - dependency_policy: job.spec.dependency_policy, - dependencies: job.dependencies.clone(), - queue_key: job.spec.queue_key.clone(), - spec_hash: job.spec_hash.clone(), - accept_seq, + } + let now = now_ms(); + let mut started = Vec::new(); + let mut written = Vec::new(); + for job in resolved { + let seq = state.next_seq; + state.next_seq += 1; + let record = StoredJob { + format: JOB_RECORD_FORMAT, + job_id: job.job_id.clone(), + request_id: params.request_id.clone(), + name: job.name, + spec_hash: job.spec_hash, + seq, status: JobStatus::Queued, + depends_on: job.depends_on, + dependency_policy: job.dependency_policy, + queue_key: job.queue_key, created_at_ms: now, - queued_at_ms: Some(now), + queued_at_ms: None, started_at_ms: None, finished_at_ms: None, - exit_code: None, - orphaned_descendants: false, - failure: None, - output_chunks: Vec::new(), - output_next_seq: 0, + cancel_requested_at_ms: None, + duration_ms: None, + exit: None, + stopped_by: None, + leftovers: None, + error: None, + spawn: None, + output_end_bytes: 0, }; - self.persist_record(&record)?; - accepted.push(key.clone()); - state - .secret_envs - .insert(key.clone(), job.secret_env.clone()); - state.jobs.insert(key, record); + written.push(self.inner.store.write_record(&job.job_id, record.encode())); + started.push(job.job_id.clone()); + state.jobs.insert( + job.job_id, + JobEntry { + record, + pending: Some(job.command), + run: None, + output: None, + }, + ); } + (started, written) + }; + // The writes were queued under the lock, so the scheduler's later + // writes land behind them; the start is answered once they are on + // disk. + let (started, written) = records; + for written in written { + let _ = written.await; } - - self.schedule_ready_jobs().await; - - let state = self.state.lock().await; + self.schedule(); + let state = self.state(); Ok(StartJobsResponse { - jobs: accepted - .into_iter() - .map(|key| { - state - .jobs - .get(&key) - .map(JobRecord::summary) - .unwrap_or_else(|| lost_summary(&key.0, &key.1)) - }) + jobs: started + .iter() + .filter_map(|job_id| state.jobs.get(job_id)) + .map(|entry| entry.record.summary()) .collect(), }) } - pub async fn read_jobs( + pub async fn read( &self, + ctx: &RequestContext, params: ReadJobsParams, - ) -> Result { - validate_id_component("namespace", ¶ms.namespace)?; - if params.jobs.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "job/read requires at least one job id", + ) -> Result { + let limits = &self.inner.config.limits; + check_job_count("jobs", params.jobs.len(), limits.max_request_jobs)?; + if params.max_bytes > limits.max_read_bytes { + return Err(ProtocolError::above_ceiling( + "maxBytes", + params.max_bytes, + limits.max_read_bytes, )); } + if params.wait_ms > self.inner.config.max_wait_ms { + return Err(ProtocolError::above_ceiling( + "waitMs", + params.wait_ms, + self.inner.config.max_wait_ms, + )); + } + for (index, entry) in params.jobs.iter().enumerate() { + if entry.after.is_some() && entry.tail_bytes.is_some() { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].tailBytes"), + "after and tailBytes exclude each other", + )); + } + if let Some(max_bytes) = entry.max_bytes + && max_bytes > params.max_bytes + { + return Err(ProtocolError::above_ceiling( + format!("jobs[{index}].maxBytes"), + max_bytes, + params.max_bytes, + )); + } + } + self.maintain(); + // The read pages from its own handles on terminal jobs' output: the + // shared cache may evict a log between loading and paging. + let mut logs: BTreeMap> = BTreeMap::new(); + for entry in params + .jobs + .iter() + .filter(|entry| needs_output(¶ms, entry)) + { + if !logs.contains_key(&entry.job_id) + && let Some(log) = self.ensure_output(&entry.job_id).await + { + logs.insert(entry.job_id.clone(), log); + } + } - let deadline = params - .wait_ms - .map(|wait_ms| tokio::time::Instant::now() + Duration::from_millis(wait_ms)); + let (starts, initial) = { + let state = self.state(); + let mut starts = Vec::new(); + let mut initial = Vec::new(); + for (index, entry) in params.jobs.iter().enumerate() { + let job = state.jobs.get(&entry.job_id); + let end = job.map_or(0, output_end); + let start = match (entry.after, entry.tail_bytes) { + (Some(after), _) => after, + (None, Some(tail)) => end.saturating_sub(tail), + (None, None) => 0, + }; + if job.is_some() && start > end { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].after"), + format!("after {start} is past the end of the output ({end})"), + )); + } + starts.push(start); + initial.push(job.map(|job| job.record.status)); + } + (starts, initial) + }; + let deadline = tokio::time::Instant::now() + Duration::from_millis(params.wait_ms); + let mut changes = self.inner.changed.subscribe(); loop { - let response = { - let state = self.state.lock().await; - let response = build_read_response(&state, ¶ms); - let all_terminal = response - .jobs - .iter() - .all(|result| result.summary.status.is_terminal()); - let should_wait = params.wait_ms.is_some() - && !all_terminal - && deadline.is_some_and(|deadline| tokio::time::Instant::now() < deadline); - if should_wait { None } else { Some(response) } - }; - if let Some(response) = response { - return Ok(response); + changes.borrow_and_update(); + if self.read_ready(¶ms, &starts, &initial) { + break; } - - let Some(deadline) = deadline else { - continue; - }; tokio::select! { - _ = self.notify.notified() => {} - _ = tokio::time::sleep_until(deadline) => {} + () = ctx.cancel.cancelled() => return Err(ProtocolError::cancelled()), + () = tokio::time::sleep_until(deadline) => break, + changed = changes.changed() => if changed.is_err() { break }, } } - } - pub async fn list_jobs( - &self, - params: ListJobsParams, - ) -> Result { - validate_id_component("namespace", ¶ms.namespace)?; - if matches!(params.limit, Some(0)) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "job/list limit must be greater than zero", - )); - } - let state = self.state.lock().await; - let mut jobs = state - .jobs - .values() - .filter(|record| record.namespace == params.namespace) - .map(JobRecord::summary) - .collect::>(); - jobs.sort_by(|left, right| { - right - .created_at_ms - .cmp(&left.created_at_ms) - .then_with(|| left.job_id.cmp(&right.job_id)) - }); - if let Some(limit) = params.limit { - jobs.truncate(limit); + // Jobs that finished during the wait: take their logs under the + // lock that pages them, loading any the cache no longer holds. + let mut attempted = BTreeSet::new(); + let state = loop { + let missing = match self.take_logs(¶ms, &mut logs, &mut attempted) { + Ok(state) => break state, + Err(missing) => missing, + }; + for job_id in missing { + if let Some(log) = self.ensure_output(&job_id).await { + logs.insert(job_id, log); + } + } + }; + let mut budget = params.max_bytes; + let mut items = MAX_PAGE_CHUNKS; + let mut jobs = Vec::with_capacity(params.jobs.len()); + for (entry, start) in params.jobs.iter().zip(starts) { + let Some(job) = state.jobs.get(&entry.job_id) else { + jobs.push(JobReadEntry::Error { + job_id: entry.job_id.clone(), + error: unknown_job(&entry.job_id), + }); + continue; + }; + let entry_budget = entry.max_bytes.map_or(budget, |max| max.min(budget)); + let page = |log: &OutputLog| (log.page(start, entry_budget, items), log.end()); + let output = match &job.run { + Some(_) => with_log(job, page), + None => logs.get(&entry.job_id).map(|log| page(log)), + }; + let output = match output { + Some((page, end)) => { + budget -= page.bytes; + items = items.saturating_sub(page.items()); + OutputPage { + chunks: page.chunks, + next: page.next, + end, + gaps: page.gaps, + } + } + None => OutputPage { + next: start, + end: output_end(job), + ..OutputPage::default() + }, + }; + jobs.push(JobReadEntry::Job { + summary: Box::new(job.record.summary()), + output, + }); } - Ok(ListJobsResponse { jobs }) + Ok(ReadJobsResponse { jobs }) } - pub async fn cancel_jobs( + pub async fn cancel( &self, + _ctx: &RequestContext, params: CancelJobsParams, - ) -> Result { - validate_id_component("namespace", ¶ms.namespace)?; - if params.jobs.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "job/cancel requires at least one job id", - )); - } - - let now = now_ms(); - let mut cancel_handles = Vec::new(); - let target_ids = { - let state = self.state.lock().await; - resolve_cancel_scope(&state, ¶ms) - }; - - { - let mut state = self.state.lock().await; - for key in &target_ids { - let Some(status) = state.jobs.get(key).map(|record| record.status) else { - continue; - }; - if status.is_terminal() { - continue; + ) -> Result { + let limits = &self.inner.config.limits; + check_job_count("jobIds", params.job_ids.len(), limits.max_request_jobs)?; + self.maintain(); + let response = { + let mut state = self.state(); + let mut named = Vec::new(); + for job_id in ¶ms.job_ids { + if !named.contains(job_id) { + named.push(job_id.clone()); } - let running = state.running.get(key).cloned(); - let Some(record) = state.jobs.get_mut(key) else { - continue; - }; - match status { - JobStatus::Running | JobStatus::CancelRequested => { - record.status = JobStatus::CancelRequested; - record.failure = Some(if params.force { - "job force cancellation requested".to_owned() - } else { - "job cancellation requested".to_owned() - }); - if let Some(running) = running { - cancel_handles.push(running); - } + } + let mut dependents = Vec::new(); + if params.scope == JobCancelScope::Dependents { + let mut affected: BTreeSet = named.iter().cloned().collect(); + loop { + let next: Vec = state + .jobs + .values() + .filter(|entry| { + !entry.record.status.is_terminal() + && !affected.contains(&entry.record.job_id) + && entry + .record + .depends_on + .iter() + .any(|dependency| affected.contains(dependency)) + }) + .map(|entry| entry.record.job_id.clone()) + .collect(); + if next.is_empty() { + break; } - _ => { - record.status = JobStatus::Cancelled; - record.finished_at_ms = Some(now); - record.failure = Some("job cancelled before start".to_owned()); + for job_id in next { + affected.insert(job_id.clone()); + dependents.push(job_id); } } - self.persist_record(record)?; } - } - - for running in cancel_handles { - running.cancel.notify_one(); - } - self.notify.notify_waiters(); - self.schedule_ready_jobs().await; - - let state = self.state.lock().await; - Ok(CancelJobsResponse { - jobs: target_ids + let now = now_ms(); + for job_id in named.iter().chain(&dependents) { + if let Some(entry) = state.jobs.get_mut(job_id) { + self.cancel_entry(entry, params.force, now); + } + } + let max_entries = limits.max_request_jobs as usize; + let mut jobs: Vec = named .iter() - .map(|key| { - state - .jobs - .get(key) - .map(JobRecord::summary) - .unwrap_or_else(|| lost_summary(&key.0, &key.1)) + .map(|job_id| match state.jobs.get(job_id) { + Some(entry) => JobSummaryEntry::Job { + summary: Box::new(entry.record.summary()), + }, + None => JobSummaryEntry::Error { + job_id: job_id.clone(), + error: unknown_job(job_id), + }, }) - .collect(), - }) + .collect(); + let room = max_entries.saturating_sub(jobs.len()); + let unlisted = dependents.len().saturating_sub(room); + jobs.extend(dependents.iter().take(room).filter_map(|job_id| { + state.jobs.get(job_id).map(|entry| JobSummaryEntry::Job { + summary: Box::new(entry.record.summary()), + }) + })); + CancelJobsResponse { + jobs, + unlisted_job_count: u32::try_from(unlisted).unwrap_or(u32::MAX), + } + }; + self.bump(); + self.schedule(); + Ok(response) + } + + /// Cancels one job: a queued job ends at once; a running one is + /// signalled and ends `cancelled` when its root exits. + fn cancel_entry(&self, entry: &mut JobEntry, force: bool, now: i64) { + match entry.record.status { + JobStatus::Queued => { + entry.pending = None; + entry.record.status = JobStatus::Cancelled; + entry.record.finished_at_ms = Some(now); + self.inner + .store + .queue_record(&entry.record.job_id, entry.record.encode()); + } + JobStatus::Running => { + if entry.record.cancel_requested_at_ms.is_none() { + entry.record.cancel_requested_at_ms = Some(now); + self.inner + .store + .queue_record(&entry.record.job_id, entry.record.encode()); + } + if let Some(run) = &entry.run { + run.execution.stop(if force { + StoppedBy::Kill + } else { + StoppedBy::Terminate + }); + } + } + _ => {} + } } - pub async fn interrupt_all(&self) -> Result<(), EnvironmentProtocolError> { - let now = now_ms(); - let mut running = Vec::new(); - { - let mut state = self.state.lock().await; - state.accepting = false; - let keys = state - .jobs - .iter() - .filter(|(_, record)| !record.status.is_terminal()) - .map(|(key, _)| key.clone()) - .collect::>(); - for key in keys { - let active = state.running.get(&key).cloned(); - let Some(record) = state.jobs.get_mut(&key) else { - continue; + /// Validates a start and resolves its dependencies within the request. + fn resolve_start(&self, params: &StartJobsParams) -> Result, ProtocolError> { + let limits = &self.inner.config.limits; + check_job_count("jobs", params.jobs.len(), limits.max_request_jobs)?; + let mut ids = BTreeSet::new(); + let mut names = BTreeMap::new(); + for (index, job) in params.jobs.iter().enumerate() { + let field = format!("jobs[{index}].command"); + validate_command(&job.command, &field)?; + if matches!(job.command.input, ExecInput::Pipe {}) { + return Err(ProtocolError::invalid_field( + format!("{field}.input"), + "jobs take no pipe input; use bytes", + )); + } + if !ids.insert(job.job_id.clone()) { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].jobId"), + format!("job id {} appears twice", job.job_id), + )); + } + if let Some(name) = &job.name + && names.insert(name.clone(), job.job_id.clone()).is_some() + { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].name"), + format!("job name {name} appears twice"), + )); + } + } + let mut resolved = Vec::new(); + for (index, job) in params.jobs.iter().enumerate() { + let mut depends_on = Vec::new(); + for dependency in &job.depends_on { + let job_id = match dependency { + JobDependency::JobId { job_id } => job_id.clone(), + JobDependency::Name { name } => names.get(name).cloned().ok_or_else(|| { + ProtocolError::invalid_field( + format!("jobs[{index}].dependsOn"), + format!("no job of this request is named {name}"), + ) + })?, }; - if let Some(active) = active { - active.interrupted.store(true, Ordering::Release); - record.status = JobStatus::CancelRequested; - record.failure = - Some("environment close requested job interruption".to_owned()); - running.push(active); - } else { - record.status = JobStatus::Interrupted; - record.finished_at_ms = Some(now); - record.failure = Some("environment closed before job started".to_owned()); + if !depends_on.contains(&job_id) { + depends_on.push(job_id); } - self.persist_record(record)?; } + let spec_hash = spec_hash( + job.name.as_ref(), + &job.command, + &depends_on, + job.dependency_policy, + job.queue_key.as_ref(), + ); + resolved.push(ResolvedJob { + job_id: job.job_id.clone(), + name: job.name.clone(), + command: job.command.clone(), + depends_on, + dependency_policy: job.dependency_policy, + queue_key: job.queue_key.clone(), + spec_hash, + }); } - for active in running { - active.cancel.notify_one(); - } - self.notify.notify_waiters(); - Ok(()) + reject_cycles(&resolved)?; + Ok(resolved) } - async fn schedule_ready_jobs(&self) { + /// Fails jobs whose dependencies can no longer succeed and starts every + /// runnable job whose queue is free, in start order. + fn schedule(&self) { + let mut state = self.state(); + let mut changed = false; loop { - let ready = { - let mut state = self.state.lock().await; - if let Err(error) = self.mark_dependency_failures(&mut state) { - eprintln!("lightspeed-envd job dependency update failed: {error:?}"); + let now = now_ms(); + let mut progressed = false; + let doomed: Vec = state + .jobs + .values() + .filter(|entry| { + entry.record.status == JobStatus::Queued + && entry.record.dependency_policy == JobDependencyPolicy::AllSucceeded + && entry.record.depends_on.iter().any(|dependency| { + state.jobs.get(dependency).is_none_or(|dependency| { + dependency.record.status.is_terminal() + && dependency.record.status != JobStatus::Succeeded + }) + }) + }) + .map(|entry| entry.record.job_id.clone()) + .collect(); + for job_id in doomed { + if let Some(entry) = state.jobs.get_mut(&job_id) { + entry.pending = None; + entry.record.status = JobStatus::DependencyFailed; + entry.record.finished_at_ms = Some(now); + self.inner + .store + .queue_record(&job_id, entry.record.encode()); + progressed = true; } + } - let mut busy_queues = state - .running - .keys() - .filter_map(|key| state.jobs.get(key)) - .filter_map(|record| { - record - .queue_key - .as_ref() - .map(|queue_key| (record.namespace.clone(), queue_key.clone())) - }) - .collect::>(); - let mut candidates = state - .jobs - .iter() - .filter_map(|(key, record)| { - if !matches!(record.status, JobStatus::Accepted | JobStatus::Queued) - || state.running.contains_key(key) - || !dependencies_satisfied(&state, record) - { - return None; - } - Some(( - key.clone(), - record.accept_seq, - record.queued_at_ms.unwrap_or(record.created_at_ms), - record.created_at_ms, - record.job_id.clone(), - )) - }) - .collect::>(); - candidates.sort_by(|left, right| { - left.1 - .cmp(&right.1) - .then_with(|| left.2.cmp(&right.2)) - .then_with(|| left.3.cmp(&right.3)) - .then_with(|| left.4.cmp(&right.4)) - }); - - let mut ready_ids = Vec::new(); - for (key, _, _, _, _) in candidates { - let Some(record) = state.jobs.get(&key) else { - continue; - }; - if let Some(queue_key) = record.queue_key.as_ref() - && !busy_queues.insert((record.namespace.clone(), queue_key.clone())) - { - continue; - } - ready_ids.push(key.clone()); + let mut busy: BTreeSet = state + .jobs + .values() + .filter(|entry| entry.record.status == JobStatus::Running) + .filter_map(|entry| entry.record.queue_key.clone()) + .collect(); + let mut runnable: Vec<(u64, JobId)> = state + .jobs + .values() + .filter(|entry| { + entry.record.status == JobStatus::Queued + && entry.record.depends_on.iter().all(|dependency| { + state.jobs.get(dependency).is_none_or(|dependency| { + match entry.record.dependency_policy { + JobDependencyPolicy::AllSucceeded => { + dependency.record.status == JobStatus::Succeeded + } + JobDependencyPolicy::AllTerminal => { + dependency.record.status.is_terminal() + } + } + }) + }) + }) + .map(|entry| (entry.record.seq, entry.record.job_id.clone())) + .collect(); + runnable.sort(); + for (_, job_id) in runnable { + let entry = state.jobs.get_mut(&job_id).expect("runnable job exists"); + let newly_runnable = entry.record.queued_at_ms.is_none(); + if newly_runnable { + entry.record.queued_at_ms = Some(now); } - let now = now_ms(); - let mut ready = Vec::new(); - for key in ready_ids { - let mut ready_record = None; - if let Some(record) = state.jobs.get_mut(&key) { - record.status = JobStatus::Running; - record.started_at_ms = Some(now); - record.failure = None; - if let Err(error) = self.persist_record(record) { - eprintln!( - "lightspeed-envd failed to persist running job {}: {error:?}", - job_key_id(&key) - ); - } - ready_record = Some(record.clone()); - } - if let Some(record) = ready_record { - let secret_env = state.secret_envs.get(&key).cloned().unwrap_or_default(); - let running = Arc::new(RunningJob { - cancel: Notify::new(), - interrupted: AtomicBool::new(false), - }); - state.running.insert(key, running.clone()); - ready.push((record.clone(), running, secret_env)); + if let Some(key) = &entry.record.queue_key + && !busy.insert(key.clone()) + { + if newly_runnable { + self.inner + .store + .queue_record(&job_id, entry.record.encode()); } + continue; } - ready - }; - - if ready.is_empty() { - break; + self.start_entry(entry, now); + progressed = true; } - for (record, running, secret_env) in ready { - let manager = self.clone(); - tokio::spawn(async move { - manager.run_job(record, running, secret_env).await; - }); + changed |= progressed; + if !progressed { + break; } } + drop(state); + if changed { + self.bump(); + } } - fn mark_dependency_failures( - &self, - state: &mut JobManagerState, - ) -> Result<(), EnvironmentProtocolError> { - loop { - let mut changed = false; - let failed = state - .jobs - .iter() - .filter(|(_, record)| { - matches!(record.status, JobStatus::Accepted | JobStatus::Queued) - && record.dependency_policy == JobDependencyPolicy::AllSucceeded - && record.dependencies.iter().any(|dependency| { - state - .jobs - .get(&job_key(&record.namespace, dependency)) - .map(|dependency| { - dependency.status.is_terminal() - && dependency.status != JobStatus::Succeeded - }) - .unwrap_or(true) - }) - }) - .map(|(job_id, _)| job_id.clone()) - .collect::>(); - if failed.is_empty() { - return Ok(()); - } - let now = now_ms(); - for job_id in failed { - let Some(record) = state.jobs.get_mut(&job_id) else { - continue; - }; - if record.status.is_terminal() { - continue; - } - record.status = JobStatus::DependencyFailed; - record.finished_at_ms = Some(now); - record.failure = Some("one or more dependencies did not succeed".to_owned()); - self.persist_record(record)?; - changed = true; + /// Starts one runnable job, or fails it when its `cwd` or spawn fails. + fn start_entry(&self, entry: &mut JobEntry, now: i64) { + let config = &self.inner.config; + let job_id = entry.record.job_id.clone(); + let Some(command) = entry.pending.take() else { + entry.record.status = JobStatus::Failed; + entry.record.finished_at_ms = Some(now); + entry.record.error = Some(ProtocolError::internal("the job's command was lost")); + self.inner + .store + .queue_record(&job_id, entry.record.encode()); + return; + }; + let options = ExecOptions { + timeout: command.timeout_ms.map(Duration::from_millis), + terminate_grace: Duration::from_millis(config.limits.cancel_grace_ms), + after_exit: AfterExit::Manual, + }; + let spawned = resolve_cwd(config, command.cwd.as_ref()).and_then(|cwd| { + let sink = JobOutput { + job_id: job_id.clone(), + log: OutputLog::default(), + cap: config.limits.max_retained_output_bytes, + store: self.inner.store.clone(), + changed: self.inner.changed.clone(), + }; + PreparedCommand::new(command, cwd, &config.scrubbed_env, None).spawn(sink, options, 0) + }); + match spawned { + Ok(execution) => { + entry.record.status = JobStatus::Running; + entry.record.started_at_ms = Some(now); + entry.record.spawn = Some(SpawnRecord::new(execution.group(), &config.instance_id)); + self.inner + .store + .queue_record(&job_id, entry.record.encode()); + execution.start_input(); + entry.run = Some(Run { + execution: execution.clone(), + started: Instant::now(), + }); + let manager = self.clone(); + tokio::spawn(async move { manager.supervise(job_id, execution).await }); } - if !changed { - return Ok(()); + Err(error) => { + entry.record.status = JobStatus::Failed; + entry.record.finished_at_ms = Some(now); + entry.record.error = Some(error); + self.inner + .store + .queue_record(&job_id, entry.record.encode()); } - self.notify.notify_waiters(); } } - async fn run_job( - &self, - record: JobRecord, - running: Arc, - secret_env: BTreeMap, - ) { - let result = self - .spawn_and_wait(record.clone(), running.clone(), secret_env) - .await; - let (status, exit_code, orphaned_descendants, failure) = match result { - Ok(outcome) => { - let (status, failure) = match outcome.finish { - FinishKind::Exit => { - if outcome.exit_code == Some(0) { - (JobStatus::Succeeded, None) - } else { - ( - JobStatus::Failed, - Some(match outcome.exit_code { - Some(code) => format!("job exited with status {code}"), - None => "job exited without a status code".to_owned(), - }), - ) - } - } - FinishKind::Cancelled => { - (JobStatus::Cancelled, Some("job cancelled".to_owned())) - } - FinishKind::Interrupted => ( - JobStatus::Interrupted, - Some("job interrupted because the environment closed".to_owned()), - ), - FinishKind::TimedOut => (JobStatus::TimedOut, Some("job timed out".to_owned())), - }; - ( - status, - outcome.exit_code, - outcome.orphaned_descendants, - failure, - ) - } - Err(error) => (JobStatus::Failed, None, false, Some(error.message)), + /// Follows a running job to its end: root exit, sweep, settled output, + /// final record, then the scheduler. + async fn supervise(self, job_id: JobId, execution: Execution) { + execution.wait_until(|state| state.exited()).await; + let leftovers = execution.sample_leftovers(LeftoverDisposition::Swept); + if leftovers.is_some() || execution.group_alive() { + execution + .group() + .terminate(Duration::from_millis( + self.inner.config.limits.cancel_grace_ms, + )) + .await; + } + execution.settle(true).await; + let (exit, stopped_by, error, log) = execution.with_state(|state| { + ( + state.exit().cloned(), + state.stopped_by(), + state.error().cloned(), + Arc::new(state.sink.log.clone()), + ) + }); + let status = match (&exit, stopped_by) { + (_, Some(StoppedBy::Timeout)) => JobStatus::TimedOut, + (_, Some(StoppedBy::Terminate | StoppedBy::Kill)) => JobStatus::Cancelled, + (Some(ExitStatus::Code { code: 0 }), None) if error.is_none() => JobStatus::Succeeded, + _ => JobStatus::Failed, }; + let error = error.or_else(|| match (&exit, status) { + (Some(exit), JobStatus::Failed) if !matches!(exit, ExitStatus::Code { .. }) => { + Some(ProtocolError::process_failed(describe_exit(exit))) + } + _ => None, + }); + let final_record = { + let state = self.state(); + let Some(entry) = state.jobs.get(&job_id) else { + return; + }; + let mut record = entry.record.clone(); + record.status = status; + record.exit = exit; + record.stopped_by = stopped_by; + record.leftovers = leftovers; + record.error = error; + record.finished_at_ms = Some(now_ms()); + record.duration_ms = entry + .run + .as_ref() + .map(|run| u64::try_from(run.started.elapsed().as_millis()).unwrap_or(u64::MAX)); + record.output_end_bytes = log.end(); + record + }; + // The output and the record are on disk before anyone sees the job + // terminal, so a later load from disk finds everything. + self.inner + .store + .finish(&job_id, final_record.encode()) + .await; { - let mut state = self.state.lock().await; - let key = job_key(&record.namespace, &record.job_id); - state.running.remove(&key); - state.secret_envs.remove(&key); - if let Some(record) = state.jobs.get_mut(&key) { - record.status = status; - record.exit_code = exit_code; - record.orphaned_descendants = orphaned_descendants; - record.failure = failure; - record.finished_at_ms = Some(now_ms()); - if let Err(error) = self.persist_record(record) { - eprintln!( - "lightspeed-envd failed to persist finished job {}: {error:?}", - record.job_id - ); + let mut state = self.state(); + let JobsState { jobs, cached, .. } = &mut *state; + if let Some(entry) = jobs.get_mut(&job_id) { + let cancel_requested_at_ms = entry.record.cancel_requested_at_ms; + entry.record = final_record; + if entry.record.cancel_requested_at_ms != cancel_requested_at_ms { + entry.record.cancel_requested_at_ms = cancel_requested_at_ms; + self.inner + .store + .queue_record(&job_id, entry.record.encode()); } + entry.run = None; + entry.output = Some(log); + cache_output(jobs, cached, &job_id); } } - self.notify.notify_waiters(); - self.kick_scheduler(); - } - - fn kick_scheduler(&self) { - let manager = self.clone(); - tokio::spawn(async move { - manager.schedule_ready_jobs().await; - }); - } - - async fn spawn_and_wait( - &self, - record: JobRecord, - running: Arc, - secret_env: BTreeMap, - ) -> Result { - if record.argv.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "job argv must not be empty", - )); - } - let cwd = record - .cwd - .as_ref() - .map(|path| self.resolve_cwd(path)) - .transpose()? - .unwrap_or_else(|| self.cwd.clone()); - - let mut command = Command::new(&record.argv[0]); - for name in secret_env.keys() { - if record.env.contains_key(name) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("job env collides with secret env: {name}"), - )); - } - } - for name in self.scrubbed_env.iter() { - command.env_remove(name); - } - command - .args(&record.argv[1..]) - .current_dir(cwd) - .envs(record.env.iter()) - .stdin(if record.stdin.is_some() { - Stdio::piped() - } else { - Stdio::null() - }) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - for (name, value) in &secret_env { - command.env(name, value.expose()); - } - process_group::spawn_in_own_group(&mut command); - - let mut child = command.spawn().map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - format!("spawn job {:?}: {error}", record.argv), - ) - })?; - let pgid = child.id(); - let stdout = child.stdout.take(); - let stderr = child.stderr.take(); - if let Some(input) = record.stdin.as_ref() { - let Some(stdin) = child.stdin.as_mut() else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - "job stdin was not available", - )); - }; - stdin.write_all(input.as_slice()).await.map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - })?; - child.stdin.take(); - } - - let stdout_task = stdout.map(|stdout| { - tokio::spawn(read_job_stream( - self.clone(), - record.namespace.clone(), - record.job_id.clone(), - stdout, - JobOutputStream::Stdout, - redactions_for_secret_env(&secret_env), - )) - }); - let stderr_task = stderr.map(|stderr| { - tokio::spawn(read_job_stream( - self.clone(), - record.namespace.clone(), - record.job_id.clone(), - stderr, - JobOutputStream::Stderr, - redactions_for_secret_env(&secret_env), - )) - }); - - let kill_job_group = || { - if let Some(pgid) = pgid { - process_group::kill_group(pgid); - } - }; - let (finish, status) = if let Some(timeout_ms) = record.timeout_ms { - tokio::select! { - status = child.wait() => { - (FinishKind::Exit, status) - } - _ = running.cancel.notified() => { - kill_job_group(); - let _ = child.start_kill(); - let finish = if running.interrupted.load(Ordering::Acquire) { - FinishKind::Interrupted - } else { - FinishKind::Cancelled - }; - (finish, child.wait().await) - } - _ = tokio::time::sleep(Duration::from_millis(timeout_ms)) => { - kill_job_group(); - let _ = child.start_kill(); - (FinishKind::TimedOut, child.wait().await) - } - } - } else { - tokio::select! { - status = child.wait() => { - (FinishKind::Exit, status) - } - _ = running.cancel.notified() => { - kill_job_group(); - let _ = child.start_kill(); - let finish = if running.interrupted.load(Ordering::Acquire) { - FinishKind::Interrupted - } else { - FinishKind::Cancelled - }; - (finish, child.wait().await) - } + self.bump(); + self.schedule(); + } + + /// A terminal job's output: the cached log, or the log loaded from disk + /// (and cached). `None` for an unknown or running job. + async fn ensure_output(&self, job_id: &JobId) -> Option> { + let (dir, end) = { + let mut state = self.state(); + let JobsState { jobs, cached, .. } = &mut *state; + let entry = jobs.get(job_id)?; + if entry.run.is_some() || !entry.record.status.is_terminal() { + return None; } - }; - - // The root process's exit is ground truth for the job outcome. - // Descendants may still hold the output pipes open, so drain them for - // a bounded grace period, sweep whatever the script left running, and - // abandon the pipes rather than letting completion depend on EOF. - let mut stream_tasks = [stdout_task, stderr_task]; - // Probe for survivors only on a natural exit; on cancel/timeout the - // group was killed deliberately and not-yet-reaped children would - // read as false positives. - let mut orphaned_descendants = - matches!(finish, FinishKind::Exit) && pgid.is_some_and(process_group::group_alive); - let drained = - drain_stream_tasks(&mut stream_tasks, process_group::OUTPUT_DRAIN_GRACE).await; - orphaned_descendants |= !drained; - if orphaned_descendants && let Some(pgid) = pgid { - process_group::sweep_group(pgid).await; - } - if !drained - && !drain_stream_tasks(&mut stream_tasks, process_group::OUTPUT_DRAIN_GRACE).await - { - for task in stream_tasks.iter().flatten() { - task.abort(); + if let Some(log) = entry.output.clone() { + cache_output(jobs, cached, job_id); + return Some(log); } - } - - let status = status.map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - })?; - Ok(SpawnOutcome { - finish, - exit_code: status.code(), - orphaned_descendants, - }) - } - - fn resolve_cwd(&self, path: &EnvironmentPath) -> Result { - let candidate = if path.is_absolute() { - PathBuf::from(path.as_str()) - } else if path.as_str() == "." { - self.cwd.clone() - } else { - self.cwd.join(path.as_str()) - }; - let normalized = normalize_path(candidate); - if !normalized.starts_with(&self.fs_root) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Forbidden, - format!( - "job cwd is outside bridge fs root: {} (root {})", - normalized.display(), - self.fs_root.display() - ), - )); - } - Ok(normalized) - } - - fn persist_record(&self, record: &JobRecord) -> Result<(), EnvironmentProtocolError> { - persist_record_at(&self.jobs_root, record).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - format!("persist job {}: {error}", record.job_id), + ( + self.inner.store.root().join(job_id.as_str()), + entry.record.output_end_bytes, ) - }) - } -} - -/// Joins the output-reader tasks against one shared deadline. Returns true -/// when every reader finished; unfinished readers stay in their slot. -async fn drain_stream_tasks(tasks: &mut [Option>; 2], grace: Duration) -> bool { - let deadline = tokio::time::Instant::now() + grace; - let mut drained = true; - for slot in tasks.iter_mut() { - let Some(task) = slot.as_mut() else { - continue; }; - if tokio::time::timeout_at(deadline, task).await.is_ok() { - *slot = None; - } else { - drained = false; - } - } - drained -} - -fn require_accepting(state: &JobManagerState) -> Result<(), EnvironmentProtocolError> { - if state.accepting { - Ok(()) - } else { - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Conflict, - "environment target is closed and no longer accepts jobs", - )) - } -} - -async fn read_job_stream( - manager: JobManager, - namespace: String, - job_id: JobId, - mut reader: R, - stream: JobOutputStream, - redactions: Vec>, -) where - R: AsyncRead + Unpin, -{ - let mut buffer = vec![0; 8192]; - loop { - let read = match reader.read(&mut buffer).await { - Ok(0) => { - manager.notify.notify_waiters(); - return; + let cap = self.inner.config.limits.max_retained_output_bytes; + let loaded = tokio::task::spawn_blocking(move || load_output(&dir, end, cap)).await; + let log = match loaded { + Ok(Ok(log)) => log, + Ok(Err(error)) => { + eprintln!("lightspeed-envd: could not load output of job {job_id}: {error}"); + OutputLog::from_records([], end) } - Ok(read) => read, Err(error) => { - let mut state = manager.state.lock().await; - if let Some(record) = state.jobs.get_mut(&job_key(&namespace, &job_id)) - && !record.status.is_terminal() - { - record.failure = Some(error.to_string()); - if let Err(error) = manager.persist_record(record) { - eprintln!( - "lightspeed-envd failed to persist stream failure for {job_id}: {error:?}" - ); - } - } - manager.notify.notify_waiters(); - return; + eprintln!("lightspeed-envd: could not load output of job {job_id}: {error}"); + OutputLog::from_records([], end) } }; - let mut state = manager.state.lock().await; - if let Some(record) = state.jobs.get_mut(&job_key(&namespace, &job_id)) { - if record.status.is_terminal() { - // The job completed without waiting for pipe EOF; late output - // from orphaned descendants must not mutate the terminal - // record. - return; - } - let seq = record.output_next_seq; - record.output_next_seq += 1; - record.output_chunks.push(JobOutputChunk { - seq, - stream, - chunk: ByteChunk::from(redact_bytes(&buffer[..read], &redactions)), - }); - if let Err(error) = manager.persist_record(record) { - eprintln!("lightspeed-envd failed to persist output for {job_id}: {error:?}"); + let mut state = self.state(); + let JobsState { jobs, cached, .. } = &mut *state; + let entry = jobs.get_mut(job_id)?; + let log = match &entry.output { + Some(log) => log.clone(), + None => { + entry.record.output_end_bytes = entry.record.output_end_bytes.max(log.end()); + let log = Arc::new(log); + entry.output = Some(log.clone()); + log } - } - manager.notify.notify_waiters(); + }; + cache_output(jobs, cached, job_id); + Some(log) } -} -fn validate_and_resolve_start( - state: &JobManagerState, - params: &StartJobsParams, -) -> Result, EnvironmentProtocolError> { - if params.jobs.is_empty() { - return Err(invalid_request("job/start requires at least one job")); - } - validate_id_component("namespace", ¶ms.namespace)?; - validate_id_component("request_id", ¶ms.request_id)?; - - let mut seen_job_ids = BTreeSet::new(); - let mut seen_names = BTreeMap::new(); - for spec in ¶ms.jobs { - validate_id_component("job id", spec.job_id.as_str())?; - if spec.argv.is_empty() { - return Err(invalid_request(format!( - "job {} argv must not be empty", - spec.job_id - ))); - } - for name in spec.secret_env.keys() { - validate_nonempty("secret env name", name)?; - if spec.env.contains_key(name) { - return Err(invalid_request(format!( - "job {} env collides with secret env: {name}", - spec.job_id - ))); + /// Adds to `logs` the cached output of every terminal job the read + /// pages, and answers the locked state to page from. When a terminal + /// job's output is in neither the cache nor `logs` (and was not + /// `attempted`), answers those jobs instead: load them and call again. + fn take_logs( + &self, + params: &ReadJobsParams, + logs: &mut BTreeMap>, + attempted: &mut BTreeSet, + ) -> Result, Vec> { + let state = self.state(); + let mut missing = Vec::new(); + for entry in params + .jobs + .iter() + .filter(|entry| needs_output(params, entry)) + { + let Some(job) = state.jobs.get(&entry.job_id) else { + continue; + }; + if job.run.is_some() || logs.contains_key(&entry.job_id) { + continue; } - } - if !seen_job_ids.insert(spec.job_id.as_str().to_owned()) { - return Err(invalid_request(format!( - "duplicate job id in start request: {}", - spec.job_id - ))); - } - if let Some(name) = spec.name.as_deref() { - validate_nonempty("job name", name)?; - if seen_names - .insert(name.to_owned(), spec.job_id.clone()) - .is_some() - { - return Err(invalid_request(format!("duplicate job name: {name}"))); + if let Some(log) = &job.output { + logs.insert(entry.job_id.clone(), log.clone()); + } else if job.record.status.is_terminal() && attempted.insert(entry.job_id.clone()) { + missing.push(entry.job_id.clone()); } } - validate_optional_id_component("queue_key", spec.queue_key.as_deref())?; + if missing.is_empty() { + Ok(state) + } else { + Err(missing) + } } - let request_job_ids = params - .jobs - .iter() - .map(|spec| spec.job_id.as_str().to_owned()) - .collect::>(); - - let mut resolved = Vec::new(); - for spec in ¶ms.jobs { - let dependencies = resolve_explicit_dependencies(&seen_names, spec)?; - for dependency in &dependencies { - if !request_job_ids.contains(dependency.as_str()) - && !state - .jobs - .contains_key(&job_key(¶ms.namespace, dependency)) - { - return Err(invalid_request(format!( - "job {} depends on unknown job id {}", - spec.job_id, dependency - ))); - } + fn read_ready( + &self, + params: &ReadJobsParams, + starts: &[u64], + initial: &[Option], + ) -> bool { + let state = self.state(); + let entries = params.jobs.iter().zip(starts).zip(initial); + let all_terminal = params.jobs.iter().all(|entry| { + state + .jobs + .get(&entry.job_id) + .is_none_or(|job| job.record.status.is_terminal()) + }); + if all_terminal { + return true; } - - let spec_hash = job_spec_hash(params, spec, &dependencies)?; - let key = job_key(¶ms.namespace, &spec.job_id); - if let Some(existing) = state.jobs.get(&key) - && existing.spec_hash != spec_hash - { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Conflict, - format!( - "job id already exists with different input in namespace {}: {}", - params.namespace, spec.job_id - ), - )); + match params.return_on { + ReturnOn::Exit => false, + ReturnOn::AnyOutput => entries.into_iter().any(|((entry, start), initial)| { + let job = state.jobs.get(&entry.job_id); + job.map(|job| job.record.status) != *initial + || job.is_some_and(|job| output_end(job) > *start) + }), + ReturnOn::Window => entries.into_iter().any(|((entry, start), _)| { + let budget = entry.max_bytes.unwrap_or(params.max_bytes); + state + .jobs + .get(&entry.job_id) + .is_some_and(|job| output_end(job).saturating_sub(*start) >= budget) + }), } - - resolved.push(ResolvedJob { - spec: spec.clone(), - dependencies, - secret_env: spec.secret_env.clone(), - spec_hash, - }); } - reject_cycles(&resolved)?; - Ok(resolved) -} - -fn resolve_explicit_dependencies( - names: &BTreeMap, - spec: &JobStartSpec, -) -> Result, EnvironmentProtocolError> { - let mut dependencies = Vec::new(); - for dependency in &spec.depends_on { - match (&dependency.job_id, &dependency.name) { - (Some(job_id), None) => push_unique_job_id(&mut dependencies, job_id.clone()), - (None, Some(name)) => { - validate_nonempty("dependency name", name)?; - let Some(job_id) = names.get(name) else { - return Err(invalid_request(format!( - "job {} depends on unknown local name {name}", - spec.job_id - ))); - }; - push_unique_job_id(&mut dependencies, job_id.clone()); - } - (Some(_), Some(_)) => { - return Err(invalid_request(format!( - "job {} dependency must use either jobId or name, not both", - spec.job_id - ))); - } - (None, None) => { - return Err(invalid_request(format!( - "job {} dependency must include jobId or name", - spec.job_id - ))); - } - } - } - for dependency in &dependencies { - if dependency == &spec.job_id { - return Err(invalid_request(format!( - "job {} cannot depend on itself", - spec.job_id - ))); + /// Garbage-collects terminal jobs past the record retention. + fn maintain(&self) { + let retention = + i64::try_from(self.inner.config.limits.record_retention_ms).unwrap_or(i64::MAX); + let now = now_ms(); + let mut state = self.state(); + let expired: Vec = state + .jobs + .values() + .filter(|entry| { + entry.record.status.is_terminal() + && entry + .record + .finished_at_ms + .is_some_and(|finished| now.saturating_sub(finished) >= retention) + }) + .map(|entry| entry.record.job_id.clone()) + .collect(); + for job_id in expired { + state.jobs.remove(&job_id); + state.cached.retain(|cached| cached != &job_id); + self.inner.store.remove(&job_id); } } - Ok(dependencies) -} -fn reject_cycles(resolved: &[ResolvedJob]) -> Result<(), EnvironmentProtocolError> { - let graph = resolved - .iter() - .map(|job| { - ( - job.spec.job_id.as_str().to_owned(), - job.dependencies - .iter() - .filter(|dependency| { - resolved - .iter() - .any(|job| job.spec.job_id.as_str() == dependency.as_str()) - }) - .map(|dependency| dependency.as_str().to_owned()) - .collect::>(), - ) - }) - .collect::>(); - let mut visiting = BTreeSet::new(); - let mut visited = BTreeSet::new(); - for job_id in graph.keys() { - visit_job(job_id, &graph, &mut visiting, &mut visited)?; + fn state(&self) -> MutexGuard<'_, JobsState> { + self.inner + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) } - Ok(()) -} -fn visit_job( - job_id: &str, - graph: &BTreeMap>, - visiting: &mut BTreeSet, - visited: &mut BTreeSet, -) -> Result<(), EnvironmentProtocolError> { - if visited.contains(job_id) { - return Ok(()); - } - if !visiting.insert(job_id.to_owned()) { - return Err(invalid_request(format!( - "job dependency cycle includes {job_id}" - ))); - } - for dependency in graph.get(job_id).into_iter().flatten() { - visit_job(dependency, graph, visiting, visited)?; + fn bump(&self) { + self.inner + .changed + .send_modify(|version| *version = version.wrapping_add(1)); } - visiting.remove(job_id); - visited.insert(job_id.to_owned()); - Ok(()) } -fn push_unique_job_id(dependencies: &mut Vec, job_id: JobId) { - if !dependencies +/// An identical retry replays every job; any other overlap with existing +/// jobs is `jobIdInUse`. `None` when every job is new. +fn replay_or_conflict( + state: &JobsState, + request_id: &JobRequestId, + resolved: &[ResolvedJob], +) -> Result>, ProtocolError> { + let existing = resolved .iter() - .any(|existing| existing.as_str() == job_id.as_str()) - { - dependencies.push(job_id); - } -} - -fn dependencies_satisfied(state: &JobManagerState, record: &JobRecord) -> bool { - record.dependencies.iter().all(|dependency| { - let Some(dependency) = state.jobs.get(&job_key(&record.namespace, dependency)) else { - return false; + .filter(|job| state.jobs.contains_key(&job.job_id)) + .count(); + if existing == 0 { + return Ok(None); + } + let mut summaries = Vec::new(); + for job in resolved { + let in_use = |detail: &str| { + ProtocolError::failed_precondition( + PreconditionReason::JobIdInUse, + format!( + "job id {} {detail}; choose a new job id for a new run", + job.job_id + ), + ) }; - match record.dependency_policy { - JobDependencyPolicy::AllSucceeded => dependency.status == JobStatus::Succeeded, - JobDependencyPolicy::AllTerminal => dependency.status.is_terminal(), + let Some(entry) = state.jobs.get(&job.job_id) else { + return Err(in_use( + "is new while other jobs of this request already exist", + )); + }; + if &entry.record.request_id != request_id { + return Err(in_use("was started by another request")); } - }) -} - -fn build_read_response(state: &JobManagerState, params: &ReadJobsParams) -> ReadJobsResponse { - ReadJobsResponse { - jobs: params - .jobs - .iter() - .map( - |job_id| match state.jobs.get(&job_key(¶ms.namespace, job_id)) { - Some(record) => { - let (output_chunks, output_next_seq) = - select_output_chunks(record, params.after_seq, params.max_bytes); - JobReadResult { - summary: record.summary(), - output_chunks, - output_next_seq, - artifacts: Vec::new(), - } - } - None => JobReadResult { - summary: lost_summary(¶ms.namespace, job_id), - output_chunks: Vec::new(), - output_next_seq: 0, - artifacts: Vec::new(), - }, - }, - ) - .collect(), + if entry.record.spec_hash != job.spec_hash { + return Err(in_use("was started with a different spec")); + } + summaries.push(entry.record.summary()); } + Ok(Some(summaries)) } -fn select_output_chunks( - record: &JobRecord, - after_seq: Option, - max_bytes: Option, -) -> (Vec, u64) { - let after_seq = after_seq.unwrap_or(0); - let mut chunks = Vec::new(); - let mut bytes = 0usize; - let mut next_seq = after_seq; - - for chunk in record - .output_chunks +fn reject_cycles(resolved: &[ResolvedJob]) -> Result<(), ProtocolError> { + let index: BTreeMap<&JobId, usize> = resolved .iter() - .filter(|chunk| chunk.seq >= after_seq) - { - let chunk_bytes = chunk.chunk.as_slice(); - if let Some(max_bytes) = max_bytes { - if bytes >= max_bytes { - break; - } - let remaining = max_bytes - bytes; - if chunk_bytes.len() > remaining { - chunks.push(JobOutputChunk { - seq: chunk.seq, - stream: chunk.stream, - chunk: ByteChunk::from(chunk_bytes[..remaining].to_vec()), - }); - next_seq = chunk.seq + 1; - break; + .enumerate() + .map(|(position, job)| (&job.job_id, position)) + .collect(); + // 0 unvisited, 1 on the stack, 2 done. + let mut marks = vec![0u8; resolved.len()]; + fn visit( + position: usize, + resolved: &[ResolvedJob], + index: &BTreeMap<&JobId, usize>, + marks: &mut [u8], + ) -> Result<(), usize> { + match marks[position] { + 2 => return Ok(()), + 1 => return Err(position), + _ => {} + } + marks[position] = 1; + for dependency in &resolved[position].depends_on { + if let Some(&next) = index.get(dependency) { + visit(next, resolved, index, marks)?; } } - bytes += chunk_bytes.len(); - next_seq = chunk.seq + 1; - chunks.push(chunk.clone()); - } - - if chunks.is_empty() && next_seq < record.output_next_seq { - next_seq = record.output_next_seq; + marks[position] = 2; + Ok(()) } - - (chunks, next_seq) -} - -fn resolve_cancel_scope(state: &JobManagerState, params: &CancelJobsParams) -> Vec { - let mut target_ids = params - .jobs - .iter() - .map(|job_id| job_key(¶ms.namespace, job_id)) - .collect::>(); - match params.scope { - JobCancelScope::Job => {} - JobCancelScope::Dependents => { - let mut changed = true; - while changed { - changed = false; - for (key, record) in &state.jobs { - if record.namespace != params.namespace - || record.status.is_terminal() - || target_ids.contains(key) - { - continue; - } - if record.dependencies.iter().any(|dependency| { - target_ids.contains(&job_key(¶ms.namespace, dependency)) - }) { - changed |= target_ids.insert(key.clone()); - } - } - } + for position in 0..resolved.len() { + if let Err(cycle) = visit(position, resolved, &index, &mut marks) { + return Err(ProtocolError::invalid_field( + format!("jobs[{cycle}].dependsOn"), + format!( + "job {} depends on itself through a cycle", + resolved[cycle].job_id + ), + )); } } - - target_ids.into_iter().collect() + Ok(()) } -fn job_spec_hash( - params: &StartJobsParams, - spec: &JobStartSpec, - dependencies: &[JobId], -) -> Result { - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct JobHashMaterial<'a> { - namespace: &'a str, - job: JobHashSpec<'a>, - } - +/// The identity hash of a job's spec. Secret values and input bytes enter +/// only as their own digests. +fn spec_hash( + name: Option<&JobName>, + command: &CommandSpec, + depends_on: &[JobId], + dependency_policy: JobDependencyPolicy, + queue_key: Option<&QueueKey>, +) -> String { #[derive(Serialize)] #[serde(rename_all = "camelCase")] - struct JobHashSpec<'a> { - job_id: &'a JobId, - name: &'a Option, + struct Material<'a> { + name: Option<&'a JobName>, argv: &'a [String], - cwd: &'a Option, + cwd: Option<&'a EnvironmentPath>, env: &'a BTreeMap, - secret_env_names: Vec<&'a String>, - stdin: &'a Option, + secret_env: BTreeMap<&'a str, String>, + input: String, timeout_ms: Option, - dependencies: &'a [JobId], + depends_on: &'a [JobId], dependency_policy: JobDependencyPolicy, - queue_key: &'a Option, - } - - let material = JobHashMaterial { - namespace: params.namespace.as_str(), - job: JobHashSpec { - job_id: &spec.job_id, - name: &spec.name, - argv: &spec.argv, - cwd: &spec.cwd, - env: &spec.env, - secret_env_names: spec.secret_env.keys().collect(), - stdin: &spec.stdin, - timeout_ms: spec.timeout_ms, - dependencies, - dependency_policy: spec.dependency_policy, - queue_key: &spec.queue_key, + queue_key: Option<&'a QueueKey>, + } + let digest = |bytes: &[u8]| encode_hex(&Sha256::digest(bytes)); + let material = Material { + name, + argv: &command.argv, + cwd: command.cwd.as_ref(), + env: &command.env, + secret_env: command + .secret_env + .iter() + .map(|(name, value)| (name.as_str(), digest(value.expose().as_bytes()))) + .collect(), + input: match &command.input { + ExecInput::None {} => "none".to_owned(), + ExecInput::Bytes { data } => format!("bytes:{}", digest(data.as_slice())), + ExecInput::Pipe {} => "pipe".to_owned(), }, + timeout_ms: command.timeout_ms, + depends_on, + dependency_policy, + queue_key, }; - let bytes = serde_json::to_vec(&material).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - format!("encode job idempotency hash: {error}"), - ) - })?; - Ok(format!("{:016x}", fnv1a64(&bytes))) + digest(&serde_json::to_vec(&material).expect("hash material encodes")) } -fn fnv1a64(bytes: &[u8]) -> u64 { - let mut hash = 0xcbf29ce484222325u64; - for byte in bytes { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x100000001b3); +fn resolve_cwd( + config: &JobConfig, + cwd: Option<&EnvironmentPath>, +) -> Result { + match cwd { + Some(path) => config.confinement.resolve_directory(path), + None => Ok(config.cwd.clone()), } - hash } -impl JobRecord { - fn summary(&self) -> JobSummary { - JobSummary { - namespace: self.namespace.clone(), - job_id: self.job_id.clone(), - name: self.name.clone(), - status: self.status, - dependencies: self.dependencies.clone(), - created_at_ms: self.created_at_ms, - queued_at_ms: self.queued_at_ms, - started_at_ms: self.started_at_ms, - finished_at_ms: self.finished_at_ms, - exit_code: self.exit_code, - orphaned_descendants: self.orphaned_descendants, - failure: self.failure.clone(), - queue_key: self.queue_key.clone(), - } - } +/// Whether a `job/read` entry pages output (rather than only a summary). +fn needs_output(params: &ReadJobsParams, entry: &JobReadRequest) -> bool { + params.max_bytes > 0 || entry.after.is_some() || entry.tail_bytes.is_some() } -fn lost_summary(namespace: &str, job_id: &JobId) -> JobSummary { - JobSummary { - namespace: namespace.to_owned(), - job_id: job_id.clone(), - name: None, - status: JobStatus::Lost, - dependencies: Vec::new(), - created_at_ms: 0, - queued_at_ms: None, - started_at_ms: None, - finished_at_ms: Some(now_ms()), - exit_code: None, - orphaned_descendants: false, - failure: Some("unknown job id".to_owned()), - queue_key: None, +/// Runs `f` on the job's retained output: the live log while it runs, the +/// cached log once terminal. `None` when neither is in memory. +fn with_log(entry: &JobEntry, f: impl FnOnce(&OutputLog) -> R) -> Option { + if let Some(run) = &entry.run { + return Some(run.execution.with_state(|state| f(&state.sink.log))); } + entry.output.as_deref().map(f) } -fn persist_record_at(jobs_root: &PathBuf, record: &JobRecord) -> anyhow::Result<()> { - std::fs::create_dir_all(jobs_root)?; - let stem = record_file_stem(&record.namespace, &record.job_id); - let path = jobs_root.join(format!("{stem}.json")); - let temp_path = jobs_root.join(format!("{stem}.json.tmp")); - let bytes = serde_json::to_vec_pretty(record)?; - std::fs::write(&temp_path, bytes)?; - std::fs::rename(temp_path, path)?; - Ok(()) -} - -fn job_key(namespace: &str, job_id: &JobId) -> JobKey { - (namespace.to_owned(), job_id.clone()) -} - -fn job_key_id(key: &JobKey) -> String { - format!("{}/{}", key.0, key.1) +fn output_end(entry: &JobEntry) -> u64 { + with_log(entry, OutputLog::end).unwrap_or(entry.record.output_end_bytes) } -fn record_file_stem(namespace: &str, job_id: &JobId) -> String { - format!("{namespace}--{}", job_id.as_str()) +fn cache_output( + jobs: &mut BTreeMap, + cached: &mut VecDeque, + job_id: &JobId, +) { + cached.retain(|cached| cached != job_id); + cached.push_back(job_id.clone()); + while cached.len() > MAX_CACHED_OUTPUTS { + if let Some(evicted) = cached.pop_front() + && let Some(entry) = jobs.get_mut(&evicted) + { + entry.output = None; + } + } } -fn normalize_path(path: PathBuf) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(std::path::MAIN_SEPARATOR.to_string()), - Component::CurDir => {} - Component::ParentDir => { - normalized.pop(); - } - Component::Normal(segment) => normalized.push(segment), - } +fn check_job_count(field: &str, count: usize, max: u32) -> Result<(), ProtocolError> { + if count == 0 { + return Err(ProtocolError::invalid_field(field, "name at least one job")); } - normalized + if count > max as usize { + return Err(ProtocolError::above_ceiling( + field, + count as u64, + u64::from(max), + )); + } + Ok(()) } -fn now_ms() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64) - .unwrap_or_default() +fn unknown_job(job_id: &JobId) -> ProtocolError { + ProtocolError::not_found(format!("unknown job id: {job_id}")) } -fn validate_optional_id_component( - label: &str, - value: Option<&str>, -) -> Result<(), EnvironmentProtocolError> { - if let Some(value) = value { - validate_id_component(label, value)?; +fn status_name(status: JobStatus) -> &'static str { + match status { + JobStatus::Queued => "queued", + JobStatus::Running => "running", + JobStatus::Succeeded => "succeeded", + JobStatus::Failed => "failed", + JobStatus::TimedOut => "timedOut", + JobStatus::Cancelled => "cancelled", + JobStatus::DependencyFailed => "dependencyFailed", + JobStatus::Interrupted => "interrupted", } - Ok(()) } -fn validate_id_component(label: &str, value: &str) -> Result<(), EnvironmentProtocolError> { - validate_nonempty(label, value)?; - if value.len() > 128 { - return Err(invalid_request(format!( - "{label} must be at most 128 bytes" - ))); +/// Reads one job directory's record. +fn load_record(dir: &Path) -> Result { + if !dir.is_dir() { + return Err("not a job directory".to_owned()); } - let mut chars = value.chars(); - let Some(first) = chars.next() else { - return Err(invalid_request(format!("{label} must not be empty"))); - }; - if !first.is_ascii_alphanumeric() { - return Err(invalid_request(format!( - "{label} must start with an ASCII alphanumeric character" - ))); + let bytes = std::fs::read(dir.join("record.json")).map_err(|error| error.to_string())?; + let record: StoredJob = serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + if record.format != JOB_RECORD_FORMAT { + return Err(format!( + "record format {} is not {JOB_RECORD_FORMAT}", + record.format + )); } - if !chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | ':' | '-')) { - return Err(invalid_request(format!( - "{label} may contain only ASCII alphanumeric characters, '_', '.', ':', or '-'" - ))); + if dir.file_name().and_then(|name| name.to_str()) != Some(record.job_id.as_str()) { + return Err("the directory does not match the job id".to_owned()); } - Ok(()) + Ok(record) } -fn validate_nonempty(label: &str, value: &str) -> Result<(), EnvironmentProtocolError> { - if value.trim().is_empty() { - return Err(invalid_request(format!("{label} must not be empty"))); +/// Moves an unreadable job aside so it never blocks startup again. +fn quarantine(path: &Path, quarantine_dir: &Path, reason: &str) { + let name = path + .file_name() + .map_or_else(|| "job".into(), |name| name.to_string_lossy().into_owned()); + let target = quarantine_dir.join(format!("{name}.{}", now_ms())); + let moved = + std::fs::create_dir_all(quarantine_dir).and_then(|()| std::fs::rename(path, &target)); + match moved { + Ok(()) => eprintln!( + "lightspeed-envd: quarantined unreadable job record {} ({reason}) at {}", + path.display(), + target.display() + ), + Err(error) => eprintln!( + "lightspeed-envd: could not quarantine unreadable job record {} ({reason}): {error}", + path.display() + ), } - Ok(()) } -fn invalid_request(message: impl Into) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::InvalidRequest, message) +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) + }) } #[cfg(test)] mod tests { - use std::{collections::BTreeMap, time::Duration}; - use environment_protocol::{ - data::jobs::{ - JobCancelScope, JobDependency, JobDependencyPolicy, JobStatus, ListJobsParams, - ReadJobsParams, StartJobsParams, - }, - shared::{JobId, SecretString}, + bytes::ByteChunk, + data::{exec::GapReason, job::JobSpec}, + error::{ErrorData, ProtocolErrorKind}, }; use super::*; - const TEST_NAMESPACE: &str = "session_1"; + fn limits() -> JobLimits { + JobLimits { + max_read_bytes: 4 * 1024 * 1024, + max_request_jobs: 64, + max_retained_output_bytes: 2 * 1024 * 1024, + record_retention_ms: 7 * 24 * 60 * 60 * 1000, + cancel_grace_ms: 200, + } + } - #[tokio::test(flavor = "current_thread")] - async fn job_manager_runs_parallel_jobs_and_retains_output() { + struct Fixture { + _temp: tempfile::TempDir, + root: PathBuf, + state_dir: PathBuf, + manager: JobManager, + } + + fn fixture_with(limits: JobLimits) -> Fixture { let temp = tempfile::tempdir().expect("tempdir"); let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); + let state_dir = root.join("state"); + let manager = manager_at(&root, &state_dir, limits); + Fixture { + _temp: temp, + root, + state_dir, + manager, + } + } - let response = manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "parallel".to_owned(), - jobs: vec![job("job-a", "printf a"), job("job-b", "printf b")], - }) - .await - .expect("start"); + fn fixture() -> Fixture { + fixture_with(limits()) + } - assert_eq!(response.jobs.len(), 2); - let read = wait_for_jobs(&manager, ["job-a", "job-b"]).await; - assert!( - read.jobs - .iter() - .all(|job| job.summary.status == JobStatus::Succeeded) - ); - assert!( - read.jobs - .iter() - .all(|job| !job.summary.orphaned_descendants) - ); - let output = read - .jobs - .iter() - .flat_map(|job| { - job.output_chunks - .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) - }) - .collect::>(); - assert!(output.contains(&b'a')); - assert!(output.contains(&b'b')); + fn manager_at(root: &Path, state_dir: &Path, limits: JobLimits) -> JobManager { + let confinement = + Confinement::new(root.to_owned(), state_dir.to_owned()).expect("confinement"); + JobManager::new(JobConfig { + cwd: root.to_owned(), + confinement: Arc::new(confinement), + state_dir: state_dir.to_owned(), + scrubbed_env: Vec::new(), + instance_id: DaemonInstanceId::new("envd_test").unwrap(), + limits, + max_wait_ms: 60_000, + }) + .expect("manager") } - #[tokio::test(flavor = "current_thread")] - async fn job_manager_injects_secret_env_without_persisting_values() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - let mut spec = job("secret-job", "printf \"$SECRET_TOKEN\""); - spec.secret_env.insert( - "SECRET_TOKEN".to_owned(), - SecretString::new("super-secret-token"), - ); + fn ctx() -> RequestContext { + RequestContext::detached() + } - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "secret".to_owned(), - jobs: vec![spec], - }) - .await - .expect("start"); + fn id(value: &str) -> JobId { + JobId::new(value).unwrap() + } - let read = wait_for_jobs(&manager, ["secret-job"]).await; - let result = result(&read, "secret-job"); - assert_eq!(result.summary.status, JobStatus::Succeeded); - let output = result - .output_chunks - .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) - .collect::>(); - assert_eq!(output, b""); - - let record_path = root - .join(".lightspeed") - .join("jobs") - .join("session_1--secret-job.json"); - let persisted = std::fs::read_to_string(record_path).expect("persisted job record"); - assert!(!persisted.contains("super-secret-token")); - assert!(persisted.contains("SECRET_TOKEN")); + fn request(value: &str) -> JobRequestId { + JobRequestId::new(value).unwrap() } - #[tokio::test(flavor = "current_thread")] - async fn job_manager_honors_queue_key_order() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let marker = root.join("order.txt"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut first = job("queued-z", "printf 1 >> order.txt"); - let mut second = job("queued-a", "printf 2 >> order.txt"); - let mut third = job("queued-m", "printf 3 >> order.txt"); - first.queue_key = Some("repo".to_owned()); - second.queue_key = Some("repo".to_owned()); - third.queue_key = Some("repo".to_owned()); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "queue".to_owned(), - jobs: vec![first, second, third], - }) + fn job(job_id: &str, script: &str) -> JobSpec { + JobSpec { + job_id: id(job_id), + name: None, + command: CommandSpec { + argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], + cwd: None, + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input: ExecInput::None {}, + timeout_ms: None, + }, + depends_on: Vec::new(), + dependency_policy: JobDependencyPolicy::AllSucceeded, + queue_key: None, + } + } + + fn start(request_id: &str, jobs: Vec) -> StartJobsParams { + StartJobsParams { + request_id: request(request_id), + jobs, + } + } + + fn entry(job_id: &str) -> JobReadRequest { + JobReadRequest { + job_id: id(job_id), + after: None, + tail_bytes: None, + max_bytes: None, + } + } + + /// Waits until every job is terminal and reads their output heads. + async fn finished(manager: &JobManager, job_ids: &[&str]) -> Vec<(JobSummary, OutputPage)> { + let response = manager + .read( + &ctx(), + ReadJobsParams { + jobs: job_ids.iter().map(|job_id| entry(job_id)).collect(), + max_bytes: 4 * 1024 * 1024, + wait_ms: 30_000, + return_on: ReturnOn::Exit, + }, + ) .await - .expect("start"); + .expect("read"); + response + .jobs + .into_iter() + .map(|entry| match entry { + JobReadEntry::Job { summary, output } => { + assert!(summary.status.is_terminal(), "{summary:?}"); + (*summary, output) + } + JobReadEntry::Error { job_id, error } => panic!("{job_id}: {error}"), + }) + .collect() + } - let read = wait_for_jobs(&manager, ["queued-z", "queued-a", "queued-m"]).await; - assert!( - read.jobs + fn text(page: &OutputPage) -> String { + String::from_utf8( + page.chunks .iter() - .all(|job| job.summary.status == JobStatus::Succeeded) - ); - assert_eq!(std::fs::read_to_string(marker).expect("marker"), "123"); + .flat_map(|chunk| chunk.data.as_slice().to_vec()) + .collect(), + ) + .unwrap() + } + + fn precondition(error: &ProtocolError) -> Option { + match &error.data { + ErrorData::FailedPrecondition { reason } => Some(*reason), + _ => None, + } } #[tokio::test(flavor = "current_thread")] - async fn job_manager_honors_explicit_dependencies_and_dependency_failure() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut dependent = job("dependent", "printf should-not-run"); - dependent.depends_on = vec![JobDependency::name("setup")]; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "deps".to_owned(), - jobs: vec![named_job("setup", "setup", "exit 7"), dependent], - }) + async fn jobs_run_and_report_exits() { + let fixture = fixture(); + let response = fixture + .manager + .start( + &ctx(), + start("r1", vec![job("ok", "printf done"), job("bad", "exit 4")]), + ) .await .expect("start"); - - let read = wait_for_jobs(&manager, ["setup", "dependent"]).await; - let setup = result(&read, "setup"); - let dependent = result(&read, "dependent"); - assert_eq!(setup.summary.status, JobStatus::Failed); - assert_eq!(dependent.summary.status, JobStatus::DependencyFailed); + assert_eq!(response.jobs.len(), 2); + let jobs = finished(&fixture.manager, &["ok", "bad"]).await; + assert_eq!(jobs[0].0.status, JobStatus::Succeeded); + assert_eq!(text(&jobs[0].1), "done"); + assert_eq!(jobs[0].1.end, 4); + assert!(jobs[0].0.duration_ms.is_some()); + assert_eq!(jobs[1].0.status, JobStatus::Failed); + assert_eq!(jobs[1].0.exit, Some(ExitStatus::Code { code: 4 })); + assert_eq!(fixture.manager.running_count().await, 0); } #[tokio::test(flavor = "current_thread")] - async fn job_manager_all_terminal_dependencies_run_after_failed_dependency() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut cleanup = job("cleanup", "printf cleanup"); - cleanup.depends_on = vec![JobDependency::job_id("fails")]; - cleanup.dependency_policy = JobDependencyPolicy::AllTerminal; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "cleanup".to_owned(), - jobs: vec![job("fails", "exit 2"), cleanup], - }) + async fn a_resubmitted_job_id_is_refused_and_an_identical_retry_replays() { + let fixture = fixture(); + let script = "echo run >> runs.txt"; + fixture + .manager + .start(&ctx(), start("first", vec![job("build", script)])) .await .expect("start"); + finished(&fixture.manager, &["build"]).await; - let read = wait_for_jobs(&manager, ["fails", "cleanup"]).await; - assert_eq!(result(&read, "fails").summary.status, JobStatus::Failed); - assert_eq!( - result(&read, "cleanup").summary.status, - JobStatus::Succeeded - ); + let replay = fixture + .manager + .start(&ctx(), start("first", vec![job("build", script)])) + .await + .expect("replay"); + assert_eq!(replay.jobs[0].status, JobStatus::Succeeded); + + let error = fixture + .manager + .start(&ctx(), start("second", vec![job("build", script)])) + .await + .unwrap_err(); + assert_eq!(precondition(&error), Some(PreconditionReason::JobIdInUse)); + let error = fixture + .manager + .start(&ctx(), start("first", vec![job("build", "echo changed")])) + .await + .unwrap_err(); + assert_eq!(precondition(&error), Some(PreconditionReason::JobIdInUse)); + let error = fixture + .manager + .start( + &ctx(), + start("first", vec![job("build", script), job("new", "true")]), + ) + .await + .unwrap_err(); + assert_eq!(precondition(&error), Some(PreconditionReason::JobIdInUse)); + + let runs = std::fs::read_to_string(fixture.root.join("runs.txt")).unwrap(); + assert_eq!(runs, "run\n", "nothing ran twice"); } #[tokio::test(flavor = "current_thread")] - async fn job_manager_cancels_running_jobs_and_queued_dependents() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut dependent = job("after", "printf after"); - dependent.depends_on = vec![JobDependency::job_id("sleep")]; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "cancel".to_owned(), - jobs: vec![job("sleep", "sleep 5"), dependent], - }) + async fn large_input_never_keeps_a_timeout_from_firing() { + // The child never reads its input, so the timeout must be armed + // before the input is written or the job runs forever. + let fixture = fixture(); + let mut stuck = job("stuck", "sleep 30"); + stuck.command.input = ExecInput::Bytes { + data: ByteChunk::from(vec![b'x'; 1024 * 1024]), + }; + stuck.command.timeout_ms = Some(300); + let mut echo = job("echo", "cat"); + let input: Vec = (0..512 * 1024u32).map(|i| b'a' + (i % 26) as u8).collect(); + echo.command.input = ExecInput::Bytes { + data: ByteChunk::from(input.clone()), + }; + fixture + .manager + .start(&ctx(), start("r", vec![stuck, echo])) .await .expect("start"); - - tokio::time::sleep(Duration::from_millis(50)).await; - manager - .cancel_jobs(CancelJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - jobs: vec![JobId::new("sleep")], - scope: JobCancelScope::Dependents, - force: false, - }) - .await - .expect("cancel"); - let read = wait_for_jobs(&manager, ["sleep", "after"]).await; - assert_eq!(result(&read, "sleep").summary.status, JobStatus::Cancelled); - assert_eq!(result(&read, "after").summary.status, JobStatus::Cancelled); + let jobs = finished(&fixture.manager, &["stuck", "echo"]).await; + assert_eq!(jobs[0].0.status, JobStatus::TimedOut); + assert_eq!(jobs[0].0.stopped_by, Some(StoppedBy::Timeout)); + assert_eq!(jobs[1].0.status, JobStatus::Succeeded); + assert_eq!(text(&jobs[1].1).as_bytes(), input.as_slice()); } #[tokio::test(flavor = "current_thread")] - async fn job_manager_interrupts_jobs_and_rejects_starts_after_close() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut dependent = job("after-close", "printf should-not-run"); - dependent.depends_on = vec![JobDependency::job_id("active-at-close")]; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "close-active".to_owned(), - jobs: vec![job("active-at-close", "sleep 5"), dependent], - }) + async fn verbose_output_keeps_head_and_tail_and_reads_back_after_a_restart() { + let cap = 64 * 1024; + let fixture = fixture_with(JobLimits { + max_retained_output_bytes: cap, + ..limits() + }); + fixture + .manager + .start( + &ctx(), + start( + "r", + vec![job( + "verbose", + "i=0; while [ $i -lt 30000 ]; do echo line-$i; i=$((i+1)); done; echo 'FAILED: 3 tests'", + )], + ), + ) .await .expect("start"); - - tokio::time::sleep(Duration::from_millis(50)).await; - manager.interrupt_all().await.expect("interrupt all"); - let read = wait_for_jobs(&manager, ["active-at-close", "after-close"]).await; - assert_eq!( - result(&read, "active-at-close").summary.status, - JobStatus::Interrupted - ); - assert_eq!( - result(&read, "after-close").summary.status, - JobStatus::Interrupted - ); - - let error = manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "after-close".to_owned(), - jobs: vec![job("rejected", "printf should-not-run")], - }) + let jobs = finished(&fixture.manager, &["verbose"]).await; + let (summary, page) = &jobs[0]; + assert_eq!(summary.status, JobStatus::Succeeded); + assert_eq!(page.gaps.len(), 1); + assert_eq!(page.gaps[0].reason, GapReason::Overflow); + assert_eq!(page.gaps[0].offset, cap / 2); + let shown = text(page); + assert!(shown.starts_with("line-0\n")); + assert!(shown.ends_with("FAILED: 3 tests\n")); + assert_eq!(shown.len() as u64 + page.gaps[0].omitted_bytes, page.end); + + // Head and tail in one read, as the job service asks for them. + let response = fixture + .manager + .read( + &ctx(), + ReadJobsParams { + jobs: vec![ + JobReadRequest { + max_bytes: Some(10), + ..entry("verbose") + }, + JobReadRequest { + tail_bytes: Some(16), + ..entry("verbose") + }, + ], + max_bytes: 26, + wait_ms: 0, + return_on: ReturnOn::Exit, + }, + ) .await - .expect_err("closed manager must reject starts"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::Conflict); + .expect("head and tail"); + let pages: Vec = response + .jobs + .into_iter() + .map(|entry| match entry { + JobReadEntry::Job { output, .. } => output, + JobReadEntry::Error { error, .. } => panic!("{error}"), + }) + .collect(); + assert_eq!(text(&pages[0]), "line-0\nlin"); + assert_eq!(text(&pages[1]), "FAILED: 3 tests\n"); + + let restarted = manager_at( + &fixture.root, + &fixture.state_dir, + JobLimits { + max_retained_output_bytes: cap, + ..limits() + }, + ); + let reloaded = finished(&restarted, &["verbose"]).await; + assert_eq!(reloaded[0].0.status, JobStatus::Succeeded); + let reread = &reloaded[0].1; + assert_eq!(text(reread), shown, "the output reads back from disk"); + assert_eq!(reread.gaps, page.gaps); + assert_eq!(reread.end, page.end); } - #[cfg(unix)] - async fn wait_for_process_gone(pid: i32) { - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - if rustix::process::test_kill_process( - rustix::process::Pid::from_raw(pid).expect("valid child pid"), + #[tokio::test(flavor = "current_thread")] + async fn pages_continue_exactly_where_they_stopped() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + start( + "r", + vec![job( + "page", + "i=0; while [ $i -lt 2000 ]; do printf '%09d\\n' $i; i=$((i+1)); done", + )], + ), ) - .is_err() - { - return; + .await + .expect("start"); + finished(&fixture.manager, &["page"]).await; + let mut after = 0; + let mut collected = String::new(); + loop { + let response = fixture + .manager + .read( + &ctx(), + ReadJobsParams { + jobs: vec![JobReadRequest { + after: Some(after), + ..entry("page") + }], + max_bytes: 100, + wait_ms: 0, + return_on: ReturnOn::Exit, + }, + ) + .await + .expect("page"); + let JobReadEntry::Job { output, .. } = &response.jobs[0] else { + panic!("job read failed"); + }; + collected.push_str(&text(output)); + if output.next == output.end { + break; } - assert!( - std::time::Instant::now() < deadline, - "orphaned descendant {pid} was not terminated" - ); - tokio::time::sleep(Duration::from_millis(20)).await; + assert!(output.next > after); + after = output.next; } + assert_eq!(collected.len(), 20_000, "no byte of 20 000 was skipped"); + assert!(collected.ends_with("000001999\n")); } - #[cfg(unix)] - fn read_pid_file(path: &std::path::Path) -> i32 { - std::fs::read_to_string(path) - .expect("orphan pid file") - .trim() - .parse() - .expect("orphan pid") - } - - #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn job_manager_completes_when_orphaned_descendants_hold_pipes() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "orphan".to_owned(), - jobs: vec![job( - "orphan", - "sleep 30 & echo $! > orphan.pid; printf started; exit 3", - )], - }) + async fn a_read_of_more_jobs_than_the_cache_holds_returns_every_output() { + let fixture = fixture(); + let count = MAX_CACHED_OUTPUTS + 4; + let names: Vec = (0..count).map(|index| format!("job-{index}")).collect(); + fixture + .manager + .start( + &ctx(), + start( + "r", + names + .iter() + .map(|name| job(name, &format!("printf {name}"))) + .collect(), + ), + ) .await .expect("start"); - - let read = wait_for_jobs(&manager, ["orphan"]).await; - let result = result(&read, "orphan"); - assert_eq!(result.summary.status, JobStatus::Failed); - assert_eq!(result.summary.exit_code, Some(3)); - assert!(result.summary.orphaned_descendants); - let output = result - .output_chunks - .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) - .collect::>(); - assert_eq!(output, b"started"); - - wait_for_process_gone(read_pid_file(&root.join("orphan.pid"))).await; + let ids: Vec<&str> = names.iter().map(String::as_str).collect(); + finished(&fixture.manager, &ids).await; + + // A restarted daemon holds no output in memory, so every log is + // loaded from disk during this one read. + let restarted = manager_at(&fixture.root, &fixture.state_dir, limits()); + for manager in [&fixture.manager, &restarted] { + let jobs = finished(manager, &ids).await; + for (name, (_, page)) in names.iter().zip(&jobs) { + assert_eq!(&text(page), name); + assert_eq!(page.next, page.end); + } + } } - #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn job_manager_completes_when_escaped_descendants_hold_pipes() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - // `set -m` puts the background child in its own process group, so the - // group sweep cannot reach it and completion must abandon the pipes. - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "escapee".to_owned(), - jobs: vec![job( - "escapee", - "set -m; sleep 30 & echo $! > escapee.pid; printf started; exit 7", - )], - }) + async fn unknown_jobs_are_per_entry_not_found() { + let fixture = fixture(); + let response = fixture + .manager + .read( + &ctx(), + ReadJobsParams { + jobs: vec![entry("missing")], + max_bytes: 0, + wait_ms: 1_000, + return_on: ReturnOn::Exit, + }, + ) .await - .expect("start"); - - let read = wait_for_jobs(&manager, ["escapee"]).await; - let result = result(&read, "escapee"); - assert_eq!(result.summary.status, JobStatus::Failed); - assert_eq!(result.summary.exit_code, Some(7)); - assert!(result.summary.orphaned_descendants); - - // The escapee is outside the job's process group; clean it up so it - // does not outlive the test. - let pid = read_pid_file(&root.join("escapee.pid")); - let _ = rustix::process::kill_process( - rustix::process::Pid::from_raw(pid).expect("valid child pid"), - rustix::process::Signal::KILL, - ); + .expect("read"); + match &response.jobs[0] { + JobReadEntry::Error { job_id, error } => { + assert_eq!(job_id, &id("missing")); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + } + JobReadEntry::Job { .. } => panic!("no summary is invented"), + } } + #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn job_manager_cancel_terminates_process_group() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "cancel-group".to_owned(), - jobs: vec![job("group", "sleep 30 & sleep 30")], - }) + async fn cancel_is_graceful_unless_forced() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + start( + "r", + vec![ + job( + "graceful", + "trap 'echo trapped; exit 0' TERM; sleep 30 & wait", + ), + job( + "forced", + "trap 'echo trapped; exit 0' TERM; sleep 30 & wait", + ), + job("queued", "true"), + ] + .into_iter() + .enumerate() + .map(|(index, mut spec)| { + if index == 2 { + spec.depends_on = vec![JobDependency::JobId { + job_id: id("forced"), + }]; + } + spec + }) + .collect(), + ), + ) .await .expect("start"); - - tokio::time::sleep(Duration::from_millis(50)).await; - manager - .cancel_jobs(CancelJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - jobs: vec![JobId::new("group")], - scope: JobCancelScope::Job, - force: false, - }) + tokio::time::sleep(Duration::from_millis(200)).await; + fixture + .manager + .cancel( + &ctx(), + CancelJobsParams { + job_ids: vec![id("graceful")], + scope: JobCancelScope::Job, + force: false, + }, + ) .await .expect("cancel"); - - // The backgrounded child holds the job's pipes; without group - // termination the drain would wedge until the read wait expires. - let read = wait_for_jobs(&manager, ["group"]).await; - let result = result(&read, "group"); - assert_eq!(result.summary.status, JobStatus::Cancelled); - assert!(!result.summary.orphaned_descendants); - } - - #[tokio::test(flavor = "current_thread")] - async fn job_manager_timeout_terminates_process_group() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - let mut spec = job("timeout-group", "sleep 30 & sleep 30"); - spec.timeout_ms = Some(50); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "timeout-group".to_owned(), - jobs: vec![spec], - }) + let cancelled = fixture + .manager + .cancel( + &ctx(), + CancelJobsParams { + job_ids: vec![id("forced"), id("missing")], + scope: JobCancelScope::Dependents, + force: true, + }, + ) .await - .expect("start"); - - let read = wait_for_jobs(&manager, ["timeout-group"]).await; + .expect("force cancel"); + assert_eq!(cancelled.jobs.len(), 3, "named jobs, then the dependent"); + assert!( + matches!(&cancelled.jobs[1], JobSummaryEntry::Error { error, .. } if error.kind() == ProtocolErrorKind::NotFound) + ); + let jobs = finished(&fixture.manager, &["graceful", "forced", "queued"]).await; + assert_eq!(jobs[0].0.status, JobStatus::Cancelled); + assert_eq!(jobs[0].0.stopped_by, Some(StoppedBy::Terminate)); + assert!(jobs[0].0.cancel_requested_at_ms.is_some()); assert_eq!( - result(&read, "timeout-group").summary.status, - JobStatus::TimedOut + text(&jobs[0].1), + "trapped\n", + "SIGTERM let the job clean up" ); + assert_eq!(jobs[1].0.status, JobStatus::Cancelled); + assert_eq!(jobs[1].0.stopped_by, Some(StoppedBy::Kill)); + assert_eq!(text(&jobs[1].1), "", "a forced cancel kills at once"); + assert_eq!(jobs[2].0.status, JobStatus::Cancelled); + assert_eq!(jobs[2].0.started_at_ms, None); } #[tokio::test(flavor = "current_thread")] - async fn job_manager_times_out_jobs() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - let mut spec = job("timeout", "sleep 5"); - spec.timeout_ms = Some(50); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "timeout".to_owned(), - jobs: vec![spec], - }) + async fn dependencies_and_queues_order_jobs() { + let fixture = fixture(); + let mut first = job("first", "sleep 0.2; echo first >> order.txt"); + first.name = Some(JobName::new("first job").unwrap()); + first.queue_key = Some(QueueKey::new("q").unwrap()); + let mut second = job("second", "echo second >> order.txt"); + second.queue_key = Some(QueueKey::new("q").unwrap()); + let mut after = job("after", "echo after >> order.txt; exit 1"); + after.depends_on = vec![JobDependency::Name { + name: JobName::new("first job").unwrap(), + }]; + let mut doomed = job("doomed", "echo doomed >> order.txt"); + doomed.depends_on = vec![JobDependency::JobId { + job_id: id("after"), + }]; + let mut tolerant = job("tolerant", "echo tolerant >> order.txt"); + tolerant.depends_on = vec![JobDependency::JobId { + job_id: id("after"), + }]; + tolerant.dependency_policy = JobDependencyPolicy::AllTerminal; + fixture + .manager + .start( + &ctx(), + start("r", vec![first, second, after, doomed, tolerant]), + ) .await .expect("start"); - - let read = wait_for_jobs(&manager, ["timeout"]).await; - assert_eq!(result(&read, "timeout").summary.status, JobStatus::TimedOut); - } - - #[tokio::test(flavor = "current_thread")] - async fn job_manager_retries_same_start_idempotently_and_rejects_conflict() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - let params = StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "retry".to_owned(), - jobs: vec![job("retry", "printf once")], - }; - - manager.start_jobs(params.clone()).await.expect("first"); - manager.start_jobs(params).await.expect("retry"); - let conflict = manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "retry-conflict".to_owned(), - jobs: vec![job("retry", "printf different")], - }) + let jobs = finished( + &fixture.manager, + &["first", "second", "after", "doomed", "tolerant"], + ) + .await; + assert_eq!(jobs[3].0.status, JobStatus::DependencyFailed); + assert_eq!(jobs[3].0.started_at_ms, None); + assert_eq!(jobs[4].0.status, JobStatus::Succeeded); + assert_eq!(jobs[2].0.depends_on, vec![id("first")]); + let order = std::fs::read_to_string(fixture.root.join("order.txt")).unwrap(); + let lines: Vec<&str> = order.lines().collect(); + assert_eq!(lines[0], "first", "one job per queue key, in start order"); + assert!(lines.iter().position(|line| *line == "second") > Some(0)); + assert!(!lines.contains(&"doomed")); + + let mut cyclic = job("a", "true"); + cyclic.depends_on = vec![JobDependency::JobId { job_id: id("b") }]; + let mut back = job("b", "true"); + back.depends_on = vec![JobDependency::JobId { job_id: id("a") }]; + let error = fixture + .manager + .start(&ctx(), start("cycle", vec![cyclic, back])) .await - .expect_err("conflict"); - assert_eq!(conflict.code, EnvironmentProtocolErrorCode::Conflict); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + let mut piped = job("piped", "cat"); + piped.command.input = ExecInput::Pipe {}; + let error = fixture + .manager + .start(&ctx(), start("pipe", vec![piped])) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); } + #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn job_manager_lists_latest_jobs_with_limit() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "list-1".to_owned(), - jobs: vec![job("job-old", "printf old")], - }) - .await - .expect("start old"); - tokio::time::sleep(Duration::from_millis(2)).await; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "list-2".to_owned(), - jobs: vec![job("job-new", "printf new")], - }) - .await - .expect("start new"); - - let listed = manager - .list_jobs(ListJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - limit: Some(1), - }) + async fn leftovers_are_swept_and_reported() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + start( + "r", + vec![job("leaky", "sleep 30 & echo $! > orphan.pid; exit 0")], + ), + ) .await - .expect("list jobs"); - - assert_eq!(listed.jobs.len(), 1); - assert_eq!(listed.jobs[0].job_id.as_str(), "job-new"); + .expect("start"); + let jobs = finished(&fixture.manager, &["leaky"]).await; + assert_eq!(jobs[0].0.status, JobStatus::Succeeded); + let leftovers = jobs[0] + .0 + .leftovers + .as_ref() + .expect("swept leftovers reported"); + assert_eq!(leftovers.disposition, LeftoverDisposition::Swept); + let orphan: i32 = std::fs::read_to_string(fixture.root.join("orphan.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let alive = rustix::process::Pid::from_raw(orphan) + .is_some_and(|pid| rustix::process::test_kill_process(pid).is_ok()); + assert!(!alive, "the group was swept"); } + #[cfg(any(target_os = "linux", target_os = "macos"))] #[tokio::test(flavor = "current_thread")] - async fn job_manager_marks_unfinished_persisted_jobs_interrupted_on_startup() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let jobs_root = root.join(".lightspeed").join("jobs"); - std::fs::create_dir_all(&jobs_root).expect("jobs root"); - persist_record_at( - &jobs_root, - &JobRecord { - namespace: TEST_NAMESPACE.to_owned(), - job_id: JobId::new("recover"), - name: None, - argv: vec!["sleep".to_owned(), "5".to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env_names: Vec::new(), - stdin: None, - timeout_ms: None, - dependency_policy: JobDependencyPolicy::AllSucceeded, - dependencies: Vec::new(), - queue_key: None, - spec_hash: "seed".to_owned(), - accept_seq: 0, - status: JobStatus::Running, - created_at_ms: now_ms(), - queued_at_ms: Some(now_ms()), - started_at_ms: Some(now_ms()), - finished_at_ms: None, - exit_code: None, - orphaned_descendants: false, - failure: None, - output_chunks: Vec::new(), - output_next_seq: 0, - }, - ) - .expect("persist seed"); - - let recovered = JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")) - .expect("recovered manager"); - let read = recovered - .read_jobs(ReadJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - jobs: vec![JobId::new("recover")], - after_seq: None, - max_bytes: None, - include_artifacts: false, - wait_ms: None, - }) + async fn a_restart_interrupts_unfinished_jobs_and_kills_their_groups() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + start("r", vec![job("long", "echo $$ > root.pid; sleep 30")]), + ) .await - .expect("read"); + .expect("start"); + let pid_file = fixture.root.join("root.pid"); + for _ in 0..250 { + if std::fs::read_to_string(&pid_file).is_ok_and(|pid| !pid.trim().is_empty()) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let root: i32 = std::fs::read_to_string(&pid_file) + .unwrap() + .trim() + .parse() + .unwrap(); + // The running record reaches the disk through the store thread. + tokio::time::sleep(Duration::from_millis(100)).await; + let restarted = manager_at(&fixture.root, &fixture.state_dir, limits()); + let jobs = finished(&restarted, &["long"]).await; + assert_eq!(jobs[0].0.status, JobStatus::Interrupted); assert_eq!( - result(&read, "recover").summary.status, - JobStatus::Interrupted + jobs[0].0.error.as_ref().map(ProtocolError::kind), + Some(ProtocolErrorKind::Aborted) ); - } - - fn job(job_id: &str, shell: &str) -> JobStartSpec { - named_job(job_id, job_id, shell) - } - - fn named_job(job_id: &str, name: &str, shell: &str) -> JobStartSpec { - JobStartSpec { - job_id: JobId::new(job_id), - name: Some(name.to_owned()), - argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), shell.to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(5_000), - depends_on: Vec::new(), - dependency_policy: JobDependencyPolicy::AllSucceeded, - queue_key: None, + for _ in 0..100 { + let alive = rustix::process::Pid::from_raw(root) + .is_some_and(|pid| rustix::process::test_kill_process(pid).is_ok()); + if !alive { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; } + panic!("the earlier instance's job group survived the restart"); } - async fn wait_for_jobs( - manager: &JobManager, - job_ids: [&str; N], - ) -> ReadJobsResponse { - manager - .read_jobs(ReadJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - jobs: job_ids.iter().map(|job_id| JobId::new(*job_id)).collect(), - after_seq: None, - max_bytes: None, - include_artifacts: false, - wait_ms: Some(5_000), - }) - .await - .expect("read jobs") + #[tokio::test(flavor = "current_thread")] + async fn unreadable_records_are_quarantined_instead_of_failing_startup() { + let fixture = fixture(); + let bad = fixture.state_dir.join("jobs").join("broken"); + std::fs::create_dir_all(&bad).unwrap(); + std::fs::write(bad.join("record.json"), b"{not json").unwrap(); + std::fs::write(fixture.state_dir.join("jobs").join("stray.json"), b"{}").unwrap(); + let restarted = manager_at(&fixture.root, &fixture.state_dir, limits()); + assert_eq!(restarted.running_count().await, 0); + assert!(!bad.exists()); + let quarantined = std::fs::read_dir(fixture.state_dir.join("jobs-quarantine")) + .unwrap() + .count(); + assert_eq!(quarantined, 2); } - fn result<'a>(response: &'a ReadJobsResponse, job_id: &str) -> &'a JobReadResult { - response - .jobs - .iter() - .find(|result| result.summary.job_id.as_str() == job_id) - .expect("job result") + #[tokio::test(flavor = "current_thread")] + async fn request_bounds_are_ceilings_not_clamps() { + let fixture = fixture_with(JobLimits { + max_request_jobs: 2, + ..limits() + }); + let error = fixture + .manager + .start( + &ctx(), + start( + "r", + vec![job("a", "true"), job("b", "true"), job("c", "true")], + ), + ) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::InvalidParams { + ceiling: Some(2), + .. + } + )); + let error = fixture + .manager + .read( + &ctx(), + ReadJobsParams { + jobs: vec![JobReadRequest { + max_bytes: Some(20), + ..entry("a") + }], + max_bytes: 10, + wait_ms: 0, + return_on: ReturnOn::Exit, + }, + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + let error = fixture + .manager + .cancel( + &ctx(), + CancelJobsParams { + job_ids: Vec::new(), + scope: JobCancelScope::Job, + force: false, + }, + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); } } diff --git a/crates/environment-daemon/src/lib.rs b/crates/environment-daemon/src/lib.rs index 717fc7c14..0ce58c8f3 100644 --- a/crates/environment-daemon/src/lib.rs +++ b/crates/environment-daemon/src/lib.rs @@ -1,4 +1,6 @@ pub mod config; +pub mod dispatch; +mod exec; pub mod filesystem; pub mod identity; pub mod jobs; @@ -6,7 +8,6 @@ pub mod process; mod process_group; mod redaction; pub mod registration; -pub mod rpc; pub mod server; pub mod upgrade; @@ -18,27 +19,59 @@ use std::{ time::Instant, }; +use anyhow::Context as _; use environment_protocol::{ - data::idle::IdleResponse, - shared::{ - CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentConnectionId, - ImplementationInfo, + ENVD_PROTOCOL_VERSION, + bootstrap::BuildFacts, + bytes::encode_hex, + data::{ + handshake::{ + DataCapabilities, DataLimits, FilesystemCapabilities, FilesystemLimits, GlobCeilings, + InitializeResponse, JobCapabilities, JobLimits, ProcessCapabilities, ProcessLimits, + ScanCeilings, SearchCeilings, TransferCeilings, + }, + idle::GetIdleResponse, }, + error::{Capability, ProtocolError}, + handshake::{ImplementationInfo, Plane}, + ids::DaemonInstanceId, + paths::EnvironmentPath, }; +use tokio_util::sync::CancellationToken; use crate::{ - config::DaemonConfig, filesystem::LocalFileSystem, jobs::JobManager, process::ProcessManager, + config::DaemonConfig, + filesystem::{Confinement, FilesystemConfig, LocalFileSystem}, + jobs::{JobConfig, JobManager}, + process::{ProcessConfig, ProcessManager}, }; +/// Product name in `serverInfo`, the registration `implementation` and +/// `--print-build`. +pub const DAEMON_NAME: &str = "lightspeed-envd"; + +/// Everything one daemon serves. Cheap to clone; every clone shares the +/// same managers, activity clock and shutdown token. #[derive(Clone)] pub struct DaemonRuntime { - config: Arc, - capabilities: EnvironmentCapabilities, + inner: Arc, +} + +struct RuntimeInner { + config: DaemonConfig, + instance_id: DaemonInstanceId, + server_info: ImplementationInfo, + capabilities: DataCapabilities, + limits: DataLimits, + default_cwd: EnvironmentPath, + home_directory: Option, + confinement: Arc, filesystem: LocalFileSystem, processes: ProcessManager, jobs: JobManager, - next_connection_id: Arc, - activity: Arc, + next_connection_id: AtomicU64, + activity: ActivityClock, + shutdown: CancellationToken, } /// Monotonic record of the last data-plane activity. Deliberately not a @@ -56,11 +89,17 @@ impl ActivityClock { } pub fn touch(&self) { - *self.last.lock().expect("activity clock poisoned") = Instant::now(); + *self + .last + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Instant::now(); } pub fn idle_for(&self) -> std::time::Duration { - self.last.lock().expect("activity clock poisoned").elapsed() + self.last + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .elapsed() } } @@ -80,304 +119,502 @@ pub fn install_crypto_provider() { } impl DaemonRuntime { - pub fn new(config: DaemonConfig) -> anyhow::Result { + /// Build the managers over one shared [`Confinement`]. A missing state + /// directory is created `0700`. + pub fn new(mut config: DaemonConfig) -> anyhow::Result { install_crypto_provider(); - let capabilities = environment_capabilities(&config); - let filesystem = LocalFileSystem::new( - config.fs_root.clone(), - config.cwd.clone(), - !config.read_only_fs, - ) - .with_transfer_state_dir(config.state_dir.join("transfers")) - .map_err(|e| anyhow::anyhow!("transfer state: {e:?}"))?; - let processes = ProcessManager::new(config.cwd.clone(), config.fs_root.clone()) - .with_scrubbed_env(config.scrubbed_env.clone()); - let jobs = JobManager::new( - config.cwd.clone(), - config.fs_root.clone(), - config.state_dir.clone(), - )? - .with_scrubbed_env(config.scrubbed_env.clone()); + config.state_dir = config::prepare_state_dir(&config.state_dir)?; + let default_cwd = environment_path(&config.cwd).context("cwd")?; + let home_directory = home_directory(&config); + let instance_id = mint_instance_id()?; + let capabilities = data_capabilities(&config); + let limits = data_limits(); + let confinement = Arc::new( + Confinement::new(config.fs_root.clone(), config.state_dir.clone()) + .map_err(|error| anyhow::anyhow!("filesystem confinement: {error}"))?, + ); + let filesystem = LocalFileSystem::new(FilesystemConfig { + confinement: confinement.clone(), + cwd: config.cwd.clone(), + writable: !config.read_only_fs, + limits: limits.filesystem.clone(), + }) + .map_err(|error| anyhow::anyhow!("filesystem: {error}"))?; + let processes = ProcessManager::new(ProcessConfig { + cwd: config.cwd.clone(), + confinement: confinement.clone(), + state_dir: config.state_dir.clone(), + scrubbed_env: config.scrubbed_env.clone(), + instance_id: instance_id.clone(), + limits: limits.process.clone(), + max_wait_ms: limits.max_wait_ms, + pty_supported: PTY_SUPPORTED, + }) + .context("process manager")?; + let jobs = JobManager::new(JobConfig { + cwd: config.cwd.clone(), + confinement: confinement.clone(), + state_dir: config.state_dir.clone(), + scrubbed_env: config.scrubbed_env.clone(), + instance_id: instance_id.clone(), + limits: limits.job.clone(), + max_wait_ms: limits.max_wait_ms, + }) + .context("job manager")?; Ok(Self { - config: Arc::new(config), - capabilities, - filesystem, - processes, - jobs, - next_connection_id: Arc::new(AtomicU64::new(1)), - activity: Arc::new(ActivityClock::new()), + inner: Arc::new(RuntimeInner { + config, + instance_id, + server_info: server_info(), + capabilities, + limits, + default_cwd, + home_directory, + confinement, + filesystem, + processes, + jobs, + next_connection_id: AtomicU64::new(1), + activity: ActivityClock::new(), + shutdown: CancellationToken::new(), + }), }) } /// Record one unit of data-plane activity. pub fn touch_activity(&self) { - self.activity.touch(); + self.inner.activity.touch(); } - /// Idle report for the power reaper: zero idle time while any process or - /// job is still executing, otherwise the time since the last request. - /// Live work keeps the clock fresh so a long-running job that finishes + /// `env/getIdle`: zero idle time while any process or job is still + /// executing, otherwise the time since the last `use` request. Live + /// work keeps the clock fresh so a long-running job that finishes /// starts the idle countdown from its end, not its start. Leftover /// process groups are counted but are not work: a server waiting for /// requests must not keep the environment awake. - pub async fn idle_report(&self) -> IdleResponse { - let running_processes = self.processes.running_count().await; - let running_jobs = self.jobs.running_count().await; - let leftover_process_groups = self.processes.leftover_group_count(); - if running_processes > 0 || running_jobs > 0 { - self.activity.touch(); - return IdleResponse { - idle_for_ms: 0, - running_processes, - running_jobs, - leftover_process_groups, - }; + pub async fn idle_report(&self) -> GetIdleResponse { + let running_process_count = self.inner.processes.running_count().await; + let running_job_count = self.inner.jobs.running_count().await; + let leftover_process_group_count = self.inner.processes.leftover_group_count(); + if running_process_count > 0 || running_job_count > 0 { + self.inner.activity.touch(); } - IdleResponse { - idle_for_ms: u64::try_from(self.activity.idle_for().as_millis()).unwrap_or(u64::MAX), - running_processes, - running_jobs, - leftover_process_groups, + let idle_for_ms = if running_process_count > 0 || running_job_count > 0 { + 0 + } else { + u64::try_from(self.inner.activity.idle_for().as_millis()).unwrap_or(u64::MAX) + }; + GetIdleResponse { + idle_for_ms, + running_process_count, + running_job_count, + leftover_process_group_count, + } + } + + /// The data-plane `initialize` response; identical on every connection + /// of this daemon instance. + pub fn initialize_response(&self) -> InitializeResponse { + InitializeResponse { + protocol_version: ENVD_PROTOCOL_VERSION, + plane: Plane::Data, + daemon_instance_id: self.inner.instance_id.clone(), + server_info: self.inner.server_info.clone(), + capabilities: self.inner.capabilities.clone(), + limits: self.inner.limits.clone(), + default_cwd: self.inner.default_cwd.clone(), + home_directory: self.inner.home_directory.clone(), } } + /// `capabilityUnavailable` unless this daemon serves `capability`. + pub fn require_capability(&self, capability: Capability) -> Result<(), ProtocolError> { + if dispatch::has_capability(&self.inner.capabilities, capability) { + return Ok(()); + } + Err(ProtocolError::capability_unavailable( + capability, + format!("this daemon does not serve {}", capability_name(capability)), + )) + } + pub fn config(&self) -> &DaemonConfig { - &self.config + &self.inner.config + } + + pub fn instance_id(&self) -> &DaemonInstanceId { + &self.inner.instance_id + } + + pub fn server_info(&self) -> &ImplementationInfo { + &self.inner.server_info + } + + pub fn capabilities(&self) -> &DataCapabilities { + &self.inner.capabilities + } + + pub fn limits(&self) -> &DataLimits { + &self.inner.limits } - pub fn implementation(&self) -> ImplementationInfo { - build_info().into() + pub fn confinement(&self) -> &Arc { + &self.inner.confinement } - pub fn capabilities(&self) -> EnvironmentCapabilities { - self.capabilities.clone() + /// Daemon-unique, increasing connection number. + pub fn next_connection_id(&self) -> u64 { + self.inner + .next_connection_id + .fetch_add(1, Ordering::Relaxed) } - pub fn next_connection_id(&self) -> EnvironmentConnectionId { - let id = self.next_connection_id.fetch_add(1, Ordering::Relaxed); - EnvironmentConnectionId::new(format!("envd-{id}")) + /// Cancelled when the daemon begins a deliberate shutdown: listeners stop + /// accepting, data connections end and the registration channel says + /// goodbye. + pub fn shutdown(&self) -> &CancellationToken { + &self.inner.shutdown } pub fn filesystem(&self) -> &LocalFileSystem { - &self.filesystem + &self.inner.filesystem } pub fn processes(&self) -> &ProcessManager { - &self.processes + &self.inner.processes } pub fn jobs(&self) -> &JobManager { - &self.jobs + &self.inner.jobs } } -fn environment_capabilities(config: &DaemonConfig) -> EnvironmentCapabilities { - EnvironmentCapabilities { - filesystem_read: true, - filesystem_write: !config.read_only_fs, - filesystem_search: true, - filesystem_glob: true, - filesystem_ranged_read: true, - filesystem_transfer: cfg!(any(target_os = "linux", target_os = "macos")), - filesystem_scan: cfg!(any(target_os = "linux", target_os = "macos")), - filesystem_capture: cfg!(any(target_os = "linux", target_os = "macos")), - filesystem_materialize: cfg!(any(target_os = "linux", target_os = "macos")) - && !config.read_only_fs, - process_start: true, - process_stdin: true, - process_terminate: true, - process_output_polling: true, - process_output_notifications: false, - process_pty: true, - job_start: true, - job_list: true, - job_read: true, - job_cancel: true, - job_wait_hint: false, - job_dependencies: true, - job_queue_keys: true, - network: true, +/// PTYs are served wherever `portable-pty` has a Unix backend. +const PTY_SUPPORTED: bool = cfg!(unix); + +fn data_capabilities(config: &DaemonConfig) -> DataCapabilities { + DataCapabilities { + filesystem: Some(FilesystemCapabilities { + write: !config.read_only_fs, + }), + process: Some(ProcessCapabilities { pty: PTY_SUPPORTED }), + job: Some(JobCapabilities {}), } } -pub fn protocol_version() -> u32 { - CURRENT_PROTOCOL_VERSION +const KIB: u64 = 1024; +const MIB: u64 = 1024 * KIB; +const GIB: u64 = 1024 * MIB; +const TIB: u64 = 1024 * GIB; + +/// The ceilings envd enforces and advertises. Every response they allow +/// fits below `MAX_MESSAGE_BYTES` after base64. +pub fn data_limits() -> DataLimits { + DataLimits { + max_concurrent_requests: 32, + max_wait_ms: 1_800_000, + process: ProcessLimits { + max_read_bytes: 4 * MIB, + max_retained_output_bytes: MIB, + retention_after_exit_ms: 60_000, + unread_retention_ms: 7_200_000, + max_exited_retained_bytes: 64 * MIB, + terminate_grace_ms: 5_000, + max_input_queue_bytes: MIB, + }, + job: JobLimits { + max_read_bytes: 4 * MIB, + max_request_jobs: 64, + max_retained_output_bytes: 2 * MIB, + record_retention_ms: 604_800_000, + cancel_grace_ms: 5_000, + }, + filesystem: FilesystemLimits { + max_read_file_bytes: 8 * MIB, + max_write_file_bytes: 8 * MIB, + max_directory_entries: 1_000, + search: SearchCeilings { + max_matches: 10_000, + max_files: 100_000, + max_bytes: GIB, + max_duration_ms: 60_000, + max_line_bytes: 4_096, + }, + glob: GlobCeilings { + max_matches: 100_000, + max_files: 1_000_000, + max_duration_ms: 60_000, + }, + scan: ScanCeilings { + max_roots: 32, + max_patterns: 32, + max_entries: 10_000, + max_file_bytes: 256 * KIB, + max_total_bytes: 4 * MIB, + max_duration_ms: 30_000, + }, + transfer: TransferCeilings { + max_active_operations: 8, + max_content_chunk_bytes: 4 * MIB, + max_inventory_page_entries: 1_000, + receipt_retention_ms: 86_400_000, + max_entries: 100_000, + max_depth: 64, + max_file_bytes: TIB, + max_total_bytes: TIB, + max_manifest_bytes: 32 * MIB, + max_duration_ms: 86_400_000, + }, + }, + } } -/// The facts this binary reports about itself: to the gateway in every -/// handshake, and to an operator through `--print-build`. The protocol -/// version is the one that decides whether a gateway admits the daemon; the -/// rest is provenance. -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] -#[serde(rename_all = "camelCase")] -pub struct BuildInfo { - pub name: &'static str, - pub version: &'static str, - pub git_sha: &'static str, - pub target: &'static str, - pub protocol_version: u32, +fn capability_name(capability: Capability) -> &'static str { + match capability { + Capability::Filesystem => "the filesystem family", + Capability::FilesystemWrite => "filesystem writes", + Capability::Process => "the process family", + Capability::ProcessPty => "process PTYs", + Capability::Job => "the job family", + Capability::AdoptTarget => "target adoption", + Capability::Ingress => "ingress", + } } -pub fn build_info() -> BuildInfo { - BuildInfo { - name: "lightspeed-envd", - version: release_info::VERSION, - git_sha: release_info::GIT_SHA, - target: release_info::TARGET, - protocol_version: CURRENT_PROTOCOL_VERSION, - } +/// `daemonInstanceId`: `envd_` and 32 random hex digits, minted per start. +fn mint_instance_id() -> anyhow::Result { + let mut bytes = [0u8; 16]; + rand::TryRng::try_fill_bytes(&mut rand::rngs::SysRng, &mut bytes) + .context("generate daemon instance id")?; + Ok(DaemonInstanceId::new(format!( + "envd_{}", + encode_hex(&bytes) + ))?) } -impl From for ImplementationInfo { - fn from(build: BuildInfo) -> Self { - ImplementationInfo { - name: build.name.to_owned(), - version: Some(build.version.to_owned()), - git_sha: Some(build.git_sha.to_owned()), - target: Some(build.target.to_owned()), - } - } +fn environment_path(path: &std::path::Path) -> anyhow::Result { + let text = path + .to_str() + .with_context(|| format!("{} is not UTF-8", path.display()))?; + Ok(EnvironmentPath::new(text)?) } -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; +/// `$HOME` when it is a valid path inside the filesystem root. +fn home_directory(config: &DaemonConfig) -> Option { + let home = std::env::var_os("HOME")?; + let home = std::path::PathBuf::from(home); + if !home.starts_with(&config.fs_root) { + return None; + } + environment_path(&home).ok() +} - use environment_protocol::{ - data::process::{ - ProcessSignal, ReadProcessParams, StartProcessParams, TerminateProcessParams, - }, - shared::ProcessId, +/// `serverInfo` and the registration `implementation`: this build's facts, +/// leaving out any the build left empty. +pub fn server_info() -> ImplementationInfo { + let info = ImplementationInfo::new(DAEMON_NAME).expect("the daemon name is a valid field"); + let info = match info.clone().with_version(release_info::VERSION) { + Ok(with) => with, + Err(_) => info, + }; + let info = match info.clone().with_git_sha(release_info::GIT_SHA) { + Ok(with) => with, + Err(_) => info, }; + match info.clone().with_target(release_info::TARGET) { + Ok(with) => with, + Err(_) => info, + } +} +/// What `--print-build` prints: the facts an orchestrator checks a +/// downloaded daemon against the deployment's discovery document with. +pub fn build_facts() -> BuildFacts { + BuildFacts { + name: DAEMON_NAME.to_owned(), + version: release_info::VERSION.to_owned(), + git_sha: release_info::GIT_SHA.to_owned(), + target: release_info::TARGET.to_owned(), + protocol_version: ENVD_PROTOCOL_VERSION, + } +} + +#[cfg(test)] +pub(crate) mod test_support { use super::*; - fn runtime() -> (tempfile::TempDir, DaemonRuntime) { + /// A runtime rooted in a fresh temporary directory, its state dir + /// beside the root rather than inside it. + pub(crate) fn runtime() -> (tempfile::TempDir, DaemonRuntime) { + runtime_with(|_| {}) + } + + pub(crate) fn runtime_with( + adjust: impl FnOnce(&mut DaemonConfig), + ) -> (tempfile::TempDir, DaemonRuntime) { let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let runtime = DaemonRuntime::new(DaemonConfig { - listen: Some("127.0.0.1:0".parse().expect("listen")), + let base = temp.path().canonicalize().expect("canonical temp"); + let root = base.join("root"); + let state = base.join("state"); + std::fs::create_dir_all(&root).expect("root"); + std::fs::create_dir_all(&state).expect("state"); + let mut config = DaemonConfig { + listen: None, cwd: root.clone(), - fs_root: root.clone(), - state_dir: root.join("state"), + fs_root: root, + state_dir: state, read_only_fs: false, registration: None, scrubbed_env: vec!["LIGHTSPEED_ENVD_REGISTRATION_KEY".to_owned()], - }) - .expect("runtime"); + }; + adjust(&mut config); + let runtime = DaemonRuntime::new(config).expect("runtime"); (temp, runtime) } +} + +#[cfg(test)] +mod tests { + use environment_protocol::{ + data::handshake::DataLimits, + error::ProtocolErrorKind, + limits::{MAX_MESSAGE_BYTES, MIN_CONCURRENT_REQUESTS}, + manifest::{EnvGetIdle, ProcessStart, ProcessTerminate}, + }; + use serde_json::json; + + use super::{test_support::runtime, *}; + use crate::dispatch::{RequestContext, handle_request}; + + async fn call( + runtime: &DaemonRuntime, + method: &str, + params: serde_json::Value, + ) -> serde_json::Value { + handle_request(runtime, &RequestContext::detached(), method, params) + .await + .unwrap_or_else(|error| panic!("{method}: {error}")) + .value + } + + #[test] + fn instance_ids_are_random_per_start_and_use_the_general_grammar() { + let first = mint_instance_id().unwrap(); + let second = mint_instance_id().unwrap(); + assert_ne!(first, second); + assert!(first.as_str().starts_with("envd_")); + assert_eq!(first.as_str().len(), "envd_".len() + 32); + } + + /// Every advertised ceiling keeps its worst-case response, base64 + /// included, below the message limit. + #[test] + fn advertised_payload_ceilings_fit_the_message_limit() { + let limits: DataLimits = data_limits(); + let base64 = |bytes: u64| bytes.div_ceil(3) * 4; + for (name, bytes) in [ + ("fs.readFile", limits.filesystem.max_read_file_bytes), + ("fs.writeFile", limits.filesystem.max_write_file_bytes), + ( + "transfer.chunk", + limits.filesystem.transfer.max_content_chunk_bytes, + ), + ("process.read", limits.process.max_read_bytes), + ("job.read", limits.job.max_read_bytes), + ] { + assert!(base64(bytes) < MAX_MESSAGE_BYTES * 3 / 4, "{name}"); + } + assert!(limits.max_concurrent_requests >= MIN_CONCURRENT_REQUESTS); + } #[tokio::test(flavor = "current_thread")] - async fn idle_report_counts_running_work_and_resets_on_activity() { + async fn initialize_reports_the_instance_cwd_and_capability_families() { let (_temp, runtime) = runtime(); - tokio::time::sleep(std::time::Duration::from_millis(30)).await; - let quiet = runtime.idle_report().await; - assert!(quiet.is_quiescent()); - assert!(quiet.idle_for_ms >= 30, "idle_for_ms={}", quiet.idle_for_ms); - - runtime.touch_activity(); - let touched = runtime.idle_report().await; - assert!( - touched.idle_for_ms < 30, - "idle_for_ms={}", - touched.idle_for_ms + let response = runtime.initialize_response(); + assert_eq!(response.protocol_version, ENVD_PROTOCOL_VERSION); + assert_eq!(response.plane, Plane::Data); + assert_eq!(&response.daemon_instance_id, runtime.instance_id()); + assert_eq!( + response.default_cwd.as_str(), + runtime.config().cwd.to_str().unwrap() ); + let filesystem = response.capabilities.filesystem.expect("filesystem"); + assert!(filesystem.write); + assert!(response.capabilities.process.is_some()); + assert!(response.capabilities.job.is_some()); + assert_eq!(response.limits.job.max_request_jobs, 64); + } - let process_id = ProcessId::new("sleeper"); - runtime - .processes() - .start_process(StartProcessParams { - process_id: process_id.clone(), - argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), "sleep 30".to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(60_000), - tty: false, - }) - .await - .expect("start"); + #[tokio::test(flavor = "current_thread")] + async fn a_read_only_daemon_refuses_writes_before_any_side_effect() { + let (_temp, runtime) = super::test_support::runtime_with(|config| { + config.read_only_fs = true; + }); + let path = runtime.config().cwd.join("written"); + let error = handle_request( + &runtime, + &RequestContext::detached(), + "fs/writeFile", + json!({"path": path.to_str().unwrap(), "data": "aGk="}), + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::CapabilityUnavailable); + assert!(!path.exists()); + } + + #[tokio::test(flavor = "current_thread")] + async fn idle_report_holds_zero_while_a_process_runs_and_resets_on_use() { + let (_temp, runtime) = runtime(); + tokio::time::sleep(std::time::Duration::from_millis(30)).await; + let quiet: GetIdleResponse = + serde_json::from_value(call(&runtime, EnvGetIdle::NAME, json!({})).await).unwrap(); + assert!(quiet.idle_for_ms >= 30, "idle_for_ms={}", quiet.idle_for_ms); + assert_eq!(quiet.running_process_count, 0); + + call( + &runtime, + ProcessStart::NAME, + json!({ + "processId": "sleeper", + "command": {"argv": ["/bin/sh", "-c", "sleep 30"], "timeoutMs": 60000}, + }), + ) + .await; let busy = runtime.idle_report().await; - assert_eq!(busy.running_processes, 1); + assert_eq!(busy.running_process_count, 1); assert_eq!(busy.idle_for_ms, 0); - assert!(!busy.is_quiescent()); - - runtime - .processes() - .terminate_process(TerminateProcessParams { - process_id, - signal: ProcessSignal::Kill, - }) - .await - .expect("terminate"); + + call( + &runtime, + ProcessTerminate::NAME, + json!({"processId": "sleeper", "signal": "kill"}), + ) + .await; for _ in 0..100 { - if runtime.idle_report().await.running_processes == 0 { + if runtime.idle_report().await.running_process_count == 0 { break; } tokio::time::sleep(std::time::Duration::from_millis(20)).await; } let after = runtime.idle_report().await; - assert_eq!(after.running_processes, 0); + assert_eq!(after.running_process_count, 0); assert!(after.idle_for_ms < 5_000); } - #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn idle_report_counts_leftover_groups_without_treating_them_as_busy() { + async fn observation_methods_leave_the_idle_clock_alone() { let (_temp, runtime) = runtime(); - let process_id = ProcessId::new("service"); - runtime - .processes() - .start_process(StartProcessParams { - process_id: process_id.clone(), - argv: vec![ - "/bin/sh".to_owned(), - "-c".to_owned(), - "nohup sleep 30 >/dev/null 2>&1 & exit 0".to_owned(), - ], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(60_000), - tty: false, - }) - .await - .expect("start"); - let output = runtime - .processes() - .read_process(ReadProcessParams { - process_id: process_id.clone(), - after_seq: None, - max_bytes: None, - wait_ms: None, - }) - .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.leftover_processes.len(), 1); - - let report = runtime.idle_report().await; - assert_eq!(report.running_processes, 0); - assert_eq!(report.leftover_process_groups, 1); - assert!(report.is_quiescent(), "leftovers are not running work"); - - runtime - .processes() - .terminate_process(TerminateProcessParams { - process_id, - signal: ProcessSignal::Kill, - }) - .await - .expect("kill leftovers"); - for _ in 0..100 { - if runtime.idle_report().await.leftover_process_groups == 0 { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - } - assert_eq!(runtime.idle_report().await.leftover_process_groups, 0); + tokio::time::sleep(std::time::Duration::from_millis(40)).await; + call(&runtime, EnvGetIdle::NAME, json!({})).await; + let cwd = runtime.config().cwd.to_str().unwrap().to_owned(); + let _ = handle_request( + &runtime, + &RequestContext::detached(), + "fs/scan", + json!({"roots": [{"path": cwd, "maxDepth": 0}]}), + ) + .await; + assert!(runtime.idle_report().await.idle_for_ms >= 40); } } diff --git a/crates/environment-daemon/src/main.rs b/crates/environment-daemon/src/main.rs index 1cb0146a7..d38e4a4d0 100644 --- a/crates/environment-daemon/src/main.rs +++ b/crates/environment-daemon/src/main.rs @@ -1,33 +1,102 @@ +use std::process::ExitCode; + use clap::Parser as _; use environment_daemon::{ DaemonRuntime, config::{DaemonArgs, DaemonCommand}, - server, upgrade, + server::{self, DaemonExit}, + upgrade, }; -#[tokio::main(flavor = "current_thread")] -async fn main() -> anyhow::Result<()> { +/// Worker threads of the async runtime. Blocking work runs on the blocking +/// pool; the workers keep connections, heartbeats and idle reports moving +/// even if one of them is held up. +const WORKER_THREADS: usize = 2; + +fn main() -> ExitCode { + // Configuration is read, and secret-bearing variables are removed from + // the process environment, before the async runtime starts any thread. let _ = dotenvy::dotenv(); let args = DaemonArgs::parse(); if args.print_build { - println!( - "{}", - serde_json::to_string(&environment_daemon::build_info())? - ); - return Ok(()); + return match serde_json::to_string(&environment_daemon::build_facts()) { + Ok(json) => { + println!("{json}"); + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("lightspeed-envd: {error}"); + ExitCode::FAILURE + } + }; } if args.command == Some(DaemonCommand::Upgrade) { - let installed = upgrade::install(args.upgrade_request()?).await?; - println!( - "installed lightspeed-envd {} ({}, protocol {}) at {}", - installed.version, - installed.git_sha, - installed.protocol_version, - installed.path.display() - ); - return Ok(()); + return upgrade_command(&args); + } + let exit = match args.into_config() { + Ok(config) => serve(config), + Err(error) => DaemonExit::Configuration(error), + }; + if !matches!(exit, DaemonExit::Shutdown) { + eprintln!("lightspeed-envd {exit}"); + } + ExitCode::from(exit.status()) +} + +fn serve(config: environment_daemon::config::DaemonConfig) -> DaemonExit { + let runtime = match async_runtime() { + Ok(runtime) => runtime, + Err(error) => return DaemonExit::Failed(error.into()), + }; + runtime.block_on(async move { + let daemon = match DaemonRuntime::new(config) { + Ok(daemon) => daemon, + Err(error) => return DaemonExit::Configuration(error), + }; + if let Err(error) = server::spawn_signal_listener(&daemon) { + return DaemonExit::Failed(error); + } + server::run(daemon).await + }) +} + +fn upgrade_command(args: &DaemonArgs) -> ExitCode { + let request = match args.upgrade_request() { + Ok(request) => request, + Err(error) => { + eprintln!("lightspeed-envd upgrade: {error:#}"); + return ExitCode::from(2); + } + }; + let installed = match async_runtime() { + Ok(runtime) => runtime.block_on(upgrade::install(request)), + Err(error) => { + eprintln!("lightspeed-envd upgrade: {error}"); + return ExitCode::FAILURE; + } + }; + match installed { + Ok(installed) => { + println!( + "installed lightspeed-envd {} ({}, protocol {}) at {}", + installed.version, + installed.git_sha, + installed.protocol_version, + installed.path.display() + ); + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("lightspeed-envd upgrade: {error}"); + ExitCode::FAILURE + } } - let config = args.into_config()?; - let runtime = DaemonRuntime::new(config)?; - server::run(runtime).await +} + +fn async_runtime() -> std::io::Result { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(WORKER_THREADS) + .thread_name("lightspeed-envd") + .enable_all() + .build() } diff --git a/crates/environment-daemon/src/process.rs b/crates/environment-daemon/src/process.rs index f1f8aa307..f17d29805 100644 --- a/crates/environment-daemon/src/process.rs +++ b/crates/environment-daemon/src/process.rs @@ -1,1970 +1,1332 @@ -//! Interactive process execution. +//! `process/*`: interactive processes on the shared execution primitive. //! //! Every command starts as the leader of its own process group. Its natural //! exit never sweeps the group: whatever the command left running keeps the -//! environment's lifetime, is sampled once at exit and reported to the -//! caller, and keeps somewhere to write because the output readers stay on -//! the pipes until end of file. Only a timeout, an explicit kill, or the -//! daemon's own cancellation paths kill the group. +//! environment's lifetime and is reported as leftovers, sampled when a read +//! returns the settled exit. Only a timeout or an explicit terminate or kill +//! signals the group. //! -//! Output is delivered through a daemon-owned cursor so a handle used from a -//! later connection continues where the previous read stopped, and unread -//! output is retained as a bounded head-and-tail buffer. +//! Output is one byte stream per process, read through a daemon-owned +//! cursor that advances only after a response was written, or at explicit +//! offsets that never consume. Retention is bounded per process, per exited +//! process and across exited processes; an evicted handle leaves a +//! tombstone that answers `notFound {expired}`. Spawn records let a +//! restarted daemon adopt the groups an earlier instance left running and +//! answer their handles with `notFound {daemonRestarted}`. use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, - io::{Read as _, Write as _}, - path::{Component, PathBuf}, - process::Stdio, - sync::{Arc, Mutex as StdMutex}, - time::Duration, + collections::{BTreeMap, VecDeque}, + fmt::Write as _, + path::PathBuf, + sync::{Arc, Mutex, MutexGuard}, + time::{Duration, Instant}, }; use environment_protocol::{ - data::process::{ - LeftoverProcess, ProcessOutputChunk, ProcessOutputStream, ProcessSignal, - ProcessTermination, ReadProcessParams, ReadProcessResponse, ResizeProcessParams, - ResizeProcessResponse, StartProcessParams, StartProcessResponse, TerminateProcessParams, - TerminateProcessResponse, WriteProcessParams, WriteProcessResponse, WriteProcessStatus, + data::{ + exec::{ + ExecInput, ExecState, LeftoverDisposition, OutputPage, OutputStream, PtySize, ReturnOn, + StoppedBy, + }, + handshake::ProcessLimits, + process::{ + ProcessSignal, ReadProcessParams, ReadProcessResponse, StartProcessParams, + StartProcessResponse, TerminateProcessParams, TerminateProcessResponse, + WriteProcessParams, WriteProcessResponse, + }, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::{ByteChunk, EnvironmentPath, ProcessId}, -}; -use portable_pty::{CommandBuilder, MasterPty, PtySize, native_pty_system}; -use tokio::{ - io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, - process::{Child, Command}, - sync::{Mutex, Notify}, - task::JoinHandle, - time::Instant, + error::{Capability, NotFoundReason, ProtocolError}, + ids::{DaemonInstanceId, ProcessId}, + limits::MAX_PAGE_CHUNKS, }; +use serde::{Deserialize, Serialize}; use crate::{ - process_group, - redaction::{redact_bytes, redactions_for_secret_env}, + dispatch::{Reply, RequestContext}, + exec::{ + AfterExit, ExecOptions, Execution, OutputSink, PreparedCommand, + log::OutputLog, + outcome::describe_exit, + spawn_record::{SpawnRecord, write_json_atomic}, + validate_command, + }, + filesystem::Confinement, + process_group::{self, Group}, }; -/// How long a finished process entry stays readable after the first read -/// that observed its exit. -#[cfg(not(test))] -pub(crate) const TERMINAL_PROCESS_RETENTION: Duration = Duration::from_secs(60); -#[cfg(test)] -pub(crate) const TERMINAL_PROCESS_RETENTION: Duration = Duration::from_millis(10); - -/// How often the exit watcher polls a running child. Readers only notify on -/// pipe events, and leftovers holding the pipes can suppress EOF, so exit -/// observation must not depend on them. -#[cfg(not(test))] -const EXIT_POLL_INTERVAL: Duration = Duration::from_millis(100); -#[cfg(test)] -const EXIT_POLL_INTERVAL: Duration = Duration::from_millis(10); - -/// Unread output retained per process. Beyond it the middle is dropped and -/// the next read reports how many bytes went missing; the oldest half and -/// the newest chunks are kept. -#[cfg(not(test))] -const RETAINED_OUTPUT_CAP: usize = 1024 * 1024; -#[cfg(test)] -const RETAINED_OUTPUT_CAP: usize = 16 * 1024; - -const READ_BUFFER_BYTES: usize = 8192; +/// How long a tombstone answers `notFound {expired}`. +const TOMBSTONE_RETENTION: Duration = Duration::from_secs(24 * 60 * 60); + +/// Most tombstones kept; the oldest go first. +const MAX_TOMBSTONES: usize = 4096; + +/// Everything a [`ProcessManager`] needs from the daemon. +pub struct ProcessConfig { + /// Canonical default `cwd`. + pub cwd: PathBuf, + /// Resolves every `cwd`. + pub confinement: Arc, + /// Spawn records live in `/processes`. + pub state_dir: PathBuf, + /// Daemon variables no child inherits. + pub scrubbed_env: Vec, + pub instance_id: DaemonInstanceId, + pub limits: ProcessLimits, + /// `limits.maxWaitMs`, the `waitMs` ceiling. + pub max_wait_ms: u64, + /// `capabilities.process.pty`. + pub pty_supported: bool, +} +/// The daemon's processes. Cheap to clone. #[derive(Clone)] pub struct ProcessManager { - cwd: PathBuf, - fs_root: PathBuf, - processes: Arc>>>, - /// Daemon configuration variables no child may inherit. - scrubbed_env: Arc>, - /// Process groups whose root exited while members were still alive. - /// Pruned lazily once the group is empty. - leftover_groups: Arc>>, -} - -struct ProcessEntry { - state: Mutex, - notify: Notify, - readers: Mutex>>, + inner: Arc, } -struct ProcessState { - child: ProcessChild, - pid: Option, - pgid: Option, - stdin: Option, - pty_master: Option>, - redactions: Vec>, - output: OutputBuffer, - /// Output readers still attached to a pipe or PTY. - readers_open: u32, - exited: bool, - exit_observed_at: Option, - exit_code: Option, - termination: Option, - leftover_processes: Vec, - failure: Option, - cleanup_scheduled: bool, - /// The root exited and the drain grace passed while leftovers still held - /// the pipes: readers keep reading so nothing blocks the leftovers, but - /// the bytes are dropped. - discarding: bool, +struct ManagerInner { + config: ProcessConfig, + records_dir: PathBuf, + table: Mutex, + /// Groups no live entry owns any more that still had members when last + /// looked at: groups of evicted entries and groups adopted from an + /// earlier daemon instance. Each remembers its spawn record, deleted + /// once the group is empty. + leftover_groups: Mutex>, } -enum ProcessChild { - Pipe(Child), - Pty(Box), +#[derive(Default)] +struct Table { + slots: BTreeMap, + /// Tombstoned ids in eviction order. + tombstones: VecDeque<(ProcessId, Instant)>, } -enum ProcessInput { - Pty(Box), +enum Slot { + /// A start is spawning under this id. + Reserved, + Live(Arc), + /// Evicted; remembers how it ended. + Tombstone(String), + /// A handle of an earlier daemon instance. + Restarted, } -/// Retained output between the cursor and the newest chunk. -#[derive(Default)] -struct OutputBuffer { - chunks: VecDeque, - retained_bytes: usize, - next_seq: u64, - delivered_seq: u64, +struct Entry { + execution: Execution, + record_path: PathBuf, + /// When a reply returning the settled exit with nothing left unread was + /// written. + fully_read_at: Mutex>, } -struct RetainedChunk { - chunk: ProcessOutputChunk, - /// Bytes dropped from the buffer immediately before this chunk. - omitted_before: u64, +/// A process's retained output and its daemon cursor. +struct ProcessOutput { + log: OutputLog, + cursor: u64, + cap: u64, } -impl OutputBuffer { - fn push(&mut self, stream: ProcessOutputStream, bytes: Vec) { - let seq = self.next_seq; - self.next_seq += 1; - self.retained_bytes += bytes.len(); - self.chunks.push_back(RetainedChunk { - chunk: ProcessOutputChunk { - seq, - stream, - chunk: ByteChunk::from(bytes), - }, - omitted_before: 0, - }); - self.enforce_cap(); +impl OutputSink for ProcessOutput { + fn append(&mut self, stream: OutputStream, data: &[u8]) { + self.log.append(stream, data); + self.log.enforce_cap(self.cap, self.cursor); } +} - fn unread_bytes(&self) -> usize { - self.retained_bytes - } +/// The file of one spawn record. +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct StoredSpawn { + process_id: ProcessId, + spawn: SpawnRecord, +} - /// Drops chunks from just past the head until the buffer fits. The head - /// is the oldest half of the cap; the newest chunk is never dropped. - fn enforce_cap(&mut self) { - if self.retained_bytes <= RETAINED_OUTPUT_CAP { - return; - } - let head_cap = RETAINED_OUTPUT_CAP / 2; - let mut head_bytes = 0usize; - let mut head_len = 0usize; - for retained in &self.chunks { - // The head never grows past an omission: the chunk that follows - // dropped bytes stays the boundary, so every later drop extends - // the same gap instead of opening a second one. Pipe reads on - // Linux deliver uneven chunks, where a small survivor would - // otherwise be absorbed into the head. - if retained.omitted_before > 0 { - break; - } - let len = retained.chunk.chunk.as_slice().len(); - if head_bytes + len > head_cap { - break; - } - head_bytes += len; - head_len += 1; - } - let mut omitted = 0u64; - while self.retained_bytes > RETAINED_OUTPUT_CAP && self.chunks.len() > head_len + 1 { - let Some(dropped) = self.chunks.remove(head_len) else { - break; +impl ProcessManager { + /// Reconciles the spawn records an earlier instance left: groups of + /// this boot that still run are adopted as leftovers, and their ids + /// answer `notFound {daemonRestarted}`; every other record is deleted. + pub fn new(config: ProcessConfig) -> anyhow::Result { + let records_dir = config.state_dir.join("processes"); + std::fs::create_dir_all(&records_dir)?; + let boot = process_group::boot_id(); + let mut table = Table::default(); + let mut leftover_groups = BTreeMap::new(); + for entry in std::fs::read_dir(&records_dir)? { + let path = entry?.path(); + let stored = std::fs::read(&path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let Some(stored) = stored else { + eprintln!( + "lightspeed-envd: removing unreadable spawn record {}", + path.display() + ); + let _ = std::fs::remove_file(&path); + continue; }; - let len = dropped.chunk.chunk.as_slice().len(); - self.retained_bytes -= len; - omitted += len as u64 + dropped.omitted_before; - } - if omitted > 0 - && let Some(next) = self.chunks.get_mut(head_len) - { - next.omitted_before += omitted; - } - } - - /// Delivers from the cursor, advancing it. Whole chunks are taken up to - /// `max_bytes`; when the very first chunk alone exceeds the budget it is - /// split and its remainder stays at the cursor under the same sequence. - fn deliver(&mut self, max_bytes: Option) -> (Vec, u64) { - let mut delivered = Vec::new(); - let mut omitted = 0u64; - let mut bytes = 0usize; - while let Some(front) = self.chunks.front_mut() { - let len = front.chunk.chunk.as_slice().len(); - if let Some(max_bytes) = max_bytes - && bytes + len > max_bytes - { - if delivered.is_empty() && max_bytes > 0 { - let (head, rest) = front.chunk.chunk.as_slice().split_at(max_bytes); - delivered.push(ProcessOutputChunk { - seq: front.chunk.seq, - stream: front.chunk.stream, - chunk: ByteChunk::from(head.to_vec()), - }); - omitted += front.omitted_before; - front.omitted_before = 0; - front.chunk.chunk = ByteChunk::from(rest.to_vec()); - self.retained_bytes -= max_bytes; - } - break; + if !stored.spawn.same_boot(boot.as_deref()) { + let _ = std::fs::remove_file(&path); + continue; } - let retained = self.chunks.pop_front().expect("front chunk exists"); - self.retained_bytes -= len; - bytes += len; - omitted += retained.omitted_before; - delivered.push(retained.chunk); - } - self.delivered_seq = self - .chunks - .front() - .map_or(self.next_seq, |front| front.chunk.seq); - (delivered, omitted) - } - - /// Re-reads retained chunks from a sequence number without moving the - /// cursor. Best effort: delivered chunks are gone. - fn reread(&self, after_seq: u64, max_bytes: Option) -> (Vec, u64) { - let mut chunks = Vec::new(); - let mut omitted = 0u64; - let mut bytes = 0usize; - for retained in self - .chunks - .iter() - .filter(|retained| retained.chunk.seq >= after_seq) - { - let slice = retained.chunk.chunk.as_slice(); - if let Some(max_bytes) = max_bytes { - if bytes >= max_bytes { - break; - } - let remaining = max_bytes - bytes; - if slice.len() > remaining { - chunks.push(ProcessOutputChunk { - seq: retained.chunk.seq, - stream: retained.chunk.stream, - chunk: ByteChunk::from(slice[..remaining].to_vec()), - }); - omitted += retained.omitted_before; - break; - } + table.slots.insert(stored.process_id, Slot::Restarted); + if stored.spawn.group_survives() { + leftover_groups.insert(stored.spawn.pgid, (stored.spawn.group(), path)); + } else { + let _ = std::fs::remove_file(&path); } - bytes += slice.len(); - omitted += retained.omitted_before; - chunks.push(retained.chunk.clone()); - } - (chunks, omitted) - } -} - -impl ProcessManager { - pub fn new(cwd: PathBuf, fs_root: PathBuf) -> Self { - Self { - cwd: normalize_path(cwd), - fs_root: normalize_path(fs_root), - processes: Arc::new(Mutex::new(BTreeMap::new())), - scrubbed_env: Arc::new(Vec::new()), - leftover_groups: Arc::new(StdMutex::new(BTreeSet::new())), } + Ok(Self { + inner: Arc::new(ManagerInner { + config, + records_dir, + table: Mutex::new(table), + leftover_groups: Mutex::new(leftover_groups), + }), + }) } - pub fn with_scrubbed_env(mut self, names: Vec) -> Self { - self.scrubbed_env = Arc::new(names); - self - } - - /// Processes started here that have not exited yet. + /// Processes whose root has not exited. pub async fn running_count(&self) -> u32 { - let entries: Vec> = - self.processes.lock().await.values().cloned().collect(); - let mut running = 0u32; - for entry in entries { - if !entry.state.lock().await.exited { - running += 1; - } - } - running + self.maintain(); + let table = self.table(); + let running = table + .slots + .values() + .filter(|slot| match slot { + Slot::Live(entry) => !entry.execution.with_state(|state| state.exited()), + _ => false, + }) + .count(); + u32::try_from(running).unwrap_or(u32::MAX) } - /// Process groups of finished commands that still have a live member. - /// Leftovers are not running work: a service waiting for requests is - /// idle, so this never feeds the quiescence check. + /// Process groups whose root exited while members still run, adopted + /// groups of an earlier instance included. Leftovers are not running + /// work: they never hold the idle clock. pub fn leftover_group_count(&self) -> u32 { - let mut groups = self - .leftover_groups - .lock() - .expect("leftover groups poisoned"); - groups.retain(|pgid| process_group::group_alive(*pgid)); - u32::try_from(groups.len()).unwrap_or(u32::MAX) + let live = { + let table = self.table(); + table + .slots + .values() + .filter(|slot| match slot { + Slot::Live(entry) => { + entry.execution.with_state(|state| state.exited()) + && entry.execution.group_alive() + } + _ => false, + }) + .count() + }; + let mut groups = lock(&self.inner.leftover_groups); + groups.retain(|_, (group, record)| { + let alive = group.alive(); + if !alive { + let _ = std::fs::remove_file(record); + } + alive + }); + u32::try_from(live + groups.len()).unwrap_or(u32::MAX) } - pub async fn start_process( + pub async fn start( &self, + _ctx: &RequestContext, params: StartProcessParams, - ) -> Result { - if params.argv.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "process argv must not be empty", - )); - } - let cwd = params - .cwd - .as_ref() - .map(|path| self.resolve_cwd(path)) - .transpose()? - .unwrap_or_else(|| self.cwd.clone()); - for name in params.secret_env.keys() { - if params.env.contains_key(name) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("process env collides with secret env: {name}"), + ) -> Result { + let StartProcessParams { + process_id, + command, + pty, + } = params; + let config = &self.inner.config; + validate_command(&command, "command")?; + if let Some(size) = pty { + validate_pty(size, &command.input)?; + if !config.pty_supported { + return Err(ProtocolError::capability_unavailable( + Capability::ProcessPty, + "this daemon cannot run processes under a PTY", )); } } + let cwd = match &command.cwd { + Some(path) => config.confinement.resolve_directory(path)?, + None => config.cwd.clone(), + }; + self.maintain(); + self.reserve(&process_id)?; - if params.tty { - return self.start_pty_process(params, cwd).await; - } - - let mut command = Command::new(¶ms.argv[0]); - command - .args(¶ms.argv[1..]) - .current_dir(&cwd) - .envs(params.env.iter()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - for name in self.scrubbed_env.iter() { - command.env_remove(name); - } - for (name, value) in ¶ms.secret_env { - command.env(name, value.expose()); - } - // Plain pipes never keep an input open: a one-shot payload is - // written and closed at start, otherwise the child reads EOF at - // once. Interactive input needs a PTY. - if params.stdin.is_some() { - command.stdin(Stdio::piped()); - } else { - command.stdin(Stdio::null()); - } - process_group::spawn_in_own_group(&mut command); - - let mut child = command.spawn().map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - format!("spawn process {:?}: {error}", params.argv), - ) - })?; - let pid = child.id(); - let stdout = child.stdout.take(); - let stderr = child.stderr.take(); - let stdin = child.stdin.take(); - if let Some(input) = params.stdin { - let Some(mut writer) = stdin else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - "process stdin was not available", - )); - }; - writer.write_all(input.as_slice()).await.map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - })?; - drop(writer); - } - - let process_id = params.process_id; - let redactions = redactions_for_secret_env(¶ms.secret_env); - let readers_open = u32::from(stdout.is_some()) + u32::from(stderr.is_some()); - let entry = Arc::new(ProcessEntry { - state: Mutex::new(ProcessState::new( - ProcessChild::Pipe(child), - pid, - pid, - None, - None, - redactions, - readers_open, - )), - notify: Notify::new(), - readers: Mutex::new(Vec::new()), - }); - self.insert_entry(&process_id, entry.clone()).await?; - - { - let mut readers = entry.readers.lock().await; - if let Some(stdout) = stdout { - readers.push(tokio::spawn(read_stream( - entry.clone(), - stdout, - ProcessOutputStream::Stdout, - ))); - } - if let Some(stderr) = stderr { - readers.push(tokio::spawn(read_stream( - entry.clone(), - stderr, - ProcessOutputStream::Stderr, - ))); + let options = ExecOptions { + timeout: command.timeout_ms.map(Duration::from_millis), + terminate_grace: Duration::from_millis(config.limits.terminate_grace_ms), + after_exit: AfterExit::Keep, + }; + let sink = ProcessOutput { + log: OutputLog::default(), + cursor: 0, + cap: config.limits.max_retained_output_bytes, + }; + let prepared = PreparedCommand::new(command, cwd, &config.scrubbed_env, pty); + let execution = match prepared.spawn(sink, options, config.limits.max_input_queue_bytes) { + Ok(execution) => execution, + Err(error) => { + self.release(&process_id); + return Err(error); } + }; + let pid = execution.pid(); + let record_path = self + .inner + .records_dir + .join(format!("{}.json", process_id.as_str())); + let entry = Arc::new(Entry { + execution: execution.clone(), + record_path: record_path.clone(), + fully_read_at: Mutex::new(None), + }); + self.table() + .slots + .insert(process_id.clone(), Slot::Live(entry)); + let stored = StoredSpawn { + process_id, + spawn: SpawnRecord::new(execution.group(), &config.instance_id), + }; + if let Err(error) = write_json_atomic(&record_path, &stored) { + eprintln!( + "lightspeed-envd: could not persist the spawn record of {}: {error}", + stored.process_id + ); } - self.spawn_watchers(entry, params.timeout_ms); - Ok(StartProcessResponse { process_id }) + execution.start_input(); + Ok(StartProcessResponse { pid }) } - async fn start_pty_process( + pub async fn read( &self, - params: StartProcessParams, - cwd: PathBuf, - ) -> Result { - let pty = native_pty_system() - .openpty(PtySize::default()) - .map_err(process_error("open PTY"))?; - let mut command = CommandBuilder::new(¶ms.argv[0]); - command.args(¶ms.argv[1..]); - command.cwd(&cwd); - for name in self.scrubbed_env.iter() { - command.env_remove(name); - } - for (name, value) in ¶ms.env { - command.env(name, value); - } - for (name, value) in ¶ms.secret_env { - command.env(name, value.expose()); - } - let child = pty - .slave - .spawn_command(command) - .map_err(process_error("spawn PTY process"))?; - // The PTY child is a session leader, so its pid is its group id. - let pid = child.process_id(); - let reader = pty - .master - .try_clone_reader() - .map_err(process_error("clone PTY reader"))?; - let mut stdin = pty - .master - .take_writer() - .map_err(process_error("open PTY writer"))?; - if let Some(input) = params.stdin.as_ref() { - stdin - .write_all(input.as_slice()) - .map_err(process_error("write initial PTY input"))?; - stdin.flush().map_err(process_error("flush PTY input"))?; + ctx: &RequestContext, + params: ReadProcessParams, + ) -> Result, ProtocolError> { + let config = &self.inner.config; + if params.max_bytes > config.limits.max_read_bytes { + return Err(ProtocolError::above_ceiling( + "maxBytes", + params.max_bytes, + config.limits.max_read_bytes, + )); } - let process_id = params.process_id; - let redactions = redactions_for_secret_env(¶ms.secret_env); - let entry = Arc::new(ProcessEntry { - state: Mutex::new(ProcessState::new( - ProcessChild::Pty(child), - pid, - pid, - Some(ProcessInput::Pty(stdin)), - Some(pty.master), - redactions, - 1, - )), - notify: Notify::new(), - readers: Mutex::new(Vec::new()), - }); - self.insert_entry(&process_id, entry.clone()).await?; - entry - .readers - .lock() - .await - .push(read_pty_stream(entry.clone(), reader)); - self.spawn_watchers(entry, params.timeout_ms); - Ok(StartProcessResponse { process_id }) - } - - async fn insert_entry( - &self, - process_id: &ProcessId, - entry: Arc, - ) -> Result<(), EnvironmentProtocolError> { - let mut processes = self.processes.lock().await; - if processes.contains_key(process_id.as_str()) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Conflict, - format!("process id already exists: {process_id}"), + if params.wait_ms > config.max_wait_ms { + return Err(ProtocolError::above_ceiling( + "waitMs", + params.wait_ms, + config.max_wait_ms, )); } - processes.insert(process_id.to_string(), entry); - Ok(()) - } - - fn spawn_watchers(&self, entry: Arc, timeout_ms: Option) { - if let Some(timeout_ms) = timeout_ms { - tokio::spawn(timeout_process( - entry.clone(), - Duration::from_millis(timeout_ms), + if params.after.is_some() && params.tail_bytes.is_some() { + return Err(ProtocolError::invalid_field( + "tailBytes", + "after and tailBytes exclude each other", )); } - tokio::spawn(watch_exit(entry, self.leftover_groups.clone())); - } - - pub async fn read_process( - &self, - params: ReadProcessParams, - ) -> Result { - let Some(entry) = self.entry(¶ms.process_id).await else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::NotFound, - format!("unknown process id: {}", params.process_id), + self.maintain(); + let entry = self.lookup(¶ms.process_id)?; + let execution = &entry.execution; + + let (start, end) = execution.with_state(|state| { + let output = &mut state.sink; + if let Some(ack) = params.ack { + output.cursor = output.cursor.max(ack.min(output.log.end())); + } + let end = output.log.end(); + let start = match (params.after, params.tail_bytes) { + (Some(after), _) => after, + (None, Some(tail)) => end.saturating_sub(tail), + (None, None) => output.cursor, + }; + (start, end) + }); + if start > end { + return Err(ProtocolError::invalid_field( + "after", + format!("after {start} is past the end of the output ({end})"), )); - }; - let deadline = params - .wait_ms - .map(|wait_ms| Instant::now() + Duration::from_millis(wait_ms)); + } + let deadline = tokio::time::Instant::now() + Duration::from_millis(params.wait_ms); + let mut changes = execution.subscribe(); loop { - let (response, schedule_cleanup, wake_at) = { - let mut state = entry.state.lock().await; - if let Some(pgid) = update_exit_status(&mut state)? { - self.observe_exit(&entry, pgid); - } - let now = Instant::now(); - let (done, drain_deadline) = exit_settled(&state, now); - let should_return = if params.wait_ms.is_some() { - done || deadline.is_some_and(|deadline| now >= deadline) - || params - .max_bytes - .is_some_and(|max_bytes| state.output.unread_bytes() >= max_bytes) - } else { - done - }; - if should_return { - let schedule_cleanup = state.exited && !state.cleanup_scheduled; - if schedule_cleanup { - state.cleanup_scheduled = true; + changes.borrow_and_update(); + let ready = execution.with_state(|state| { + state.settled() + || match params.return_on { + ReturnOn::AnyOutput => state.sink.log.end() > start, + ReturnOn::Window => state.sink.log.end() - start >= params.max_bytes, + ReturnOn::Exit => false, } - ( - Some(response_from_state( - &mut state, - params.after_seq, - params.max_bytes, - )), - schedule_cleanup, - None, - ) - } else { - let wake_at = match (deadline, drain_deadline) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }; - (None, false, wake_at) - } - }; - if let Some(response) = response { - if schedule_cleanup { - self.schedule_terminal_cleanup(params.process_id.clone(), entry.clone()); - } - return Ok(response); + }); + if ready { + break; } - - if let Some(wake_at) = wake_at { - tokio::select! { - _ = entry.notify.notified() => {} - _ = tokio::time::sleep_until(wake_at) => {} - } - } else { - entry.notify.notified().await; + tokio::select! { + () = ctx.cancel.cancelled() => return Err(ProtocolError::cancelled()), + () = tokio::time::sleep_until(deadline) => break, + changed = changes.changed() => if changed.is_err() { break }, } } - } - pub async fn write_process( - &self, - params: WriteProcessParams, - ) -> Result { - let Some(entry) = self.entry(¶ms.process_id).await else { - return Ok(WriteProcessResponse { - status: WriteProcessStatus::UnknownProcess, - }); - }; - let mut state = entry.state.lock().await; - if let Some(pgid) = update_exit_status(&mut state)? { - self.observe_exit(&entry, pgid); - } - let chunk = params.chunk.filter(|chunk| !chunk.as_slice().is_empty()); - // No input and nothing to close is a wait; the caller's read follows. - if chunk.is_none() && !params.close_stdin { - return Ok(WriteProcessResponse { - status: WriteProcessStatus::Accepted, - }); - } - if state.exited { - return Ok(WriteProcessResponse { - status: if chunk.is_some() { - WriteProcessStatus::StdinClosed - } else { - WriteProcessStatus::Accepted + let (mut response, settled) = execution.with_state(|state| { + let page = state + .sink + .log + .page(start, params.max_bytes, MAX_PAGE_CHUNKS); + let settled = state.settled(); + let response = ReadProcessResponse { + output: OutputPage { + chunks: page.chunks, + next: page.next, + end: state.sink.log.end(), + gaps: page.gaps, }, - }); - } - let Some(stdin) = state.stdin.as_mut() else { - return Ok(WriteProcessResponse { - status: if chunk.is_some() { - WriteProcessStatus::StdinClosed + state: if settled { + ExecState::Exited } else { - WriteProcessStatus::Accepted + ExecState::Running }, - }); - }; - if let Some(chunk) = chunk { - match stdin { - ProcessInput::Pty(stdin) => stdin - .write_all(chunk.as_slice()) - .and_then(|()| stdin.flush()), - } - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - })?; - } - if params.close_stdin { - state.stdin.take(); + exit: state.exit().filter(|_| settled).cloned(), + stopped_by: state.stopped_by().filter(|_| settled), + error: state.error().cloned(), + pid: execution.pid(), + leftovers: None, + }; + (response, settled) + }); + if settled { + response.leftovers = execution.sample_leftovers(LeftoverDisposition::Kept); } - Ok(WriteProcessResponse { - status: WriteProcessStatus::Accepted, - }) - } - pub async fn terminate_process( - &self, - params: TerminateProcessParams, - ) -> Result { - let Some(entry) = self.entry(¶ms.process_id).await else { - return Ok(TerminateProcessResponse { running: false }); - }; - let mut state = entry.state.lock().await; - if let Some(pgid) = update_exit_status(&mut state)? { - self.observe_exit(&entry, pgid); - } - if state.exited { - // The root is gone; the signal still reaches whatever it left in - // its group, which is how a handle stops the service it started. - if let Some(pgid) = state.pgid { - match params.signal { - ProcessSignal::Interrupt => { - process_group::interrupt_group(pgid); - } - ProcessSignal::Kill => { - process_group::kill_group(pgid); - } - } - } - return Ok(TerminateProcessResponse { running: false }); + let consuming = params.after.is_none() && params.tail_bytes.is_none(); + let complete = settled && response.output.next == response.output.end; + if !consuming && !complete { + return Ok(Reply::new(response)); } - match params.signal { - ProcessSignal::Interrupt => { - if let Some(pgid) = state.pgid { - process_group::interrupt_group(pgid); - } else { - interrupt_child(&mut state.child); - } + let next = response.output.next; + let entry = entry.clone(); + Ok(Reply::with_commit(response, move || { + if consuming { + entry.execution.with_state(|state| { + state.sink.cursor = state.sink.cursor.max(next); + }); } - ProcessSignal::Kill => { - if let Some(pgid) = state.pgid { - process_group::kill_group(pgid); - } - kill_child(&mut state.child).await?; - state.exited = true; - state.exit_observed_at = Some(Instant::now()); - state.exit_code = None; - state.termination = Some(ProcessTermination::Killed); - state.stdin.take(); - spawn_reader_drain(entry.clone()); - entry.notify.notify_waiters(); + if complete { + lock(&entry.fully_read_at).get_or_insert_with(Instant::now); } - } - Ok(TerminateProcessResponse { running: true }) + })) } - pub async fn resize_process( + pub async fn write( &self, - params: ResizeProcessParams, - ) -> Result { - let Some(entry) = self.entry(¶ms.process_id).await else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::NotFound, - format!("unknown process id: {}", params.process_id), - )); - }; - let state = entry.state.lock().await; - let Some(master) = state.pty_master.as_ref() else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "process is not attached to a PTY", - )); - }; - master - .resize(PtySize { - rows: params.size.rows, - cols: params.size.cols, - pixel_width: 0, - pixel_height: 0, - }) - .map_err(process_error("resize PTY"))?; - Ok(ResizeProcessResponse {}) - } - - /// Records that a newly exited root left its group alive, for the idle - /// report, and starts the drain. The readers are left on the pipes; the - /// drain task switches them to discard mode once the grace passes. - fn observe_exit(&self, entry: &Arc, pgid: u32) { - record_leftover_group(pgid, &self.leftover_groups); - spawn_group_drain(entry.clone()); + _ctx: &RequestContext, + params: WriteProcessParams, + ) -> Result { + self.maintain(); + let entry = self.lookup(¶ms.process_id)?; + let accepted_bytes = entry + .execution + .offer_input(params.data.as_slice(), params.close_input)?; + Ok(WriteProcessResponse { accepted_bytes }) } - fn resolve_cwd(&self, path: &EnvironmentPath) -> Result { - let candidate = if path.is_absolute() { - PathBuf::from(path.as_str()) - } else if path.as_str() == "." { - self.cwd.clone() - } else { - self.cwd.join(path.as_str()) + pub async fn terminate( + &self, + _ctx: &RequestContext, + params: TerminateProcessParams, + ) -> Result { + self.maintain(); + let entry = self.lookup(¶ms.process_id)?; + let running = match params.signal { + ProcessSignal::Interrupt => entry.execution.interrupt(), + ProcessSignal::Terminate => entry.execution.stop(StoppedBy::Terminate), + ProcessSignal::Kill => entry.execution.stop(StoppedBy::Kill), }; - let normalized = normalize_path(candidate); - if !normalized.starts_with(&self.fs_root) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Forbidden, - format!( - "process cwd is outside bridge fs root: {} (root {})", - normalized.display(), - self.fs_root.display() - ), - )); - } - Ok(normalized) - } - - async fn entry(&self, process_id: &ProcessId) -> Option> { - self.processes - .lock() - .await - .get(process_id.as_str()) - .cloned() - } - - /// Drops the entry after the retention period. Readers are never - /// aborted here: those a sweep abandoned are gone already, and those - /// draining a pipe a leftover still holds must stay on it until end of - /// file so the leftover never sees `EPIPE`. - fn schedule_terminal_cleanup(&self, process_id: ProcessId, entry: Arc) { - let processes = self.processes.clone(); - tokio::spawn(async move { - tokio::time::sleep(TERMINAL_PROCESS_RETENTION).await; - let mut processes = processes.lock().await; - if processes - .get(process_id.as_str()) - .is_some_and(|current| Arc::ptr_eq(current, &entry)) - { - processes.remove(process_id.as_str()); - } - }); + Ok(TerminateProcessResponse { running }) } -} -impl ProcessState { - fn new( - child: ProcessChild, - pid: Option, - pgid: Option, - stdin: Option, - pty_master: Option>, - redactions: Vec>, - readers_open: u32, - ) -> Self { - Self { - child, - pid, - pgid, - stdin, - pty_master, - redactions, - output: OutputBuffer::default(), - readers_open, - exited: false, - exit_observed_at: None, - exit_code: None, - termination: None, - leftover_processes: Vec::new(), - failure: None, - cleanup_scheduled: false, - discarding: false, + /// Claims `process_id` before any side effect: a running, retained, + /// tombstoned or earlier-instance id is `alreadyExists`. + fn reserve(&self, process_id: &ProcessId) -> Result<(), ProtocolError> { + let mut table = self.table(); + if table.slots.contains_key(process_id) { + return Err(ProtocolError::already_exists(format!( + "process id {process_id} is already in use" + ))); } + table.slots.insert(process_id.clone(), Slot::Reserved); + Ok(()) } - fn push_output(&mut self, stream: ProcessOutputStream, bytes: &[u8]) { - if self.discarding { - return; + fn release(&self, process_id: &ProcessId) { + let mut table = self.table(); + if matches!(table.slots.get(process_id), Some(Slot::Reserved)) { + table.slots.remove(process_id); } - let bytes = redact_bytes(bytes, &self.redactions); - self.output.push(stream, bytes); } - fn reader_closed(&mut self) { - self.readers_open = self.readers_open.saturating_sub(1); - } -} - -/// Whether a read may return for an exited root: the readers reached end of -/// file, or the drain grace since the exit passed and leftovers still hold -/// the pipes. Also returns when to wake up for the grace when it has not. -fn exit_settled(state: &ProcessState, now: Instant) -> (bool, Option) { - if !state.exited { - return (false, None); - } - if state.readers_open == 0 || state.discarding { - return (true, None); + fn lookup(&self, process_id: &ProcessId) -> Result, ProtocolError> { + match self.table().slots.get(process_id) { + Some(Slot::Live(entry)) => Ok(entry.clone()), + Some(Slot::Tombstone(ended)) => Err(ProtocolError::not_found_because( + NotFoundReason::Expired, + format!("process {process_id} {ended} and its output is no longer retained"), + )), + Some(Slot::Restarted) => Err(ProtocolError::not_found_because( + NotFoundReason::DaemonRestarted, + format!("process {process_id} belongs to an earlier daemon instance"), + )), + Some(Slot::Reserved) | None => Err(ProtocolError::not_found(format!( + "unknown process id: {process_id}" + ))), + } } - let Some(exit_observed_at) = state.exit_observed_at else { - return (true, None); - }; - let drain_deadline = exit_observed_at + process_group::OUTPUT_DRAIN_GRACE; - (now >= drain_deadline, Some(drain_deadline)) -} -async fn read_stream(entry: Arc, mut reader: R, stream: ProcessOutputStream) -where - R: AsyncRead + Unpin, -{ - let mut buffer = vec![0; READ_BUFFER_BYTES]; - loop { - match reader.read(&mut buffer).await { - Ok(0) => break, - Ok(read) => { - let mut state = entry.state.lock().await; - state.push_output(stream, &buffer[..read]); - entry.notify.notify_waiters(); + /// Evicts settled entries past their retention or beyond the exited + /// byte budget, oldest exit first, and prunes old tombstones. + fn maintain(&self) { + let limits = &self.inner.config.limits; + let now = Instant::now(); + let after_read = Duration::from_millis(limits.retention_after_exit_ms); + let unread = Duration::from_millis(limits.unread_retention_ms); + let mut table = self.table(); + + let mut settled = Vec::new(); + let mut evict = Vec::new(); + for (process_id, slot) in &table.slots { + let Slot::Live(entry) = slot else { + continue; + }; + let Some((settled_at, retained)) = entry.execution.with_state(|state| { + state + .settled_at() + .map(|at| (at, state.sink.log.retained_bytes())) + }) else { + continue; + }; + let read_at = *lock(&entry.fully_read_at); + let expired = + read_at.is_some_and(|at| now >= at + after_read) || now >= settled_at + unread; + if expired { + evict.push(process_id.clone()); + } else { + settled.push((settled_at, retained, process_id.clone())); } - Err(error) => { - let mut state = entry.state.lock().await; - if state.failure.is_none() && !state.exited { - state.failure = Some(error.to_string()); - } + } + settled.sort(); + let mut total: u64 = settled.iter().map(|(_, retained, _)| retained).sum(); + for (_, retained, process_id) in settled { + if total <= limits.max_exited_retained_bytes { break; } + total -= retained; + evict.push(process_id); } - } - entry.state.lock().await.reader_closed(); - entry.notify.notify_waiters(); -} -fn read_pty_stream( - entry: Arc, - mut reader: Box, -) -> JoinHandle<()> { - tokio::task::spawn_blocking(move || { - let handle = tokio::runtime::Handle::current(); - let mut buffer = vec![0; READ_BUFFER_BYTES]; - loop { - match reader.read(&mut buffer) { - Ok(0) => break, - Ok(read) => { - let entry = entry.clone(); - let bytes = buffer[..read].to_vec(); - handle.block_on(async move { - let mut state = entry.state.lock().await; - state.push_output(ProcessOutputStream::Pty, &bytes); - entry.notify.notify_waiters(); - }); - } - Err(error) => { - let entry = entry.clone(); - handle.block_on(async move { - let mut state = entry.state.lock().await; - if state.failure.is_none() && !state.exited { - state.failure = Some(error.to_string()); - } - }); - break; + for process_id in evict { + let Some(Slot::Live(entry)) = table.slots.remove(&process_id) else { + continue; + }; + let ended = entry.execution.with_state(|state| { + let mut ended = state + .exit() + .map_or_else(|| "ended".to_owned(), describe_exit); + if let Some(stopped_by) = state.stopped_by() { + let _ = write!(ended, " (stopped by {stopped_by:?})"); } + // Readers a leftover keeps on the pipes outlive the entry; + // they discard, and the retained output goes now. + state.sink.log = OutputLog::default(); + ended + }); + let group = entry.execution.group(); + if entry.execution.group_alive() { + lock(&self.inner.leftover_groups) + .insert(group.pgid, (group, entry.record_path.clone())); + } else { + let _ = std::fs::remove_file(&entry.record_path); } + table + .slots + .insert(process_id.clone(), Slot::Tombstone(ended)); + table.tombstones.push_back((process_id, now)); } - handle.block_on(async move { - entry.state.lock().await.reader_closed(); - entry.notify.notify_waiters(); - }); - }) -} -/// Observes the child's exit independently of pipe events so a silent child -/// whose leftovers hold the pipes open still completes promptly. -async fn watch_exit(entry: Arc, leftover_groups: Arc>>) { - loop { - tokio::time::sleep(EXIT_POLL_INTERVAL).await; - let done = { - let mut state = entry.state.lock().await; - if state.exited { - true - } else { - match update_exit_status(&mut state) { - Ok(Some(pgid)) => { - record_leftover_group(pgid, &leftover_groups); - spawn_group_drain(entry.clone()); - true - } - Ok(None) => state.exited, - Err(_) => true, - } + while let Some((_, evicted_at)) = table.tombstones.front() { + if table.tombstones.len() <= MAX_TOMBSTONES && now < *evicted_at + TOMBSTONE_RETENTION { + break; + } + let (process_id, _) = table.tombstones.pop_front().expect("front exists"); + if matches!(table.slots.get(&process_id), Some(Slot::Tombstone(_))) { + table.slots.remove(&process_id); } - }; - if done { - entry.notify.notify_waiters(); - return; } } -} -async fn timeout_process(entry: Arc, timeout: Duration) { - tokio::time::sleep(timeout).await; - { - let mut state = entry.state.lock().await; - if state.exited { - return; - } - if let Some(pgid) = state.pgid { - process_group::kill_group(pgid); - } - let _ = kill_child(&mut state.child).await; - state.exited = true; - state.exit_observed_at = Some(Instant::now()); - state.exit_code = None; - state.termination = Some(ProcessTermination::TimedOut); - state.stdin.take(); + fn table(&self) -> MutexGuard<'_, Table> { + lock(&self.inner.table) } - drain_or_abort_readers(&entry).await; - entry.notify.notify_waiters(); } -/// Polls the child for exit. When the exit is newly observed while members -/// of its process group are still alive, returns the group id so the caller -/// can record the leftovers. -fn update_exit_status(state: &mut ProcessState) -> Result, EnvironmentProtocolError> { - if state.exited { - return Ok(None); - } - let result = match &mut state.child { - ProcessChild::Pipe(child) => child - .try_wait() - .map(|status| status.map(|status| status.code())), - ProcessChild::Pty(child) => child - .try_wait() - .map(|status| status.map(|status| Some(status.exit_code() as i32))), - }; - match result { - Ok(Some(status)) => { - state.exited = true; - state.exit_observed_at = Some(Instant::now()); - state.exit_code = status; - state.stdin.take(); - Ok(state.pgid.filter(|pgid| process_group::group_alive(*pgid))) - } - Ok(None) => Ok(None), - Err(error) => { - state.failure = Some(error.to_string()); - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - format!("poll process exit: {error}"), - )) - } - } +fn validate_pty(size: PtySize, input: &ExecInput) -> Result<(), ProtocolError> { + let valid = 1..=PtySize::MAX_DIMENSION; + if !valid.contains(&size.rows) || !valid.contains(&size.cols) { + return Err(ProtocolError::invalid_field( + "pty", + format!("PTY rows and cols must be 1 to {}", PtySize::MAX_DIMENSION), + )); + } + if matches!(input, ExecInput::Pipe {}) { + return Err(ProtocolError::invalid_field( + "command.input", + "a PTY is the input; pipe input cannot be combined with pty", + )); + } + Ok(()) } -fn record_leftover_group(pgid: u32, leftover_groups: &StdMutex>) { - leftover_groups +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex .lock() - .expect("leftover groups poisoned") - .insert(pgid); + .unwrap_or_else(|poisoned| poisoned.into_inner()) } -async fn kill_child(child: &mut ProcessChild) -> Result<(), EnvironmentProtocolError> { - let result = match child { - ProcessChild::Pipe(child) => child.kill().await, - ProcessChild::Pty(child) => child.kill(), - }; - result.map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - }) -} +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; -fn interrupt_child(child: &mut ProcessChild) { - let pid = match child { - ProcessChild::Pipe(child) => child.id(), - ProcessChild::Pty(child) => child.process_id(), + use environment_protocol::{ + bytes::{ByteChunk, SecretString}, + data::exec::{CommandSpec, ExitStatus, GapReason, SignalName}, + error::{ErrorData, PreconditionReason, ProtocolErrorKind}, }; - #[cfg(unix)] - if let Some(pid) = pid - .and_then(|pid| i32::try_from(pid).ok()) - .and_then(rustix::process::Pid::from_raw) - { - let _ = rustix::process::kill_process(pid, rustix::process::Signal::INT); - } - #[cfg(not(unix))] - let _ = pid; -} -fn process_error( - context: &'static str, -) -> impl FnOnce(E) -> EnvironmentProtocolError { - move |error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - format!("{context}: {error}"), - ) - } -} - -/// After a natural exit, gives the readers the drain grace to reach end of -/// file so bytes the root wrote just before exiting are kept, then leaves -/// them on the pipes in discard mode. The pipe's own end of file, when the -/// last leftover holding it exits, ends them. The group is never signalled. -fn spawn_group_drain(entry: Arc) { - tokio::spawn(async move { - tokio::time::sleep(process_group::OUTPUT_DRAIN_GRACE).await; - let mut state = entry.state.lock().await; - if state.readers_open > 0 { - state.discarding = true; - } - drop(state); - entry.notify.notify_waiters(); - }); -} - -fn spawn_reader_drain(entry: Arc) { - tokio::spawn(async move { - drain_or_abort_readers(&entry).await; - entry.notify.notify_waiters(); - }); -} + use super::*; -/// After the group was killed: waits the drain grace for the readers, then -/// abandons any still stuck on a pipe an escaped descendant holds. -async fn drain_or_abort_readers(entry: &ProcessEntry) { - let mut readers = std::mem::take(&mut *entry.readers.lock().await); - let deadline = Instant::now() + process_group::OUTPUT_DRAIN_GRACE; - for reader in &mut readers { - if tokio::time::timeout_at(deadline, &mut *reader) - .await - .is_err() - { - reader.abort(); + fn limits() -> ProcessLimits { + ProcessLimits { + max_read_bytes: 4 * 1024 * 1024, + max_retained_output_bytes: 4 * 1024 * 1024, + retention_after_exit_ms: 60_000, + unread_retention_ms: 7_200_000, + max_exited_retained_bytes: 64 * 1024 * 1024, + terminate_grace_ms: 200, + max_input_queue_bytes: 64 * 1024, } } - let mut state = entry.state.lock().await; - state.readers_open = 0; - state.discarding = true; -} -fn response_from_state( - state: &mut ProcessState, - after_seq: Option, - max_bytes: Option, -) -> ReadProcessResponse { - let (chunks, omitted_bytes, next_seq) = match after_seq { - None => { - let (chunks, omitted) = state.output.deliver(max_bytes); - (chunks, omitted, state.output.delivered_seq) - } - Some(after_seq) => { - let (chunks, omitted) = state.output.reread(after_seq, max_bytes); - let next_seq = chunks - .last() - .map_or(after_seq.max(state.output.delivered_seq), |chunk| { - chunk.seq + 1 - }); - (chunks, omitted, next_seq) - } - }; - // Leftovers are sampled when the caller looks, so the list reflects - // what is alive now rather than the instant the root exited, when a - // background child may still have been mid-exec. - if state.exited { - state.leftover_processes = match state.pgid.filter(|pgid| process_group::group_alive(*pgid)) - { - Some(pgid) => sample_group_members(pgid), - None => Vec::new(), - }; - } - ReadProcessResponse { - chunks, - next_seq, - exited: state.exited, - exit_code: state.exit_code, - closed: state.exited, - failure: state.failure.clone(), - termination: state.termination, - pid: state.pid, - omitted_bytes, - leftover_processes: state.leftover_processes.clone(), + struct Fixture { + _temp: tempfile::TempDir, + root: PathBuf, + state_dir: PathBuf, + manager: ProcessManager, } -} -fn normalize_path(path: PathBuf) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(std::path::MAIN_SEPARATOR.to_string()), - Component::CurDir => {} - Component::ParentDir => { - normalized.pop(); - } - Component::Normal(segment) => normalized.push(segment), + fn fixture_with(limits: ProcessLimits) -> Fixture { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path().canonicalize().expect("canonical root"); + let state_dir = root.join("state"); + let manager = manager_at(&root, &state_dir, limits, "envd_first"); + Fixture { + _temp: temp, + root, + state_dir, + manager, } } - normalized -} -/// Live, non-zombie members of a process group with their command lines, -/// best effort: `/proc` on Linux, libproc on macOS, empty elsewhere. -#[cfg(target_os = "linux")] -fn sample_group_members(pgid: u32) -> Vec { - let Ok(entries) = std::fs::read_dir("/proc") else { - return Vec::new(); - }; - let mut members = Vec::new(); - for entry in entries.flatten() { - let Some(pid) = entry - .file_name() - .to_str() - .and_then(|name| name.parse::().ok()) - else { - continue; - }; - let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) else { - continue; - }; - // "pid (comm) state ppid pgrp ..."; comm may contain spaces and - // parentheses, so split at the last closing parenthesis. - let Some(open) = stat.find('(') else { - continue; - }; - let Some(close) = stat.rfind(')') else { - continue; - }; - let mut fields = stat[close + 1..].split_whitespace(); - let state = fields.next(); - let _ppid = fields.next(); - let pgrp = fields.next().and_then(|value| value.parse::().ok()); - if pgrp != Some(pgid) || state == Some("Z") { - continue; - } - let command = std::fs::read(entry.path().join("cmdline")) - .ok() - .map(|bytes| { - bytes - .split(|byte| *byte == 0) - .filter(|part| !part.is_empty()) - .map(|part| String::from_utf8_lossy(part).into_owned()) - .collect::>() - .join(" ") - }) - .filter(|command| !command.is_empty()) - .unwrap_or_else(|| stat[open + 1..close].to_owned()); - members.push(LeftoverProcess { pid, command }); + fn fixture() -> Fixture { + fixture_with(limits()) + } + + fn manager_at( + root: &std::path::Path, + state_dir: &std::path::Path, + limits: ProcessLimits, + instance: &str, + ) -> ProcessManager { + let confinement = + Confinement::new(root.to_owned(), state_dir.to_owned()).expect("confinement"); + ProcessManager::new(ProcessConfig { + cwd: root.to_owned(), + confinement: Arc::new(confinement), + state_dir: state_dir.to_owned(), + scrubbed_env: Vec::new(), + instance_id: DaemonInstanceId::new(instance).unwrap(), + limits, + max_wait_ms: 60_000, + pty_supported: true, + }) + .expect("manager") } - members.sort_by_key(|member| member.pid); - members -} -#[cfg(target_os = "macos")] -fn sample_group_members(pgid: u32) -> Vec { - use std::{ffi::c_void, mem}; - - let Ok(group) = libc::pid_t::try_from(pgid) else { - return Vec::new(); - }; - let pid_size = mem::size_of::(); - // libproc's group listing returns pid counts, not bytes: with a null - // buffer the count of every process on the system, which is an upper - // bound for the group, and with a buffer the count it filled. - let upper_bound = unsafe { libc::proc_listpgrppids(group, std::ptr::null_mut(), 0) }; - if upper_bound <= 0 { - return Vec::new(); - } - let mut pids = vec![0 as libc::pid_t; upper_bound as usize + 16]; - let count = unsafe { - libc::proc_listpgrppids( - group, - pids.as_mut_ptr() as *mut c_void, - (pids.len() * pid_size) as libc::c_int, - ) - }; - if count <= 0 { - return Vec::new(); - } - pids.truncate(count as usize); - let mut members = Vec::new(); - for pid in pids { - if pid <= 0 { - continue; - } - let mut info: libc::proc_bsdinfo = unsafe { mem::zeroed() }; - let info_size = mem::size_of::() as libc::c_int; - let read = unsafe { - libc::proc_pidinfo( - pid, - libc::PROC_PIDTBSDINFO, - 0, - &mut info as *mut libc::proc_bsdinfo as *mut c_void, - info_size, - ) - }; - if read == info_size && info.pbi_status == libc::SZOMB { - continue; - } - let mut path = vec![0u8; 4096]; - let len = - unsafe { libc::proc_pidpath(pid, path.as_mut_ptr() as *mut c_void, path.len() as u32) }; - let command = if len > 0 { - String::from_utf8_lossy(&path[..len as usize]).into_owned() - } else { - String::new() - }; - members.push(LeftoverProcess { - pid: pid as u32, - command, - }); + fn ctx() -> RequestContext { + RequestContext::detached() } - members.sort_by_key(|member| member.pid); - members -} - -#[cfg(not(any(target_os = "linux", target_os = "macos")))] -fn sample_group_members(_pgid: u32) -> Vec { - Vec::new() -} - -#[cfg(test)] -mod tests { - use super::*; - use environment_protocol::shared::SecretString; - fn manager() -> (tempfile::TempDir, PathBuf, ProcessManager) { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = ProcessManager::new(root.clone(), root.clone()); - (temp, root, manager) + fn id(value: &str) -> ProcessId { + ProcessId::new(value).unwrap() } - fn params(process_id: &str, script: &str) -> StartProcessParams { + fn shell(process_id: &str, script: &str) -> StartProcessParams { StartProcessParams { - process_id: ProcessId::new(process_id), - argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(30_000), - tty: false, + process_id: id(process_id), + command: CommandSpec { + argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], + cwd: None, + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input: ExecInput::None {}, + timeout_ms: Some(30_000), + }, + pty: None, } } - fn read_params(process_id: &str, wait_ms: Option) -> ReadProcessParams { + fn read_params(process_id: &str, return_on: ReturnOn, wait_ms: u64) -> ReadProcessParams { ReadProcessParams { - process_id: ProcessId::new(process_id), - after_seq: None, - max_bytes: None, + process_id: id(process_id), + after: None, + tail_bytes: None, + ack: None, + max_bytes: 4 * 1024 * 1024, wait_ms, + return_on, } } - fn stream_bytes(response: &ReadProcessResponse, stream: ProcessOutputStream) -> Vec { - response - .chunks - .iter() - .filter(|chunk| chunk.stream == stream) - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) - .collect() + fn written(reply: Reply) -> ReadProcessResponse { + if let Some(hook) = reply.on_written { + hook(); + } + reply.value } - fn all_bytes(response: &ReadProcessResponse) -> Vec { + fn data(response: &ReadProcessResponse) -> Vec { response + .output .chunks .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) + .flat_map(|chunk| chunk.data.as_slice().to_vec()) .collect() } - #[tokio::test(flavor = "current_thread")] - async fn process_reports_stdout_stderr_exit_code_and_pid() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params("proc-1", "printf out; printf err >&2")) - .await - .expect("start"); - - let output = manager - .read_process(read_params("proc-1", None)) - .await - .expect("read"); - - assert!(output.exited); - assert_eq!(output.exit_code, Some(0)); - assert!(output.pid.is_some()); - assert_eq!(output.termination, None); - assert!(output.leftover_processes.is_empty()); - assert_eq!(stream_bytes(&output, ProcessOutputStream::Stdout), b"out"); - assert_eq!(stream_bytes(&output, ProcessOutputStream::Stderr), b"err"); + /// Cursor reads until the settled exit with nothing left unread. + async fn read_to_end( + manager: &ProcessManager, + process_id: &str, + ) -> (Vec, ReadProcessResponse) { + let mut bytes = Vec::new(); + loop { + let reply = manager + .read(&ctx(), read_params(process_id, ReturnOn::AnyOutput, 10_000)) + .await + .expect("read"); + let response = written(reply); + bytes.extend(data(&response)); + if response.state == ExecState::Exited && response.output.next == response.output.end { + return (bytes, response); + } + } } - #[tokio::test(flavor = "current_thread")] - async fn terminal_process_records_are_pruned_and_reusable() { - let (_temp, _root, manager) = manager(); - let params = params("proc-reused", "true"); + fn not_found_reason(error: &ProtocolError) -> Option { + match &error.data { + ErrorData::NotFound { reason } => *reason, + _ => panic!("expected notFound, got {error:?}"), + } + } - manager - .start_process(params.clone()) - .await - .expect("start first"); - let output = manager - .read_process(read_params("proc-reused", None)) - .await - .expect("read terminal output"); - assert!(output.exited); + #[cfg(unix)] + async fn wait_for_pid_file(path: &std::path::Path) -> i32 { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Ok(content) = std::fs::read_to_string(path) + && let Ok(pid) = content.trim().parse() + { + return pid; + } + assert!(Instant::now() < deadline, "pid file was not written"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } - tokio::time::sleep(TERMINAL_PROCESS_RETENTION * 2).await; + #[cfg(unix)] + fn process_alive(pid: i32) -> bool { + rustix::process::Pid::from_raw(pid) + .is_some_and(|pid| rustix::process::test_kill_process(pid).is_ok()) + } - manager - .start_process(params) - .await - .expect("start reused id"); + #[cfg(unix)] + fn kill(pid: i32) { + let _ = rustix::process::kill_process( + rustix::process::Pid::from_raw(pid).expect("pid"), + rustix::process::Signal::KILL, + ); } #[tokio::test(flavor = "current_thread")] - async fn plain_pipe_process_reads_eof_on_stdin_at_once() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params("proc-cat", "cat; echo done")) + async fn reports_both_streams_exit_code_and_pid() { + let fixture = fixture(); + let started = fixture + .manager + .start(&ctx(), shell("p1", "printf out; printf err >&2; exit 3")) .await .expect("start"); - let output = tokio::time::timeout( + let (bytes, response) = read_to_end(&fixture.manager, "p1").await; + assert_eq!(response.pid, started.pid); + assert_eq!(response.exit, Some(ExitStatus::Code { code: 3 })); + assert_eq!(response.stopped_by, None); + assert!(response.leftovers.is_none()); + // The two streams interleave in arrival order. + let mut sorted = bytes; + sorted.sort_unstable(); + let mut expected = b"outerr".to_vec(); + expected.sort_unstable(); + assert_eq!(sorted, expected); + assert_eq!(response.output.end, 6); + } + + #[tokio::test(flavor = "current_thread")] + async fn large_bytes_input_never_blocks_the_start() { + // More input than any pipe buffer, to a child that writes before it + // has read everything: the start must not wait for the input. + let fixture = fixture(); + let input: Vec = (0..1024 * 1024u32).map(|i| b'a' + (i % 26) as u8).collect(); + let mut params = shell("cat", "cat"); + params.command.input = ExecInput::Bytes { + data: ByteChunk::from(input.clone()), + }; + tokio::time::timeout( Duration::from_secs(5), - manager.read_process(read_params("proc-cat", None)), + fixture.manager.start(&ctx(), params), ) .await - .expect("cat exits without waiting for input") - .expect("read"); - assert!(output.exited); - assert_eq!( - stream_bytes(&output, ProcessOutputStream::Stdout), - b"done\n" - ); - - let write = manager - .write_process(WriteProcessParams { - process_id: ProcessId::new("proc-cat"), - chunk: Some(ByteChunk::from(b"late".as_slice())), - close_stdin: false, - }) - .await - .expect("write"); - assert_eq!(write.status, WriteProcessStatus::StdinClosed); + .expect("the start answers at once") + .expect("start"); + let (bytes, response) = read_to_end(&fixture.manager, "cat").await; + assert_eq!(response.exit, Some(ExitStatus::Code { code: 0 })); + assert_eq!(bytes, input, "every byte went through"); } #[tokio::test(flavor = "current_thread")] - async fn one_shot_stdin_is_written_and_closed_at_start() { - let (_temp, _root, manager) = manager(); - let mut params = params("proc-stdin", "cat"); - params.stdin = Some(ByteChunk::from(b"hello".as_slice())); - manager.start_process(params).await.expect("start"); - let output = manager - .read_process(read_params("proc-stdin", None)) + async fn a_large_pty_write_never_freezes_the_daemon() { + // A PTY child that never reads: a write must not block the runtime + // thread under the entry lock until the child is killed. + let fixture = fixture(); + let mut params = shell("pty", "sleep 30"); + params.pty = Some(PtySize { rows: 24, cols: 80 }); + fixture.manager.start(&ctx(), params).await.expect("start"); + let chunk = vec![b'x'; 16 * 1024]; + let mut accepted = 0; + for _ in 0..64 { + let write = tokio::time::timeout( + Duration::from_secs(1), + fixture.manager.write( + &ctx(), + WriteProcessParams { + process_id: id("pty"), + data: ByteChunk::from(chunk.clone()), + close_input: false, + }, + ), + ) .await - .expect("read"); - assert!(output.exited); - assert_eq!(stream_bytes(&output, ProcessOutputStream::Stdout), b"hello"); + .expect("a write answers without blocking") + .expect("write"); + accepted += write.accepted_bytes; + } + // Whether the terminal blocks or discards input a child never reads + // differs by platform; either way no write may exceed what was sent. + assert!(accepted <= 64 * 16 * 1024); + // The runtime thread is free, and a kill still works. + tokio::time::sleep(Duration::from_millis(10)).await; + let terminated = fixture + .manager + .terminate( + &ctx(), + TerminateProcessParams { + process_id: id("pty"), + signal: ProcessSignal::Kill, + }, + ) + .await + .expect("kill"); + assert!(terminated.running); + let (_, response) = read_to_end(&fixture.manager, "pty").await; + assert_eq!(response.stopped_by, Some(StoppedBy::Kill)); } #[tokio::test(flavor = "current_thread")] - async fn pty_process_accepts_input_echoes_and_resizes() { - let (_temp, _root, manager) = manager(); - let mut params = params("proc-pty", "read line; printf '%s' \"$line\""); - params.tty = true; - manager.start_process(params).await.expect("start PTY"); - manager - .resize_process(ResizeProcessParams { - process_id: ProcessId::new("proc-pty"), - size: environment_protocol::data::process::TerminalSize { - rows: 40, - cols: 120, - }, - }) - .await - .expect("resize PTY"); - let write = manager - .write_process(WriteProcessParams { - process_id: ProcessId::new("proc-pty"), - chunk: Some(ByteChunk::from(b"hello\n".as_slice())), - close_stdin: false, - }) - .await - .expect("write PTY"); - assert_eq!(write.status, WriteProcessStatus::Accepted); - let output = manager - .read_process(read_params("proc-pty", None)) - .await - .expect("read PTY"); - assert!(output.exited); - assert!( - output - .chunks - .iter() - .all(|chunk| chunk.stream == ProcessOutputStream::Pty) + async fn pipe_input_streams_until_closed() { + let fixture = fixture(); + let mut params = shell("pipe", "cat; echo done"); + params.command.input = ExecInput::Pipe {}; + fixture.manager.start(&ctx(), params).await.expect("start"); + let write = |data: &'static [u8], close_input| WriteProcessParams { + process_id: id("pipe"), + data: ByteChunk::from(data), + close_input, + }; + assert_eq!( + fixture + .manager + .write(&ctx(), write(b"hello ", false)) + .await + .unwrap() + .accepted_bytes, + 6 + ); + assert_eq!( + fixture + .manager + .write(&ctx(), write(b"world\n", true)) + .await + .unwrap() + .accepted_bytes, + 6 ); - assert!(String::from_utf8_lossy(&all_bytes(&output)).contains("hello")); + let (bytes, response) = read_to_end(&fixture.manager, "pipe").await; + assert_eq!(bytes, b"hello world\ndone\n"); + assert_eq!(response.exit, Some(ExitStatus::Code { code: 0 })); + let error = fixture + .manager + .write(&ctx(), write(b"late", false)) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::InputClosed + } + )); } #[tokio::test(flavor = "current_thread")] - async fn wait_collects_for_the_whole_window_or_until_exit() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params( - "proc-window", - "for i in 1 2 3 4 5 6 7 8 9 10; do echo line$i; sleep 0.02; done", - )) + async fn the_cursor_moves_only_once_the_reply_is_written() { + let fixture = fixture(); + fixture + .manager + .start(&ctx(), shell("cursor", "printf first; sleep 30")) .await .expect("start"); - - let started = std::time::Instant::now(); - let output = manager - .read_process(read_params("proc-window", Some(10_000))) + let lost = fixture + .manager + .read(&ctx(), read_params("cursor", ReturnOn::AnyOutput, 5_000)) .await .expect("read"); - assert!(output.exited, "a long wait returns at exit"); - assert!(started.elapsed() < Duration::from_secs(5)); - let text = String::from_utf8_lossy(&all_bytes(&output)).into_owned(); - assert_eq!(text.lines().count(), 10, "every line arrives in one read"); - - manager - .start_process(params( - "proc-short", - "for i in 1 2 3 4 5 6 7 8 9 10; do echo line$i; sleep 0.05; done", - )) + assert_eq!(data(&lost.value), b"first"); + // The connection dropped before the frame went out: no hook runs. + drop(lost); + let again = written( + fixture + .manager + .read(&ctx(), read_params("cursor", ReturnOn::AnyOutput, 5_000)) + .await + .expect("read again"), + ); + assert_eq!(data(&again), b"first", "nothing was consumed"); + let after = fixture + .manager + .read(&ctx(), read_params("cursor", ReturnOn::AnyOutput, 0)) .await - .expect("start short"); - let started = std::time::Instant::now(); - let output = manager - .read_process(read_params("proc-short", Some(120))) + .expect("read after commit") + .value; + assert!(after.output.chunks.is_empty()); + assert_eq!(after.output.next, 5); + + // A cancelled wait answers aborted and consumes nothing either. + let cancelled = ctx(); + cancelled.cancel.cancel(); + let error = fixture + .manager + .read(&cancelled, read_params("cursor", ReturnOn::Exit, 5_000)) .await - .expect("read short"); - let elapsed = started.elapsed(); - assert!(!output.exited); - assert!( - elapsed >= Duration::from_millis(100) && elapsed < Duration::from_millis(400), - "returned at the window, not the first chunk: {elapsed:?}" - ); - let lines = String::from_utf8_lossy(&all_bytes(&output)).lines().count(); - assert!( - (1..10).contains(&lines), - "partial output at the window: {lines} lines" - ); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::Aborted); + fixture + .manager + .terminate( + &ctx(), + TerminateProcessParams { + process_id: id("cursor"), + signal: ProcessSignal::Kill, + }, + ) + .await + .expect("kill"); } #[tokio::test(flavor = "current_thread")] - async fn cursor_reads_return_disjoint_output_and_rereads_do_not_move_it() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params( - "proc-cursor", - "echo first; sleep 0.2; echo second; sleep 0.2; echo third", - )) + async fn the_end_of_a_large_output_survives_and_exit_waits_for_it() { + // A command that prints far more than is retained and then its + // summary: the read at exit shows the head, one gap and the end. + let fixture = fixture_with(ProcessLimits { + max_retained_output_bytes: 64 * 1024, + ..limits() + }); + fixture + .manager + .start( + &ctx(), + shell( + "build", + "i=0; while [ $i -lt 20000 ]; do echo line-$i; i=$((i+1)); done; echo 'FAILED: 3 tests'", + ), + ) .await .expect("start"); - - let first = manager - .read_process(read_params("proc-cursor", Some(80))) - .await - .expect("first read"); - assert_eq!(all_bytes(&first), b"first\n"); - assert!(!first.exited); - - let reread = manager - .read_process(ReadProcessParams { - process_id: ProcessId::new("proc-cursor"), - after_seq: Some(0), - max_bytes: None, - wait_ms: Some(0), - }) - .await - .expect("reread"); - assert!( - reread.chunks.is_empty(), - "delivered chunks are not retained" + let response = written( + fixture + .manager + .read(&ctx(), read_params("build", ReturnOn::Exit, 30_000)) + .await + .expect("read"), ); - - let rest = manager - .read_process(read_params("proc-cursor", Some(10_000))) - .await - .expect("second read"); - assert!(rest.exited); - assert_eq!(all_bytes(&rest), b"second\nthird\n"); + assert_eq!(response.state, ExecState::Exited); + assert_eq!( + response.output.next, response.output.end, + "nothing left unread" + ); + assert_eq!(response.output.gaps.len(), 1); + assert_eq!(response.output.gaps[0].reason, GapReason::Overflow); + let text = String::from_utf8(data(&response)).unwrap(); + assert!(text.starts_with("line-0\n")); assert!( - rest.chunks - .iter() - .all(|chunk| chunk.seq > first.chunks[0].seq), - "the second read never repeats the first" + text.ends_with("FAILED: 3 tests\n"), + "the summary at the end survives" ); + let gap = response.output.gaps[0].omitted_bytes; + assert_eq!(text.len() as u64 + gap, response.output.end); } #[tokio::test(flavor = "current_thread")] - async fn max_bytes_splits_the_first_chunk_and_keeps_the_remainder() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params("proc-split", "printf abcdefgh")) + async fn pages_at_explicit_offsets_never_consume_and_tail_reads_start_at_the_end() { + let fixture = fixture(); + fixture + .manager + .start(&ctx(), shell("pages", "printf abcdefgh")) .await .expect("start"); - let head = manager - .read_process(ReadProcessParams { - process_id: ProcessId::new("proc-split"), - after_seq: None, - max_bytes: Some(3), - wait_ms: None, - }) - .await - .expect("head"); - assert_eq!(all_bytes(&head), b"abc"); - let tail = manager - .read_process(ReadProcessParams { - process_id: ProcessId::new("proc-split"), - after_seq: None, - max_bytes: Some(100), - wait_ms: None, - }) - .await - .expect("tail"); - assert_eq!(all_bytes(&tail), b"defgh"); - assert_eq!(head.chunks[0].seq, tail.chunks[0].seq); + let mut params = read_params("pages", ReturnOn::Exit, 10_000); + // `after` past the output produced so far is refused before any + // wait, so settle the process first with a non-consuming read. + params.after = Some(0); + let settled = written(fixture.manager.read(&ctx(), params.clone()).await.unwrap()); + assert_eq!(settled.state, ExecState::Exited); + assert_eq!(settled.output.end, 8); + params.after = Some(2); + params.max_bytes = 3; + let page = written(fixture.manager.read(&ctx(), params.clone()).await.unwrap()); + assert_eq!(data(&page), b"cde"); + assert_eq!(page.output.next, 5); + params.after = None; + params.tail_bytes = Some(2); + let tail = written(fixture.manager.read(&ctx(), params.clone()).await.unwrap()); + assert_eq!(data(&tail), b"gh"); + params.tail_bytes = None; + params.after = Some(9); + let error = fixture.manager.read(&ctx(), params).await.unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + let (all, _) = read_to_end(&fixture.manager, "pages").await; + assert_eq!(all, b"abcdefgh", "offset reads left the cursor at 0"); } #[tokio::test(flavor = "current_thread")] - async fn unread_output_is_capped_to_head_and_tail() { - let (_temp, _root, manager) = manager(); - // 64 KiB of numbered lines against a 16 KiB test cap. - manager - .start_process(params( - "proc-cap", - "i=0; while [ $i -lt 4096 ]; do printf '%015d\\n' $i; i=$((i+1)); done", - )) + async fn a_duplicate_id_runs_nothing() { + let fixture = fixture(); + let script = "echo run >> runs.txt"; + fixture + .manager + .start(&ctx(), shell("dup", script)) .await .expect("start"); - let output = manager - .read_process(read_params("proc-cap", None)) + let error = fixture + .manager + .start(&ctx(), shell("dup", script)) .await - .expect("read"); - assert!(output.exited); - let delivered = all_bytes(&output); - assert!( - delivered.len() <= RETAINED_OUTPUT_CAP, - "{}", - delivered.len() - ); - assert!(output.omitted_bytes > 0); - assert_eq!( - delivered.len() as u64 + output.omitted_bytes, - 4096 * 16, - "head + omitted + tail account for every byte" - ); - let text = String::from_utf8_lossy(&delivered).into_owned(); - assert!(text.starts_with("000000000000000\n"), "head kept"); - assert!(text.ends_with("000000000004095\n"), "tail kept"); - let gaps = output - .chunks - .windows(2) - .filter(|pair| pair[1].seq != pair[0].seq + 1) - .count(); - assert_eq!(gaps, 1, "one gap where the middle was"); - - let again = manager - .read_process(read_params("proc-cap", Some(0))) - .await - .expect("second read"); - assert!(again.chunks.is_empty()); - assert_eq!(again.omitted_bytes, 0, "the omission is reported once"); - } - - #[test] - fn cap_keeps_one_contiguous_gap_when_chunk_sizes_are_uneven() { - // 16 KiB test cap, 8 KiB head. Seven 1000-byte chunks fill the head; - // the 1500-byte chunk after them is the first to be dropped, which - // leaves a 500-byte survivor right behind the head. A head recomputed - // purely by bytes would absorb that survivor on the next drop and - // open a second gap behind it. - let mut buffer = OutputBuffer::default(); - let mut push = |len: usize| buffer.push(ProcessOutputStream::Stdout, vec![b'x'; len]); - for _ in 0..7 { - push(1000); - } - push(1500); - push(500); - for _ in 0..5 { - push(1500); - } - push(1500); - let total = 7 * 1000 + 1500 + 500 + 5 * 1500 + 1500; - - let (chunks, omitted) = buffer.deliver(None); - - let delivered: usize = chunks - .iter() - .map(|chunk| chunk.chunk.as_slice().len()) - .sum(); - assert!(delivered <= RETAINED_OUTPUT_CAP, "{delivered}"); - assert!(omitted > 0); - assert_eq!(delivered as u64 + omitted, total as u64); - let gaps = chunks - .windows(2) - .filter(|pair| pair[1].seq != pair[0].seq + 1) - .count(); - assert_eq!(gaps, 1, "one contiguous gap after the head"); - assert_eq!(chunks[6].seq, 6, "the whole head survives"); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::AlreadyExists); + read_to_end(&fixture.manager, "dup").await; + let runs = std::fs::read_to_string(fixture.root.join("runs.txt")).unwrap(); + assert_eq!(runs, "run\n", "the command ran once"); } #[tokio::test(flavor = "current_thread")] - async fn empty_write_is_a_wait_until_the_entry_is_pruned() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params("proc-wait", "echo bye; exit 7")) + async fn unread_exited_processes_expire_into_tombstones() { + let fixture = fixture_with(ProcessLimits { + unread_retention_ms: 50, + ..limits() + }); + fixture + .manager + .start(&ctx(), shell("old", "true")) .await .expect("start"); for _ in 0..200 { - if manager.running_count().await == 0 { + if fixture.manager.running_count().await == 0 { break; } tokio::time::sleep(Duration::from_millis(10)).await; } - let write = manager - .write_process(WriteProcessParams { - process_id: ProcessId::new("proc-wait"), - chunk: None, - close_stdin: false, - }) + tokio::time::sleep(process_group::OUTPUT_DRAIN_GRACE + Duration::from_millis(100)).await; + fixture.manager.running_count().await; + let error = fixture + .manager + .read(&ctx(), read_params("old", ReturnOn::Exit, 0)) .await - .expect("empty write"); - assert_eq!(write.status, WriteProcessStatus::Accepted); - let output = manager - .read_process(read_params("proc-wait", Some(1_000))) + .unwrap_err(); + assert_eq!(not_found_reason(&error), Some(NotFoundReason::Expired)); + let error = fixture + .manager + .start(&ctx(), shell("old", "true")) .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.exit_code, Some(7)); - assert_eq!(all_bytes(&output), b"bye\n"); - - tokio::time::sleep(TERMINAL_PROCESS_RETENTION * 3).await; - let write = manager - .write_process(WriteProcessParams { - process_id: ProcessId::new("proc-wait"), - chunk: None, - close_stdin: false, - }) - .await - .expect("late write"); - assert_eq!(write.status, WriteProcessStatus::UnknownProcess); - } - - #[cfg(unix)] - async fn wait_for_pid_file(path: &std::path::Path) -> i32 { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); - loop { - if let Ok(content) = std::fs::read_to_string(path) - && let Ok(pid) = content.trim().parse() - { - return pid; - } - assert!( - std::time::Instant::now() < deadline, - "pid file was not written" - ); - tokio::time::sleep(Duration::from_millis(20)).await; - } - } - - #[cfg(unix)] - fn process_alive(pid: i32) -> bool { - rustix::process::Pid::from_raw(pid) - .is_some_and(|pid| rustix::process::test_kill_process(pid).is_ok()) - } - - #[cfg(unix)] - async fn wait_for_process_gone(pid: i32) { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); - loop { - if !process_alive(pid) { - return; - } - assert!( - std::time::Instant::now() < deadline, - "descendant {pid} was not terminated" - ); - tokio::time::sleep(Duration::from_millis(20)).await; - } + .unwrap_err(); + assert_eq!( + error.kind(), + ProtocolErrorKind::AlreadyExists, + "a tombstoned id is not reused" + ); } - #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn kill_terminates_descendants_in_group() { - let (_temp, root, manager) = manager(); - manager - .start_process(params( - "proc-group", - "sleep 30 & echo $! > orphan.pid; sleep 30", - )) + async fn unknown_handles_are_not_found_on_every_method() { + let fixture = fixture(); + let read = fixture + .manager + .read(&ctx(), read_params("nope", ReturnOn::Exit, 0)) .await - .expect("start"); - - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - let terminated = manager - .terminate_process(TerminateProcessParams { - process_id: ProcessId::new("proc-group"), - signal: ProcessSignal::Kill, - }) - .await - .expect("terminate"); - assert!(terminated.running); - wait_for_process_gone(pid).await; - let output = manager - .read_process(read_params("proc-group", Some(1_000))) + .unwrap_err(); + assert_eq!(not_found_reason(&read), None); + let write = fixture + .manager + .write( + &ctx(), + WriteProcessParams { + process_id: id("nope"), + data: ByteChunk::default(), + close_input: true, + }, + ) .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.termination, Some(ProcessTermination::Killed)); - } - - #[cfg(unix)] - #[tokio::test(flavor = "current_thread")] - async fn timeout_terminates_descendants_in_group() { - let (_temp, root, manager) = manager(); - let mut params = params("proc-timeout", "sleep 30 & echo $! > orphan.pid; sleep 30"); - params.timeout_ms = Some(200); - manager.start_process(params).await.expect("start"); - - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - let output = manager - .read_process(read_params("proc-timeout", None)) + .unwrap_err(); + assert_eq!(write.kind(), ProtocolErrorKind::NotFound); + let terminate = fixture + .manager + .terminate( + &ctx(), + TerminateProcessParams { + process_id: id("nope"), + signal: ProcessSignal::Kill, + }, + ) .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.termination, Some(ProcessTermination::TimedOut)); - wait_for_process_gone(pid).await; + .unwrap_err(); + assert_eq!(terminate.kind(), ProtocolErrorKind::NotFound); } #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn interrupt_reaches_the_group_and_the_next_read_observes_the_trap() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params( - "proc-int", - "trap 'echo caught; exit 3' INT; sleep 100 & wait $!", - )) + async fn signal_deaths_are_named_on_pipes_and_ptys() { + let fixture = fixture(); + fixture + .manager + .start(&ctx(), shell("segv", "kill -SEGV $$")) .await .expect("start"); - tokio::time::sleep(Duration::from_millis(200)).await; - let signalled = manager - .terminate_process(TerminateProcessParams { - process_id: ProcessId::new("proc-int"), - signal: ProcessSignal::Interrupt, - }) - .await - .expect("interrupt"); - assert!(signalled.running); - let output = manager - .read_process(read_params("proc-int", Some(5_000))) - .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.exit_code, Some(3)); - assert_eq!(output.termination, None); - assert_eq!(all_bytes(&output), b"caught\n"); + let mut pty = shell("pty-term", "kill -TERM $$"); + pty.pty = Some(PtySize { rows: 24, cols: 80 }); + fixture.manager.start(&ctx(), pty).await.expect("start PTY"); + let (_, segv) = read_to_end(&fixture.manager, "segv").await; + assert!(matches!( + segv.exit, + Some(ExitStatus::Signal { ref signal, .. }) if signal == &SignalName::new("SIGSEGV").unwrap() + )); + let (_, term) = read_to_end(&fixture.manager, "pty-term").await; + assert_eq!( + term.exit, + Some(ExitStatus::Signal { + signal: SignalName::new("SIGTERM").unwrap(), + core_dumped: false, + }), + "a PTY signal death is not exit code 1" + ); } #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn process_exit_keeps_leftover_descendants_and_reports_them() { - let (_temp, root, manager) = manager(); - manager - .start_process(params( - "proc-leftover", - "nohup sleep 30 >/dev/null 2>&1 & echo $! > orphan.pid; exit 0", - )) + async fn terminate_escalates_to_kill_after_the_grace() { + let fixture = fixture(); + fixture + .manager + .start(&ctx(), shell("stubborn", "trap '' TERM; sleep 30")) .await .expect("start"); - - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - // Let the background child finish exec'ing so the sample names it. tokio::time::sleep(Duration::from_millis(100)).await; - let started = std::time::Instant::now(); - let output = manager - .read_process(read_params("proc-leftover", None)) + let terminated = fixture + .manager + .terminate( + &ctx(), + TerminateProcessParams { + process_id: id("stubborn"), + signal: ProcessSignal::Terminate, + }, + ) .await - .expect("read"); - assert!(started.elapsed() < Duration::from_secs(5)); - assert!(output.exited); - assert_eq!(output.exit_code, Some(0)); - assert!(process_alive(pid), "the leftover survives the call"); + .expect("terminate"); + assert!(terminated.running); + let (_, response) = read_to_end(&fixture.manager, "stubborn").await; + assert_eq!(response.stopped_by, Some(StoppedBy::Terminate)); assert_eq!( - output - .leftover_processes - .iter() - .map(|member| member.pid) - .collect::>(), - vec![pid as u32], - "{:?}", - output.leftover_processes - ); - assert!( - output.leftover_processes[0].command.contains("sleep"), - "{:?}", - output.leftover_processes - ); - assert_eq!(manager.leftover_group_count(), 1); - - tokio::time::sleep(TERMINAL_PROCESS_RETENTION * 3).await; - assert!( - process_alive(pid), - "the leftover survives the entry's pruning" + response.exit, + Some(ExitStatus::Signal { + signal: SignalName::new("SIGKILL").unwrap(), + core_dumped: false, + }) ); - let stale = manager - .read_process(read_params("proc-leftover", Some(0))) - .await; - assert!(stale.is_err(), "the entry was pruned"); + } - let _ = rustix::process::kill_process( - rustix::process::Pid::from_raw(pid).expect("valid child pid"), - rustix::process::Signal::KILL, - ); - wait_for_process_gone(pid).await; - assert_eq!(manager.leftover_group_count(), 0); + #[cfg(unix)] + #[tokio::test(flavor = "current_thread")] + async fn a_timeout_stops_the_group() { + let fixture = fixture(); + let mut params = shell("slow", "sleep 30 & echo $! > orphan.pid; sleep 30"); + params.command.timeout_ms = Some(200); + fixture.manager.start(&ctx(), params).await.expect("start"); + let orphan = wait_for_pid_file(&fixture.root.join("orphan.pid")).await; + let (_, response) = read_to_end(&fixture.manager, "slow").await; + assert_eq!(response.stopped_by, Some(StoppedBy::Timeout)); + for _ in 0..100 { + if !process_alive(orphan) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!(!process_alive(orphan), "the group was stopped"); } #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn leftover_writing_to_the_pipe_after_exit_is_neither_blocked_nor_killed() { - let (_temp, root, manager) = manager(); - manager - .start_process(params( - "proc-chatty", - "(while true; do echo x; sleep 0.01; done) & echo $! > orphan.pid; exit 0", - )) + async fn natural_exit_keeps_and_reports_leftovers() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + shell( + "service", + "nohup sleep 30 >/dev/null 2>&1 & echo $! > orphan.pid; exit 0", + ), + ) .await .expect("start"); - let entry = manager - .entry(&ProcessId::new("proc-chatty")) - .await - .expect("entry exists while running"); - let output = manager - .read_process(read_params("proc-chatty", None)) - .await - .expect("read"); - assert!(output.exited); - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - - // Past the drain grace the readers discard; nothing accumulates and - // the writer keeps running. The entry handle outlives the pruning - // that the read started. - tokio::time::sleep(process_group::OUTPUT_DRAIN_GRACE * 2).await; - let (discarding, retained) = { - let state = entry.state.lock().await; - (state.discarding, state.output.unread_bytes()) - }; - assert!(discarding); + let orphan = wait_for_pid_file(&fixture.root.join("orphan.pid")).await; tokio::time::sleep(Duration::from_millis(100)).await; - assert_eq!( - entry.state.lock().await.output.unread_bytes(), - retained, - "stored output does not grow after the drain grace" - ); - assert!(process_alive(pid), "the writer was not killed by EPIPE"); - - let _ = rustix::process::kill_process( - rustix::process::Pid::from_raw(pid).expect("valid child pid"), - rustix::process::Signal::KILL, - ); - wait_for_process_gone(pid).await; + let (_, response) = read_to_end(&fixture.manager, "service").await; + let leftovers = response.leftovers.expect("leftovers reported"); + assert_eq!(leftovers.disposition, LeftoverDisposition::Kept); + assert_eq!(leftovers.processes[0].pid, orphan as u32); + assert!(process_alive(orphan), "never swept on natural exit"); + assert_eq!(fixture.manager.leftover_group_count(), 1); + assert_eq!(fixture.manager.running_count().await, 0); + kill(orphan); } - #[cfg(unix)] + #[cfg(any(target_os = "linux", target_os = "macos"))] #[tokio::test(flavor = "current_thread")] - async fn kill_after_exit_stops_the_leftover_group() { - let (_temp, root, manager) = manager(); - manager - .start_process(params( - "proc-service", - "nohup sleep 30 >/dev/null 2>&1 & echo $! > orphan.pid; exit 0", - )) + async fn a_restarted_daemon_adopts_surviving_groups() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + shell( + "survivor", + "nohup sleep 30 >/dev/null 2>&1 & echo $! > orphan.pid; exit 0", + ), + ) .await .expect("start"); - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - assert!(process_alive(pid)); - for _ in 0..200 { - if manager.running_count().await == 0 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert_eq!(manager.leftover_group_count(), 1); - - let terminated = manager - .terminate_process(TerminateProcessParams { - process_id: ProcessId::new("proc-service"), - signal: ProcessSignal::Kill, - }) + let orphan = wait_for_pid_file(&fixture.root.join("orphan.pid")).await; + fixture + .manager + .start(&ctx(), shell("gone", "true")) .await - .expect("kill"); - assert!(!terminated.running, "the root had already exited"); - wait_for_process_gone(pid).await; - assert_eq!(manager.leftover_group_count(), 0); - let output = manager - .read_process(read_params("proc-service", Some(0))) + .expect("start"); + read_to_end(&fixture.manager, "survivor").await; + read_to_end(&fixture.manager, "gone").await; + + let restarted = manager_at(&fixture.root, &fixture.state_dir, limits(), "envd_second"); + let error = restarted + .read(&ctx(), read_params("survivor", ReturnOn::Exit, 0)) .await - .expect("first read after the kill"); - assert!(output.exited); - assert!( - output.leftover_processes.is_empty(), - "a dead group is no longer reported" + .unwrap_err(); + assert_eq!( + not_found_reason(&error), + Some(NotFoundReason::DaemonRestarted) + ); + assert_eq!( + restarted.leftover_group_count(), + 1, + "the surviving group is adopted" ); + assert!(process_alive(orphan), "adopted, never killed"); + kill(orphan); + for _ in 0..100 { + if restarted.leftover_group_count() == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!(restarted.leftover_group_count(), 0); + let records = std::fs::read_dir(fixture.state_dir.join("processes")) + .unwrap() + .count(); + assert_eq!(records, 0, "records of dead groups are deleted"); } #[tokio::test(flavor = "current_thread")] - async fn process_injects_secret_env_and_redacts_output() { - let (_temp, _root, manager) = manager(); - let mut params = params("proc-secret", "printf \"$SECRET_TOKEN\""); - params.secret_env = BTreeMap::from([( - "SECRET_TOKEN".to_owned(), - SecretString::new("super-secret-token"), - )]); - manager.start_process(params).await.expect("start"); - - let output = manager - .read_process(read_params("proc-secret", None)) - .await - .expect("read"); + async fn secrets_are_redacted_and_ptys_need_the_capability() { + let fixture = fixture(); + let mut params = shell("secret", "printf \"$TOKEN\""); + params.command.secret_env = + BTreeMap::from([("TOKEN".to_owned(), SecretString::new("hunter2"))]); + fixture.manager.start(&ctx(), params).await.expect("start"); + let (bytes, _) = read_to_end(&fixture.manager, "secret").await; + assert_eq!(bytes, b""); + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let confinement = Confinement::new(root.clone(), root.join("state")).unwrap(); + let without_pty = ProcessManager::new(ProcessConfig { + cwd: root.clone(), + confinement: Arc::new(confinement), + state_dir: root.join("state"), + scrubbed_env: Vec::new(), + instance_id: DaemonInstanceId::new("envd_nopty").unwrap(), + limits: limits(), + max_wait_ms: 1_000, + pty_supported: false, + }) + .unwrap(); + let mut pty = shell("pty", "true"); + pty.pty = Some(PtySize { rows: 24, cols: 80 }); + let error = without_pty.start(&ctx(), pty).await.unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::CapabilityUnavailable); + let mut wait = read_params("pty", ReturnOn::Exit, 1_001); + wait.max_bytes = 0; + let error = without_pty.read(&ctx(), wait).await.unwrap_err(); assert_eq!( - stream_bytes(&output, ProcessOutputStream::Stdout), - b"" + error.kind(), + ProtocolErrorKind::InvalidParams, + "waitMs above the ceiling" ); } } diff --git a/crates/environment-daemon/src/process_group.rs b/crates/environment-daemon/src/process_group.rs index 787d5e4c2..610366673 100644 --- a/crates/environment-daemon/src/process_group.rs +++ b/crates/environment-daemon/src/process_group.rs @@ -1,11 +1,11 @@ //! Unix process-group lifecycle shared by job and process execution. //! -//! Job and process commands start in their own process group so that -//! descendants a script leaves behind can be signalled as a unit. Jobs sweep -//! the group when the root exits; processes sweep it only on timeout, kill, -//! or cancellation, and otherwise let leftovers live with the environment. A -//! descendant can still escape the group (`setsid`), so callers must never -//! make completion depend on the group being fully swept. +//! Every command starts as the leader of its own process group, so the +//! descendants a script leaves behind can be signalled as a unit. Jobs +//! sweep the group when the root exits; processes signal it only on +//! timeout or an explicit terminate or kill, and otherwise let leftovers +//! live with the environment. A descendant can still escape the group +//! (`setsid`), so completion never depends on the group being empty. use std::time::Duration; @@ -14,27 +14,35 @@ use tokio::process::Command; #[cfg(unix)] use rustix::process::{Pid, Signal, kill_process_group, test_kill_process_group}; -/// How long a terminated group gets between SIGTERM and SIGKILL. -#[cfg(not(test))] -pub(crate) const GROUP_TERM_GRACE: Duration = Duration::from_secs(1); -#[cfg(test)] -pub(crate) const GROUP_TERM_GRACE: Duration = Duration::from_millis(50); - -/// How long remaining pipe output is drained after the root process exits -/// before the job or process completes without it. +/// How long remaining output is drained after the root process exits +/// before the execution settles without it. #[cfg(not(test))] pub(crate) const OUTPUT_DRAIN_GRACE: Duration = Duration::from_secs(2); #[cfg(test)] pub(crate) const OUTPUT_DRAIN_GRACE: Duration = Duration::from_millis(200); +/// How often a terminating group is checked for members during its grace. +const GRACE_POLL_INTERVAL: Duration = Duration::from_millis(20); + +/// A signal the daemon sends to a whole process group. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum GroupSignal { + Interrupt, + Terminate, + Kill, +} + /// Places the spawned command in its own process group (the child becomes /// the group leader). pub(crate) fn spawn_in_own_group(command: &mut Command) { #[cfg(unix)] command.process_group(0); + #[cfg(not(unix))] + let _ = command; } -/// True when at least one process is left in the group. +/// True when at least one process is left in the group. Says nothing about +/// whose group it is; see [`Group::alive`]. pub(crate) fn group_alive(pgid: u32) -> bool { #[cfg(unix)] return group_pid(pgid).is_some_and(|pid| test_kill_process_group(pid).is_ok()); @@ -45,60 +53,302 @@ pub(crate) fn group_alive(pgid: u32) -> bool { } } -/// Immediately kills every process left in the group. -pub(crate) fn kill_group(pgid: u32) -> bool { +/// Sends `signal` to every process in the group. Returns true when the +/// group had at least one member to signal. +pub(crate) fn signal_group(pgid: u32, signal: GroupSignal) -> bool { #[cfg(unix)] - return signal_group(pgid, Signal::KILL); + { + let signal = match signal { + GroupSignal::Interrupt => Signal::INT, + GroupSignal::Terminate => Signal::TERM, + GroupSignal::Kill => Signal::KILL, + }; + group_pid(pgid).is_some_and(|pid| kill_process_group(pid, signal).is_ok()) + } #[cfg(not(unix))] { - let _ = pgid; + let _ = (pgid, signal); false } } -/// Terminates every process left in the group: SIGTERM, a bounded grace -/// period, then SIGKILL. Returns true when any process had to be signalled. -pub(crate) async fn sweep_group(pgid: u32) -> bool { - #[cfg(unix)] - { - if !signal_group(pgid, Signal::TERM) { +/// One spawned process group, named by its id and its leader's start time. +/// +/// A group id stays reserved while the group has members, so a live group +/// whose leader is gone is still this group. Once the group empties, the id +/// may name a later, unrelated group whose leader got the same pid: a live +/// process with the group's pid and another start time means the group is +/// gone. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Group { + pub pgid: u32, + pub leader_start_time: Option, +} + +impl Group { + /// The group led by `pgid`, a live process (a just spawned root). + pub fn capture(pgid: u32) -> Self { + Self { + pgid, + leader_start_time: process_start_time(pgid), + } + } + + /// The group still has members, and they are this group's. + pub fn alive(&self) -> bool { + if !group_alive(self.pgid) { return false; } - tokio::time::sleep(GROUP_TERM_GRACE).await; - signal_group(pgid, Signal::KILL); + match process_start_time(self.pgid) { + None => true, + Some(started) => self.leader_start_time == Some(started), + } + } + + /// Sends `signal` to the group while it is alive. Returns true when it + /// had a member to signal. + pub fn signal(&self, signal: GroupSignal) -> bool { + self.alive() && signal_group(self.pgid, signal) + } + + /// Terminates every process left in the group: `SIGTERM`, up to + /// `grace` for the members to exit, then `SIGKILL` to whatever remains. + /// Returns true when the group had members to signal. + pub async fn terminate(self, grace: Duration) -> bool { + if !self.signal(GroupSignal::Terminate) { + return false; + } + self.kill_after_grace(grace).await; true } - #[cfg(not(unix))] - { - let _ = pgid; - false + + /// Waits up to `grace` for the group to empty, then kills what is left. + pub async fn kill_after_grace(self, grace: Duration) { + let deadline = tokio::time::Instant::now() + grace; + while self.alive() { + if tokio::time::Instant::now() >= deadline { + self.signal(GroupSignal::Kill); + return; + } + tokio::time::sleep(GRACE_POLL_INTERVAL.min(grace)).await; + } } } #[cfg(unix)] fn group_pid(pgid: u32) -> Option { - // Group IDs 0 and 1 have special kill semantics rather than naming a child group. + // Group ids 0 and 1 have special kill semantics rather than naming a + // child group. let pgid = i32::try_from(pgid).ok().filter(|pid| *pid > 1)?; Pid::from_raw(pgid) } -#[cfg(unix)] -fn signal_group(pgid: u32, signal: Signal) -> bool { - group_pid(pgid).is_some_and(|pid| kill_process_group(pid, signal).is_ok()) +/// Sends `SIGINT` to one process group by id, such as a terminal's +/// foreground group. Returns true when it had a member to signal. +pub(crate) fn interrupt_foreground(pgid: u32) -> bool { + signal_group(pgid, GroupSignal::Interrupt) } -/// Sends `SIGINT` to every process left in the group. Returns true when the -/// group had at least one member to signal. -pub(crate) fn interrupt_group(pgid: u32) -> bool { - #[cfg(unix)] - return signal_group(pgid, Signal::INT); - #[cfg(not(unix))] +/// This boot of the machine, so spawn records of an earlier boot are never +/// mistaken for live groups: Linux's boot id, macOS's boot time. +pub(crate) fn boot_id() -> Option { + #[cfg(target_os = "linux")] { - let _ = pgid; - false + std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .ok() + .map(|id| id.trim().to_owned()) + .filter(|id| !id.is_empty()) + } + #[cfg(target_os = "macos")] + { + let mut boot = libc::timeval { + tv_sec: 0, + tv_usec: 0, + }; + let mut size = std::mem::size_of::(); + // SAFETY: the name is NUL-terminated and the buffer is a timeval of + // the size passed. + let status = unsafe { + libc::sysctlbyname( + c"kern.boottime".as_ptr(), + (&mut boot as *mut libc::timeval).cast(), + &mut size, + std::ptr::null_mut(), + 0, + ) + }; + (status == 0 && boot.tv_sec > 0) + .then(|| format!("boottime:{}.{:06}", boot.tv_sec, boot.tv_usec)) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + None + } +} + +/// When process `pid` started, in a platform unit that only needs to be +/// stable for one boot: clock ticks since boot on Linux, microseconds since +/// the epoch on macOS. Tells a group leader apart from a later process that +/// reused its pid. +pub(crate) fn process_start_time(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // "pid (comm) state ppid ..."; comm may contain spaces and + // parentheses, so split after the last closing parenthesis. The + // start time is field 22, the 20th after comm. + let close = stat.rfind(')')?; + stat[close + 1..] + .split_whitespace() + .nth(19) + .and_then(|value| value.parse().ok()) + } + #[cfg(target_os = "macos")] + { + let info = bsd_info(pid)?; + Some(info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = pid; + None } } +/// Live, non-zombie members of a process group with their command lines, +/// lowest pid first. Best effort: `/proc` on Linux, libproc on macOS, +/// nothing elsewhere. +#[cfg(target_os = "linux")] +pub(crate) fn group_members(pgid: u32) -> Vec<(u32, Option)> { + let Ok(entries) = std::fs::read_dir("/proc") else { + return Vec::new(); + }; + let mut members = Vec::new(); + for entry in entries.flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse::().ok()) + else { + continue; + }; + let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) else { + continue; + }; + let (Some(open), Some(close)) = (stat.find('('), stat.rfind(')')) else { + continue; + }; + let mut fields = stat[close + 1..].split_whitespace(); + let state = fields.next(); + let _ppid = fields.next(); + let pgrp = fields.next().and_then(|value| value.parse::().ok()); + if pgrp != Some(pgid) || state == Some("Z") { + continue; + } + let command = std::fs::read(entry.path().join("cmdline")) + .ok() + .map(|bytes| { + bytes + .split(|byte| *byte == 0) + .filter(|part| !part.is_empty()) + .map(|part| String::from_utf8_lossy(part).into_owned()) + .collect::>() + .join(" ") + }) + .filter(|command| !command.is_empty()) + .or_else(|| { + let comm = stat.get(open + 1..close).unwrap_or_default(); + (!comm.is_empty()).then(|| comm.to_owned()) + }); + members.push((pid, command)); + } + members.sort_by_key(|(pid, _)| *pid); + members +} + +#[cfg(target_os = "macos")] +pub(crate) fn group_members(pgid: u32) -> Vec<(u32, Option)> { + use std::{ffi::c_void, mem}; + + let Ok(group) = libc::pid_t::try_from(pgid) else { + return Vec::new(); + }; + let pid_size = mem::size_of::(); + // libproc's group listing returns pid counts, not bytes: with a null + // buffer the count of every process on the system, which is an upper + // bound for the group, and with a buffer the count it filled. + // SAFETY: a null buffer of size 0 only queries the count. + let upper_bound = unsafe { libc::proc_listpgrppids(group, std::ptr::null_mut(), 0) }; + if upper_bound <= 0 { + return Vec::new(); + } + let mut pids = vec![0 as libc::pid_t; upper_bound as usize + 16]; + // SAFETY: the buffer holds `pids.len()` pids and its byte size is passed. + let count = unsafe { + libc::proc_listpgrppids( + group, + pids.as_mut_ptr() as *mut c_void, + (pids.len() * pid_size) as libc::c_int, + ) + }; + if count <= 0 { + return Vec::new(); + } + pids.truncate(count as usize); + let mut members = Vec::new(); + for pid in pids { + let Ok(pid) = u32::try_from(pid) else { + continue; + }; + if pid == 0 { + continue; + } + if bsd_info(pid).is_some_and(|info| info.pbi_status == libc::SZOMB) { + continue; + } + let mut path = vec![0u8; 4096]; + // SAFETY: the buffer is 4096 bytes and its length is passed. + let len = unsafe { + libc::proc_pidpath( + pid as libc::c_int, + path.as_mut_ptr() as *mut c_void, + path.len() as u32, + ) + }; + let command = + (len > 0).then(|| String::from_utf8_lossy(&path[..len as usize]).into_owned()); + members.push((pid, command)); + } + members.sort_by_key(|(pid, _)| *pid); + members +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub(crate) fn group_members(_pgid: u32) -> Vec<(u32, Option)> { + Vec::new() +} + +#[cfg(target_os = "macos")] +fn bsd_info(pid: u32) -> Option { + use std::{ffi::c_void, mem}; + + let pid = libc::c_int::try_from(pid).ok()?; + // SAFETY: proc_bsdinfo is plain data; zeroed is a valid value. + let mut info: libc::proc_bsdinfo = unsafe { mem::zeroed() }; + let size = mem::size_of::() as libc::c_int; + // SAFETY: the buffer is a proc_bsdinfo of the size passed. + let read = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 0, + &mut info as *mut libc::proc_bsdinfo as *mut c_void, + size, + ) + }; + (read == size).then_some(info) +} + #[cfg(all(test, unix))] mod tests { use super::*; @@ -107,7 +357,38 @@ mod tests { fn process_groups_reject_special_and_out_of_range_ids() { for pgid in [0, 1, i32::MAX as u32 + 1, u32::MAX] { assert!(group_pid(pgid).is_none()); + assert!(!signal_group(pgid, GroupSignal::Kill)); } assert_eq!(group_pid(42), Pid::from_raw(42)); } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn this_process_has_a_start_time_and_the_machine_a_boot_id() { + assert!(process_start_time(std::process::id()).is_some()); + assert!(boot_id().is_some()); + } + + #[tokio::test(flavor = "current_thread")] + async fn terminate_escalates_to_kill_when_the_group_ignores_sigterm() { + let mut command = Command::new("/bin/sh"); + command + .args(["-c", "trap '' TERM; sleep 30"]) + .stdout(std::process::Stdio::null()); + spawn_in_own_group(&mut command); + let mut child = command.spawn().expect("spawn"); + let pgid = child.id().expect("pid"); + // Give the shell time to install its trap. + tokio::time::sleep(Duration::from_millis(100)).await; + let started = std::time::Instant::now(); + assert!( + Group::capture(pgid) + .terminate(Duration::from_millis(200)) + .await + ); + let status = child.wait().await.expect("wait"); + assert!(started.elapsed() >= Duration::from_millis(150)); + use std::os::unix::process::ExitStatusExt as _; + assert_eq!(status.signal(), Some(Signal::KILL.as_raw())); + } } diff --git a/crates/environment-daemon/src/redaction.rs b/crates/environment-daemon/src/redaction.rs index 47d6bde65..b921bd459 100644 --- a/crates/environment-daemon/src/redaction.rs +++ b/crates/environment-daemon/src/redaction.rs @@ -1,6 +1,6 @@ //! Shared byte redaction for captured process and job output. -use environment_protocol::shared::SecretString; +use environment_protocol::bytes::SecretString; use std::collections::BTreeMap; pub(crate) fn redactions_for_secret_env( diff --git a/crates/environment-daemon/src/registration.rs b/crates/environment-daemon/src/registration.rs index 23b230f8c..5bbacbe69 100644 --- a/crates/environment-daemon/src/registration.rs +++ b/crates/environment-daemon/src/registration.rs @@ -1,316 +1,682 @@ //! Outbound registration: dial the environment gateway, prove the daemon -//! identity, and serve reverse-dialed data connections for as long as the -//! control connection lives. +//! identity, and serve reverse-dialed streams for as long as the control +//! connection lives. //! -//! The daemon never chooses an identity mode and never retries a terminal -//! rejection: a closed environment, a refused key, or a bad signature ends -//! the process with a non-zero exit so an orchestrator sees it. +//! The control connection runs the shared JSON-RPC envelope with the daemon +//! as the client (the frame rules and liveness rule are in +//! `environment_protocol::registration`). The daemon reconnects with +//! backoff after disconnects, heartbeat timeouts and retryable refusals, +//! and ends the process on everything else, each with its own exit status +//! ([`DaemonExit`]): a deliberate shutdown says goodbye first, a terminal +//! refusal or an unsupported gateway protocol is `Refused`, and a +//! `registration/closed` notification ends it as `EnvironmentClosed` or +//! `Superseded`. use std::{ - path::Path, - process::Command, + path::{Path, PathBuf}, sync::Arc, time::{Duration, Instant}, }; use anyhow::{Context as _, Result, anyhow, bail}; use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + bytes::SecretString, + error::{ErrorData, PreconditionReason, ProtocolError}, + jsonrpc::{self, Id, Message as RpcMessage, Notification, Request, Response, ResponseError}, + limits::HEARTBEAT_MISSED_INTERVALS, + manifest::{ + RegistrationClosed, RegistrationGoodbye, RegistrationOpenStream, RegistrationRegister, + }, registration::{ - DaemonControlMessage, GatewayControlMessage, RegisterParams, RegistrationReceipt, - RegistrationRejectionCode, decode_hex, + ChallengeParams, ClosedParams, ClosedReason, GoodbyeParams, GoodbyeReason, + OpenStreamParams, OpenStreamResponse, RegisterParams, RegisterResponse, + RegistrationReceipt, StreamSpec, challenge_protocol_version, }, - shared::{CURRENT_PROTOCOL_VERSION, SecretString}, }; -use futures_util::{SinkExt as _, StreamExt as _}; +use futures_util::{SinkExt as _, StreamExt as _, stream::SplitSink, stream::SplitStream}; +use tokio::sync::mpsc; use tokio_tungstenite::{ Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config, tungstenite::{ Message, client::IntoClientRequest as _, - http::{HeaderValue, header::AUTHORIZATION}, + http::{HeaderValue, header::AUTHORIZATION, header::USER_AGENT}, }, }; use crate::{ DaemonRuntime, - config::{REEXEC_REGISTRATION_KEY_ENV, RegistrationConfig}, + config::{ + AUTO_UPGRADE_ATTEMPTED_ENV, GatewayUrls, REEXEC_REGISTRATION_KEY_ENV, RegistrationConfig, + RestartInvocation, + }, identity::DaemonIdentity, - server::{run_data_connection, tracing_line}, + server::{DaemonExit, run_data_connection, tracing_line, websocket_config}, }; -const AUTO_UPGRADE_ATTEMPTED_ENV: &str = "LIGHTSPEED_ENVD_AUTO_UPGRADE_ATTEMPTED"; - -/// How long the daemon waits for the gateway's challenge and verdict. +/// Bound on TCP connect, TLS and the WebSocket upgrade of the control +/// connection. +const DIAL_TIMEOUT: Duration = Duration::from_secs(20); +/// How long the daemon waits for the gateway's first frame, and then for +/// the register response. const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(20); +/// Bound on dialing one reverse stream, upgrade included. +const STREAM_DIAL_TIMEOUT: Duration = Duration::from_secs(10); +/// How long a shutting-down daemon spends on goodbye and the close frame. +const GOODBYE_TIMEOUT: Duration = Duration::from_secs(2); +/// Shortest heartbeat interval the liveness deadline is derived from. +const MIN_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(1); const INITIAL_BACKOFF: Duration = Duration::from_secs(1); const MAX_BACKOFF: Duration = Duration::from_secs(60); /// A control session that lasted this long resets the reconnect backoff. const STABLE_SESSION: Duration = Duration::from_secs(30); +/// The id of the daemon's only request on a control connection. +const REGISTER_ID: i64 = 1; + +type ControlSocket = WebSocketStream>; +type ControlWriter = SplitSink; +type ControlReader = SplitStream; /// Why one control session ended. +#[derive(Debug)] enum SessionEnd { - /// The gateway or network went away; reconnect with backoff. + /// The gateway or network went away, the liveness deadline passed, or + /// the gateway broke the frame rules; reconnect with backoff. Disconnected(String), - /// The gateway refused the daemon for a transient reason. - Retryable(RegistrationRejectionCode, String), + /// The gateway refused admission for a transient reason; reconnect no + /// sooner than its hint. + Retry { + reason: String, + retry_after: Option, + }, + /// The gateway speaks this other protocol version. + ProtocolMismatch(u32), + /// The gateway's first frame is unreadable: an incompatible gateway + /// whose protocol only its discovery document can tell. + IncompatibleGateway(String), + /// End the process. + Exit(DaemonExit), } -/// Registration failure the daemon must not retry. -#[derive(Debug, thiserror::Error)] -#[error("registration rejected ({code:?}): {message}")] -pub struct TerminalRejection { - pub code: RegistrationRejectionCode, - pub message: String, +/// Register with the configured gateway and serve it until the process +/// must end. +pub async fn run_outbound(runtime: DaemonRuntime, registration: RegistrationConfig) -> DaemonExit { + let outbound = match Outbound::new(runtime, registration) { + Ok(outbound) => outbound, + Err(error) => return DaemonExit::Configuration(error), + }; + outbound.run().await } -pub async fn run_outbound(runtime: DaemonRuntime, registration: RegistrationConfig) -> Result<()> { - let identity = DaemonIdentity::load_or_create(&runtime.config().state_dir) - .context("load daemon identity")?; - tracing_line(&format!( - "registering with {} as daemon public key {}", - registration.gateway_url, - identity.public_key_hex() - )); - let tls = TlsSettings::from_config(registration.ca_file.as_deref())?; - let mut backoff = INITIAL_BACKOFF; - loop { - let started = Instant::now(); - match control_session(&runtime, ®istration, &identity, &tls).await { - Ok(SessionEnd::Disconnected(reason)) => { - tracing_line(&format!("control connection ended: {reason}")); +struct Outbound { + runtime: DaemonRuntime, + registration: RegistrationConfig, + urls: GatewayUrls, + identity: DaemonIdentity, + tls: TlsSettings, + /// Held only until the first admission. + registration_key: Option, +} + +impl Outbound { + fn new(runtime: DaemonRuntime, registration: RegistrationConfig) -> Result { + let urls = registration.validate()?; + if let Some(path) = ®istration.receipt_path { + check_receipt_path(path)?; + } + let identity = DaemonIdentity::load_or_create(&runtime.config().state_dir) + .context("load daemon identity")?; + let tls = TlsSettings::from_config(registration.ca_file.as_deref())?; + let registration_key = registration.registration_key.clone(); + Ok(Self { + runtime, + registration, + urls, + identity, + tls, + registration_key, + }) + } + + async fn run(mut self) -> DaemonExit { + tracing_line(&format!( + "registering with {} as daemon public key {}", + self.urls.connect, + self.identity.public_key() + )); + let shutdown = self.runtime.shutdown().clone(); + let mut backoff = INITIAL_BACKOFF; + loop { + let started = Instant::now(); + let mut retry_after = None; + match self.session().await { + SessionEnd::Exit(exit) => return exit, + SessionEnd::Disconnected(reason) => { + tracing_line(&format!("control connection ended: {reason}")); + } + SessionEnd::Retry { + reason, + retry_after: hint, + } => { + tracing_line(&format!("registration deferred: {reason}")); + retry_after = hint; + } + SessionEnd::ProtocolMismatch(gateway_protocol) => { + match self.upgrade(Some(gateway_protocol), None).await { + SessionEnd::Exit(exit) => return exit, + SessionEnd::Retry { reason, .. } | SessionEnd::Disconnected(reason) => { + tracing_line(&reason); + } + SessionEnd::ProtocolMismatch(_) | SessionEnd::IncompatibleGateway(_) => {} + } + } + SessionEnd::IncompatibleGateway(reason) => { + match self.upgrade(None, Some(reason)).await { + SessionEnd::Exit(exit) => return exit, + SessionEnd::Retry { reason, .. } | SessionEnd::Disconnected(reason) => { + tracing_line(&reason); + } + SessionEnd::ProtocolMismatch(_) | SessionEnd::IncompatibleGateway(_) => {} + } + } } - Ok(SessionEnd::Retryable(code, message)) => { - tracing_line(&format!("registration deferred ({code:?}): {message}")); + if started.elapsed() >= STABLE_SESSION { + backoff = INITIAL_BACKOFF; } - Err(error) => { - if let Some(rejection) = error.downcast_ref::() { - tracing_line(&format!("{rejection}; not retrying")); - return Err(error); - } - tracing_line(&format!("control connection failed: {error:#}")); + let delay = jittered(backoff).max(retry_after.unwrap_or_default()); + tokio::select! { + _ = shutdown.cancelled() => return DaemonExit::Shutdown, + _ = tokio::time::sleep(delay) => {} } + backoff = (backoff * 2).min(MAX_BACKOFF); } - if started.elapsed() >= STABLE_SESSION { - backoff = INITIAL_BACKOFF; - } - tokio::time::sleep(jittered(backoff)).await; - backoff = (backoff * 2).min(MAX_BACKOFF); } -} -async fn control_session( - runtime: &DaemonRuntime, - registration: &RegistrationConfig, - identity: &DaemonIdentity, - tls: &TlsSettings, -) -> Result { - let request = registration - .gateway_url - .as_str() - .into_client_request() - .context("gateway URL")?; - let (socket, _) = connect_async_tls_with_config(request, None, false, Some(tls.connector())) - .await - .map_err(|error| describe_connect_error(®istration.gateway_url, error))?; - let (mut writer, mut reader) = socket.split(); - - let challenge = tokio::time::timeout(HANDSHAKE_TIMEOUT, next_control_message(&mut reader)) - .await - .context("waiting for gateway challenge")??; - let nonce = match challenge { - Some(GatewayControlMessage::Challenge { - protocol_version, - nonce, - }) => { - if protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(protocol_mismatch_error(registration, protocol_version).await); + /// One control connection, from dial to its end. + async fn session(&mut self) -> SessionEnd { + let shutdown = self.runtime.shutdown().clone(); + let dialed = tokio::select! { + _ = shutdown.cancelled() => return SessionEnd::Exit(DaemonExit::Shutdown), + dialed = tokio::time::timeout( + DIAL_TIMEOUT, + dial(&self.urls.connect, None, &self.tls), + ) => dialed, + }; + let socket = match dialed { + Ok(Ok(socket)) => socket, + Ok(Err(error)) => return SessionEnd::Disconnected(format!("{error:#}")), + Err(_) => { + return SessionEnd::Disconnected(format!( + "connecting to {} timed out after {} s", + self.urls.connect, + DIAL_TIMEOUT.as_secs() + )); } - decode_hex(&nonce).ok_or_else(|| anyhow!("gateway challenge nonce is not hex"))? + }; + let (mut writer, mut reader) = socket.split(); + let end = self.admitted_session(&mut writer, &mut reader).await; + if !matches!(end, SessionEnd::Exit(DaemonExit::Shutdown)) { + let _ = tokio::time::timeout(GOODBYE_TIMEOUT, writer.close()).await; } - Some(GatewayControlMessage::Rejected { code, message }) => { - return rejection(code, message); + end + } + + async fn admitted_session( + &mut self, + writer: &mut ControlWriter, + reader: &mut ControlReader, + ) -> SessionEnd { + let shutdown = self.runtime.shutdown().clone(); + + // 1. The frozen first frame: its version is read before anything + // else, so a gateway of any version can tell this daemon it is + // stale. + let first = tokio::select! { + _ = shutdown.cancelled() => return SessionEnd::Exit(DaemonExit::Shutdown), + first = tokio::time::timeout(HANDSHAKE_TIMEOUT, next_text(reader)) => first, + }; + let first = match first { + Ok(Ok(text)) => text, + Ok(Err(end)) => return end, + Err(_) => return SessionEnd::Disconnected("no challenge from the gateway".to_owned()), + }; + let gateway_protocol = match challenge_protocol_version(&first) { + Ok(version) => version, + Err(error) => { + return SessionEnd::IncompatibleGateway(format!( + "{} sent an unreadable first frame ({error})", + self.urls.connect + )); + } + }; + if gateway_protocol != ENVD_PROTOCOL_VERSION { + return SessionEnd::ProtocolMismatch(gateway_protocol); } - Some(other) => bail!("unexpected gateway message before challenge: {other:?}"), - None => { - return Ok(SessionEnd::Disconnected( - "closed before challenge".to_owned(), - )); + let challenge = match decode_challenge(&first) { + Ok(challenge) => challenge, + Err(error) => return SessionEnd::Disconnected(format!("{error:#}")), + }; + + // 2. Register, signing the nonce bound to the dialed origin. + let params = RegisterParams { + protocol_version: ENVD_PROTOCOL_VERSION, + registration_key: self.registration_key.clone(), + daemon_public_key: self.identity.public_key(), + signature: self + .identity + .sign_challenge(&challenge.nonce, &self.urls.origin), + display_name: self.registration.display_name.clone(), + metadata: self.registration.metadata.clone(), + implementation: self.runtime.server_info().clone(), + }; + let request = + match Request::new(Id::Number(REGISTER_ID), RegistrationRegister::NAME, ¶ms) { + Ok(request) => request, + Err(error) => { + return SessionEnd::Exit(DaemonExit::Failed(anyhow!( + "encode register: {error}" + ))); + } + }; + if let Err(error) = writer.send(Message::Text(request.to_text().into())).await { + return SessionEnd::Disconnected(format!("send register: {error}")); } - }; + let response = tokio::select! { + _ = shutdown.cancelled() => return SessionEnd::Exit(DaemonExit::Shutdown), + response = tokio::time::timeout(HANDSHAKE_TIMEOUT, register_response(reader)) => response, + }; + let response = match response { + Ok(Ok(response)) => response, + Ok(Err(end)) => return end, + Err(_) => { + return SessionEnd::Disconnected( + "no register response from the gateway".to_owned(), + ); + } + }; + let admitted = match response.decode::() { + Ok(admitted) => admitted, + Err(ResponseError::Failed(error)) => return refusal(error), + Err(ResponseError::Undecodable(message)) => { + return SessionEnd::Disconnected(format!( + "the register response breaks the contract: {message}" + )); + } + }; - let register = DaemonControlMessage::Register(RegisterParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - registration_key: registration.registration_key.clone(), - daemon_public_key: identity.public_key_hex(), - signature: identity.sign_challenge(&nonce), - display_name: registration.display_name.clone(), - metadata: registration.metadata.clone(), - implementation: runtime.implementation(), - }); - writer - .send(Message::Text(serde_json::to_string(®ister)?.into())) - .await - .context("send register")?; - - let verdict = tokio::time::timeout(HANDSHAKE_TIMEOUT, next_control_message(&mut reader)) - .await - .context("waiting for gateway verdict")??; - let (receipt, heartbeat_interval_ms) = match verdict { - Some(GatewayControlMessage::Accepted { - receipt, - heartbeat_interval_ms, - }) => (receipt, heartbeat_interval_ms), - Some(GatewayControlMessage::Rejected { code, message }) => { - return rejection(code, message); + // 3. Admitted: the key has done its job. + self.registration_key = None; + if let Err(error) = + announce_receipt(&admitted.receipt, self.registration.receipt_path.as_deref()) + { + return SessionEnd::Exit(DaemonExit::Configuration(error)); } - Some(other) => bail!("unexpected gateway message after register: {other:?}"), - None => return Ok(SessionEnd::Disconnected("closed before verdict".to_owned())), - }; - announce_receipt(&receipt, registration.receipt_path.as_deref())?; - let _ = heartbeat_interval_ms; + let interval = + Duration::from_millis(admitted.heartbeat_interval_ms).max(MIN_HEARTBEAT_INTERVAL); + let liveness = interval * HEARTBEAT_MISSED_INTERVALS; + self.serve_admitted(writer, reader, liveness).await + } - loop { - let message = match reader.next().await { - Some(Ok(message)) => message, - Some(Err(error)) => return Ok(SessionEnd::Disconnected(error.to_string())), - None => return Ok(SessionEnd::Disconnected("gateway closed".to_owned())), - }; - match message { - Message::Text(text) => match serde_json::from_str::(&text) { - Ok(GatewayControlMessage::OpenData { token, data_url }) => { - let runtime = runtime.clone(); - let tls = tls.clone(); - tokio::spawn(async move { - if let Err(error) = - serve_data_connection(runtime, &data_url, token, &tls).await - { - tracing_line(&format!("data connection failed: {error:#}")); - } - }); + /// The admitted control connection: answer `registration/openStream`, + /// obey `registration/closed`, and treat `liveness` without any frame + /// as a dead connection. + async fn serve_admitted( + &self, + writer: &mut ControlWriter, + reader: &mut ControlReader, + liveness: Duration, + ) -> SessionEnd { + let shutdown = self.runtime.shutdown().clone(); + let (answers, mut queued) = mpsc::channel::(64); + let mut deadline = tokio::time::Instant::now() + liveness; + loop { + tokio::select! { + _ = shutdown.cancelled() => { + say_goodbye(writer).await; + return SessionEnd::Exit(DaemonExit::Shutdown); } - Ok(GatewayControlMessage::Rejected { code, message }) => { - return rejection(code, message); + Some(text) = queued.recv() => { + if let Err(error) = writer.send(Message::Text(text.into())).await { + return SessionEnd::Disconnected(format!("send to the gateway: {error}")); + } } - Ok(other) => tracing_line(&format!("ignoring gateway message {other:?}")), - Err(error) => tracing_line(&format!("ignoring undecodable gateway frame: {error}")), - }, - Message::Ping(bytes) => { - if writer.send(Message::Pong(bytes)).await.is_err() { - return Ok(SessionEnd::Disconnected("pong failed".to_owned())); + incoming = tokio::time::timeout_at(deadline, reader.next()) => { + let Ok(incoming) = incoming else { + return SessionEnd::Disconnected(format!( + "no frame from the gateway for {} ms (heartbeat timeout)", + liveness.as_millis() + )); + }; + deadline = tokio::time::Instant::now() + liveness; + let message = match incoming { + Some(Ok(message)) => message, + Some(Err(error)) => return SessionEnd::Disconnected(error.to_string()), + None => return SessionEnd::Disconnected("gateway closed".to_owned()), + }; + match message { + Message::Text(text) => { + if let Some(end) = self.on_admitted_text(text.as_str(), &answers) { + return end; + } + } + Message::Binary(_) => { + return SessionEnd::Disconnected( + "the gateway sent a binary frame".to_owned(), + ); + } + Message::Close(frame) => { + return SessionEnd::Disconnected( + frame + .map(|frame| frame.reason.to_string()) + .filter(|reason| !reason.is_empty()) + .unwrap_or_else(|| "gateway closed".to_owned()), + ); + } + Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => {} + } } } - Message::Close(frame) => { - let _ = writer.flush().await; - return Ok(SessionEnd::Disconnected( - frame - .map(|frame| frame.reason.to_string()) - .unwrap_or_else(|| "gateway closed".to_owned()), + } + } + + /// Handle one text frame after admission; `Some` ends the session. + fn on_admitted_text(&self, text: &str, answers: &mpsc::Sender) -> Option { + match jsonrpc::parse(text) { + Err(rejection) => Some(SessionEnd::Disconnected(format!( + "the gateway sent an invalid frame: {}", + rejection.error + ))), + Ok(RpcMessage::Request(request)) => { + if request.method == RegistrationOpenStream::NAME { + match request.params::() { + Ok(params) => self.open_stream(request.id, params, answers.clone()), + Err(error) => answer(answers, Response::failure(Some(request.id), error)), + } + } else { + answer( + answers, + Response::failure( + Some(request.id), + ProtocolError::method_not_found(&request.method), + ), + ); + } + None + } + Ok(RpcMessage::Notification(notification)) => self.on_notification(notification), + Ok(RpcMessage::Response(response)) => { + tracing_line(&format!( + "ignoring a gateway response to id {}", + response + .id + .map_or_else(|| "null".to_owned(), |id| id.to_string()) )); + None } - Message::Binary(_) | Message::Pong(_) | Message::Frame(_) => {} } } -} -async fn protocol_mismatch_error( - registration: &RegistrationConfig, - gateway_protocol: u32, -) -> anyhow::Error { - let base = format!( - "gateway speaks protocol {gateway_protocol}; this daemon speaks {CURRENT_PROTOCOL_VERSION}; discovery document: {}", - registration.discovery_url - ); - let Some(restart) = ®istration.auto_upgrade else { - return TerminalRejection { - code: RegistrationRejectionCode::UnsupportedProtocol, - message: format!( - "{base}; run `lightspeed-envd upgrade` or set LIGHTSPEED_ENVD_AUTO_UPGRADE=1" - ), + fn on_notification(&self, notification: Notification) -> Option { + if notification.method != RegistrationClosed::NAME { + tracing_line(&format!( + "ignoring gateway notification {}", + notification.method + )); + return None; } - .into(); - }; - if std::env::var_os(AUTO_UPGRADE_ATTEMPTED_ENV).is_some() { - return TerminalRejection { - code: RegistrationRejectionCode::UnsupportedProtocol, - message: format!( - "{base}; automatic upgrade was already attempted by this process lineage" + let params = match notification.params::() { + Ok(params) => params, + Err(error) => { + tracing_line(&format!( + "ignoring undecodable registration/closed: {error}" + )); + return None; + } + }; + if let Some(message) = ¶ms.message { + tracing_line(&format!("gateway closed the session: {}", message.as_str())); + } + Some(SessionEnd::Exit(match params.reason { + ClosedReason::Superseded => DaemonExit::Superseded { + daemon_public_key: self.identity.public_key().to_hex(), + }, + ClosedReason::EnvironmentClosed => DaemonExit::EnvironmentClosed, + })) + } + + /// Dial the stream the gateway asked for on its own task and answer + /// once it is upgraded (or failed), then serve it. + fn open_stream(&self, id: Id, params: OpenStreamParams, answers: mpsc::Sender) { + let OpenStreamParams { token, stream } = params; + let StreamSpec::DataProtocol {} = stream; + let runtime = self.runtime.clone(); + let stream_url = self.urls.stream.clone(); + let tls = self.tls.clone(); + tokio::spawn(async move { + let dialed = + tokio::time::timeout(STREAM_DIAL_TIMEOUT, dial(&stream_url, Some(&token), &tls)) + .await; + let socket = match dialed { + Ok(Ok(socket)) => socket, + Ok(Err(error)) => { + tracing_line(&format!("stream dial failed: {error:#}")); + let error = ProtocolError::unavailable(format!("dial {stream_url}: {error:#}")); + let _ = answers + .send(Response::failure(Some(id), error).to_text()) + .await; + return; + } + Err(_) => { + let error = ProtocolError::unavailable(format!( + "dialing {stream_url} timed out after {} s", + STREAM_DIAL_TIMEOUT.as_secs() + )); + let _ = answers + .send(Response::failure(Some(id), error).to_text()) + .await; + return; + } + }; + let _ = answers + .send(Response::success(id, &OpenStreamResponse {}).to_text()) + .await; + drop(answers); + run_data_connection(&runtime, socket).await; + }); + } + + /// Install the gateway's build and re-execute it, or say why not. + /// Install the gateway's envd build and re-execute it: the build of + /// `gateway_protocol` when the challenge named one, otherwise (an + /// unreadable first frame, explained by `unreadable`) whatever protocol + /// discovery publishes, provided it is not this daemon's own. Each + /// lineage tries once per target protocol, and once after an + /// unreadable frame. + async fn upgrade( + &self, + gateway_protocol: Option, + unreadable: Option, + ) -> SessionEnd { + let base = match (gateway_protocol, &unreadable) { + (Some(gateway_protocol), _) => mismatch_summary(&self.registration, gateway_protocol), + (None, reason) => format!( + "{}; it is not an environment gateway this daemon can register with; discovery document: {}", + reason.as_deref().unwrap_or("the gateway is incompatible"), + self.registration.discovery_url ), + }; + if let Some(refusal) = upgrade_refusal(&self.registration, gateway_protocol) { + return SessionEnd::Exit(DaemonExit::Refused(format!("{base}; {refusal}"))); } - .into(); + let Some(restart) = &self.registration.auto_upgrade else { + return SessionEnd::Exit(DaemonExit::Refused(base)); + }; + tracing_line(&format!( + "{base}; automatic upgrade enabled, installing the gateway's envd build" + )); + let request = crate::upgrade::UpgradeRequest { + discovery_url: self.registration.discovery_url.clone(), + ca_file: self.registration.ca_file.clone(), + install_path: restart.executable.clone(), + target: release_info::TARGET.to_owned(), + expected_protocol: gateway_protocol, + other_than: gateway_protocol.is_none().then_some(ENVD_PROTOCOL_VERSION), + }; + let installed = tokio::select! { + _ = self.runtime.shutdown().cancelled() => { + return SessionEnd::Exit(DaemonExit::Shutdown); + } + installed = crate::upgrade::install(request) => installed, + }; + let installed = match installed { + Ok(installed) => installed, + Err(error) if error.is_transient() => { + return SessionEnd::Retry { + reason: format!("{base}; automatic upgrade will retry: {error}"), + retry_after: None, + }; + } + Err(error) => { + return SessionEnd::Exit(DaemonExit::Refused(format!( + "{base}; automatic upgrade failed: {error}" + ))); + } + }; + tracing_line(&format!( + "installed lightspeed-envd {} ({}, protocol {}); re-executing {}", + installed.version, + installed.git_sha, + installed.protocol_version, + restart.executable.display() + )); + let error = exec_replacement( + restart, + installed.protocol_version, + self.registration_key.as_ref(), + ); + SessionEnd::Exit(DaemonExit::Refused(format!( + "{base}; installed the replacement but could not re-execute it: {error}" + ))) } +} - tracing_line(&format!( - "{base}; automatic upgrade enabled, installing the gateway's envd build" - )); - let request = crate::upgrade::UpgradeRequest { - discovery_url: registration.discovery_url.clone(), - ca_file: registration.ca_file.clone(), - install_path: restart.executable.clone(), - target: release_info::TARGET.to_owned(), - expected_protocol: Some(gateway_protocol), - }; - let installed = match crate::upgrade::install(request).await { - Ok(installed) => installed, - Err(error) => { - return TerminalRejection { - code: RegistrationRejectionCode::UnsupportedProtocol, - message: format!("{base}; automatic upgrade failed: {error:#}"), +/// The challenge params, decoded tolerantly: the frame is frozen and may +/// carry members this daemon does not know. +fn decode_challenge(frame: &str) -> Result { + let value: serde_json::Value = serde_json::from_str(frame).context("decode challenge")?; + let params = value + .get("params") + .cloned() + .ok_or_else(|| anyhow!("challenge has no params"))?; + serde_json::from_value(params).context("decode challenge params") +} + +/// Map a register error to what the daemon does next. +fn refusal(error: ProtocolError) -> SessionEnd { + match &error.data { + ErrorData::VersionMismatch { expected, .. } => SessionEnd::ProtocolMismatch(*expected), + ErrorData::FailedPrecondition { + reason: PreconditionReason::EnvironmentClosed, + } => SessionEnd::Exit(DaemonExit::EnvironmentClosed), + ErrorData::Busy { retry_after_ms, .. } | ErrorData::Unavailable { retry_after_ms } => { + SessionEnd::Retry { + reason: error.to_string(), + retry_after: retry_after_ms.map(Duration::from_millis), } - .into(); } - }; - tracing_line(&format!( - "installed lightspeed-envd {} ({}, protocol {}); re-executing {}", - installed.version, - installed.git_sha, - installed.protocol_version, - restart.executable.display() - )); - let error = exec_replacement( - &restart.executable, - &restart.arg0, - &restart.args, - registration.registration_key.as_ref(), - ); - TerminalRejection { - code: RegistrationRejectionCode::UnsupportedProtocol, - message: format!("{base}; installed the replacement but could not re-execute it: {error}"), + _ if error.is_retryable() => SessionEnd::Retry { + reason: error.to_string(), + retry_after: None, + }, + _ => SessionEnd::Exit(DaemonExit::Refused(error.to_string())), } - .into() +} + +fn mismatch_summary(registration: &RegistrationConfig, gateway_protocol: u32) -> String { + format!( + "gateway speaks protocol {gateway_protocol}; this daemon speaks {ENVD_PROTOCOL_VERSION}; discovery document: {}", + registration.discovery_url + ) +} + +/// Why no automatic upgrade runs for this gateway protocol (`None`: an +/// unreadable first frame), if it does not. +fn upgrade_refusal( + registration: &RegistrationConfig, + gateway_protocol: Option, +) -> Option { + if registration.auto_upgrade.is_none() { + return Some( + "run `lightspeed-envd upgrade` or set LIGHTSPEED_ENVD_AUTO_UPGRADE=1".to_owned(), + ); + } + match (gateway_protocol, registration.upgrade_attempted_for) { + (Some(gateway_protocol), Some(attempted)) if attempted == gateway_protocol => { + Some(format!( + "an automatic upgrade to protocol {gateway_protocol} already ran in this process lineage and the result still does not match" + )) + } + (None, Some(attempted)) => Some(format!( + "an automatic upgrade (to protocol {attempted}) already ran in this process lineage and the gateway's first frame is still unreadable" + )), + _ => None, + } +} + +/// Arguments of the restarted daemon: the original ones without any +/// registration key option, which travels only through the one-exec +/// variable and only while it is still needed. +fn replacement_args(args: &[std::ffi::OsString]) -> Vec { + let mut kept = Vec::with_capacity(args.len()); + let mut skip_value = false; + for arg in args { + if std::mem::take(&mut skip_value) { + continue; + } + let text = arg.to_string_lossy(); + if text == "--registration-key" || text == "--registration-key-file" { + skip_value = true; + continue; + } + if text.starts_with("--registration-key=") || text.starts_with("--registration-key-file=") { + continue; + } + kept.push(arg.clone()); + } + kept } #[cfg(unix)] fn exec_replacement( - executable: &Path, - arg0: &std::ffi::OsStr, - args: &[std::ffi::OsString], + restart: &RestartInvocation, + gateway_protocol: u32, registration_key: Option<&SecretString>, ) -> std::io::Error { use std::os::unix::process::CommandExt as _; - let mut command = Command::new(executable); + let mut command = std::process::Command::new(&restart.executable); command - .arg0(arg0) - .args(args) - .env(AUTO_UPGRADE_ATTEMPTED_ENV, "1"); - // Configuration parsing deliberately removes key variables from the - // current process. Hand the already-held value across exactly this exec - // so a first-seen daemon can still register even if its key file was - // removed; the replacement consumes and scrubs this internal variable. + .arg0(&restart.arg0) + .args(replacement_args(&restart.args)) + .env(AUTO_UPGRADE_ATTEMPTED_ENV, gateway_protocol.to_string()) + .env_remove("LIGHTSPEED_ENVD_REGISTRATION_KEY") + .env_remove("LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE") + .env_remove(REEXEC_REGISTRATION_KEY_ENV); + // Configuration removed the key variables from this process. A daemon + // not yet admitted hands the held key across exactly this exec, so it + // can still register even if its key file was removed; an admitted one + // proves itself with its private key alone. if let Some(registration_key) = registration_key { - command - .env(REEXEC_REGISTRATION_KEY_ENV, registration_key.expose()) - .env_remove("LIGHTSPEED_ENVD_REGISTRATION_KEY") - .env_remove("LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE"); + command.env(REEXEC_REGISTRATION_KEY_ENV, registration_key.expose()); } command.exec() } #[cfg(not(unix))] fn exec_replacement( - _executable: &Path, - _arg0: &std::ffi::OsStr, - _args: &[std::ffi::OsString], + _restart: &RestartInvocation, + _gateway_protocol: u32, _registration_key: Option<&SecretString>, ) -> std::io::Error { std::io::Error::new( @@ -319,59 +685,161 @@ fn exec_replacement( ) } +/// Send `registration/goodbye` and the close frame, within a short bound. +async fn say_goodbye(writer: &mut ControlWriter) { + let goodbye = Notification::new( + RegistrationGoodbye::NAME, + &GoodbyeParams { + reason: GoodbyeReason::Shutdown, + }, + ) + .expect("goodbye params encode"); + let _ = tokio::time::timeout(GOODBYE_TIMEOUT, async { + writer.send(Message::Text(goodbye.to_text().into())).await?; + writer.close().await + }) + .await; +} + +fn answer(answers: &mpsc::Sender, response: Response) { + if answers.try_send(response.to_text()).is_err() { + tracing_line("dropping an answer to the gateway: the control writer is backed up"); + } +} + +/// The next text frame before admission. Control frames are skipped; a +/// binary frame, a close or a transport error ends the session. +async fn next_text(reader: &mut ControlReader) -> Result { + loop { + match reader.next().await { + Some(Ok(Message::Text(text))) => return Ok(text.as_str().to_owned()), + Some(Ok(Message::Binary(_))) => { + return Err(SessionEnd::Disconnected( + "the gateway sent a binary frame".to_owned(), + )); + } + Some(Ok(Message::Close(_))) | None => { + return Err(SessionEnd::Disconnected( + "gateway closed before admission".to_owned(), + )); + } + Some(Ok(_)) => continue, + Some(Err(error)) => return Err(SessionEnd::Disconnected(error.to_string())), + } + } +} + +/// Wait for the register response. Until it arrives the gateway may send +/// nothing else: any request or notification breaks the frame rules. +async fn register_response(reader: &mut ControlReader) -> Result { + loop { + let text = next_text(reader).await?; + match jsonrpc::parse(&text) { + Ok(RpcMessage::Response(response)) if response.id == Some(Id::Number(REGISTER_ID)) => { + return Ok(response); + } + Ok(RpcMessage::Response(response)) => tracing_line(&format!( + "ignoring a gateway response to id {}", + response + .id + .map_or_else(|| "null".to_owned(), |id| id.to_string()) + )), + Ok(RpcMessage::Request(request)) => { + return Err(SessionEnd::Disconnected(format!( + "the gateway sent {} before the register response", + request.method + ))); + } + Ok(RpcMessage::Notification(notification)) => { + return Err(SessionEnd::Disconnected(format!( + "the gateway sent {} before the register response", + notification.method + ))); + } + Err(rejection) => { + return Err(SessionEnd::Disconnected(format!( + "the gateway sent an invalid frame: {}", + rejection.error + ))); + } + } + } +} + +/// Dial a gateway route with the protocol's message limits, optionally +/// presenting a bearer token. +async fn dial( + url: &str, + bearer: Option<&SecretString>, + tls: &TlsSettings, +) -> Result { + let mut request = url.into_client_request().context("gateway URL")?; + request.headers_mut().insert( + USER_AGENT, + HeaderValue::from_static(concat!("lightspeed-envd/", env!("CARGO_PKG_VERSION"))), + ); + if let Some(token) = bearer { + let mut value = + HeaderValue::from_str(&format!("Bearer {}", token.expose())).context("token header")?; + value.set_sensitive(true); + request.headers_mut().insert(AUTHORIZATION, value); + } + let (socket, _) = connect_async_tls_with_config( + request, + Some(websocket_config()), + false, + Some(tls.connector()), + ) + .await + .map_err(|error| describe_connect_error(url, error))?; + Ok(socket) +} + /// A gateway that answers the WebSocket upgrade with an HTTP status is /// reachable but not serving registration; say so instead of leaking a raw /// handshake error. Still retried, because a deploy can produce the same /// answer for a moment. fn describe_connect_error( - gateway_url: &str, + url: &str, error: tokio_tungstenite::tungstenite::Error, ) -> anyhow::Error { if let tokio_tungstenite::tungstenite::Error::Http(response) = &error { let status = response.status(); if status == 404 { return anyhow!( - "{gateway_url} does not serve environment registration (HTTP 404): point LIGHTSPEED_ENVD_GATEWAY_URL at a Lightspeed process running the environment-gateway role" + "{url} does not serve environment registration (HTTP 404): point LIGHTSPEED_ENVD_GATEWAY_URL at a Lightspeed process running the environment-gateway role" ); } - return anyhow!( - "environment gateway {gateway_url} answered HTTP {status} instead of upgrading" - ); + return anyhow!("environment gateway {url} answered HTTP {status} instead of upgrading"); } - anyhow::Error::new(error).context(format!("connect to environment gateway {gateway_url}")) + anyhow::Error::new(error).context(format!("connect to environment gateway {url}")) } -fn rejection(code: RegistrationRejectionCode, message: String) -> Result { - if code.is_terminal() { - Err(TerminalRejection { code, message }.into()) - } else { - Ok(SessionEnd::Retryable(code, message)) - } +/// Refuse to start when the receipt cannot be written: a daemon admitted +/// without its receipt would be unusable to whoever waits for it. +fn check_receipt_path(path: &Path) -> Result<()> { + let parent = receipt_parent(path); + std::fs::create_dir_all(&parent) + .with_context(|| format!("create receipt dir {}", parent.display()))?; + let probe = parent.join(format!( + ".{}.probe.{}", + path.file_name() + .map_or_else(|| "receipt".into(), |name| name.to_string_lossy()), + std::process::id() + )); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&probe) + .with_context(|| format!("receipt dir {} is not writable", parent.display()))?; + let _ = std::fs::remove_file(&probe); + Ok(()) } -async fn next_control_message( - reader: &mut futures_util::stream::SplitStream>, -) -> Result> -where - S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, -{ - loop { - match reader.next().await { - Some(Ok(Message::Text(text))) => { - return serde_json::from_str(&text) - .map(Some) - .context("decode gateway control message"); - } - Some(Ok(Message::Binary(bytes))) => { - return serde_json::from_slice(&bytes) - .map(Some) - .context("decode gateway control message"); - } - Some(Ok(Message::Close(_))) | None => return Ok(None), - Some(Ok(_)) => continue, - Some(Err(error)) => return Err(error.into()), - } - } +fn receipt_parent(path: &Path) -> PathBuf { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map_or_else(|| PathBuf::from("."), Path::to_path_buf) } /// Write the receipt atomically (when a path is configured) and log it once. @@ -379,43 +847,21 @@ where fn announce_receipt(receipt: &RegistrationReceipt, path: Option<&Path>) -> Result<()> { let json = serde_json::to_string(receipt)?; if let Some(path) = path { - if let Some(parent) = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - { - std::fs::create_dir_all(parent) - .with_context(|| format!("create receipt dir {}", parent.display()))?; - } + let parent = receipt_parent(path); + std::fs::create_dir_all(&parent) + .with_context(|| format!("create receipt dir {}", parent.display()))?; let temp = path.with_extension(format!("tmp.{}", std::process::id())); std::fs::write(&temp, format!("{json}\n")) .with_context(|| format!("write receipt {}", temp.display()))?; - std::fs::rename(&temp, path) - .with_context(|| format!("install receipt {}", path.display()))?; + if let Err(error) = std::fs::rename(&temp, path) { + let _ = std::fs::remove_file(&temp); + bail!("install receipt {}: {error}", path.display()); + } } tracing_line(&format!("registered {json}")); Ok(()) } -/// Dial the gateway data route for one waiting worker and run the ordinary -/// data protocol on it. The token is a one-time pairing credential issued -/// over the authenticated control connection. -async fn serve_data_connection( - runtime: DaemonRuntime, - data_url: &str, - token: SecretString, - tls: &TlsSettings, -) -> Result<()> { - let mut request = data_url.into_client_request().context("data URL")?; - request.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {}", token.expose())).context("token header")?, - ); - let (socket, _) = connect_async_tls_with_config(request, None, false, Some(tls.connector())) - .await - .context("connect data route")?; - run_data_connection(&runtime, socket).await -} - /// TLS trust for the gateway: the bundled WebPKI roots, plus an operator /// supplied PEM bundle for gateways behind a private CA. /// @@ -472,65 +918,584 @@ fn jittered(base: Duration) -> Duration { base.mul_f64(factor) } -#[allow(dead_code)] -type ControlSocket = WebSocketStream>; - #[cfg(test)] mod tests { + use std::sync::Mutex; + + use environment_protocol::{ + error::{BusyReason, PermissionReason}, + registration::{ChallengeNonce, signed_registration_message}, + routes::{CONNECT_PATH, DATA_PATH, canonical_origin}, + }; + use serde_json::{Value, json}; + use tokio::net::{TcpListener, TcpStream}; + use tokio_tungstenite::{ + accept_hdr_async_with_config, + tungstenite::handshake::server::{Request as HttpRequest, Response as HttpResponse}, + }; + use super::*; + use crate::test_support::runtime_with; + + type GatewaySocket = WebSocketStream; + + /// A scripted gateway on a loopback port. + struct FakeGateway { + listener: TcpListener, + connect_url: String, + } + + struct Accepted { + socket: GatewaySocket, + path: String, + authorization: Option, + } + + impl FakeGateway { + async fn start(prefix: &str) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let connect_url = format!( + "ws://{}{prefix}{CONNECT_PATH}", + listener.local_addr().unwrap() + ); + Self { + listener, + connect_url, + } + } + + // The callback's error type is fixed by tungstenite's `Callback`. + #[allow(clippy::result_large_err)] + async fn accept(&self) -> Accepted { + let (stream, _) = tokio::time::timeout(Duration::from_secs(10), self.listener.accept()) + .await + .expect("the daemon dials in time") + .unwrap(); + let seen = Arc::new(Mutex::new((String::new(), None))); + let record = seen.clone(); + let socket = accept_hdr_async_with_config( + stream, + move |request: &HttpRequest, response: HttpResponse| { + let authorization = request + .headers() + .get(AUTHORIZATION) + .map(|value| value.to_str().unwrap().to_owned()); + *record.lock().unwrap() = (request.uri().path().to_owned(), authorization); + Ok(response) + }, + Some(websocket_config()), + ) + .await + .unwrap(); + let (path, authorization) = seen.lock().unwrap().clone(); + Accepted { + socket, + path, + authorization, + } + } + } + + async fn send(socket: &mut GatewaySocket, frame: Value) { + socket + .send(Message::Text(frame.to_string().into())) + .await + .unwrap(); + } + + async fn recv(socket: &mut GatewaySocket) -> Value { + loop { + let frame = tokio::time::timeout(Duration::from_secs(10), socket.next()) + .await + .expect("a frame in time") + .expect("open") + .expect("frame"); + if let Message::Text(text) = frame { + return serde_json::from_str(text.as_str()).unwrap(); + } + } + } + + async fn challenge(socket: &mut GatewaySocket, nonce: [u8; 32]) { + send( + socket, + json!({ + "jsonrpc": "2.0", + "method": "registration/challenge", + "params": { + "protocolVersion": ENVD_PROTOCOL_VERSION, + "nonce": ChallengeNonce::from_bytes(nonce).to_hex(), + "addedLater": {"x": 1}, + }, + }), + ) + .await; + } + + fn receipt(admission: &str) -> Value { + json!({ + "environmentId": "environment_a", + "incarnationId": "incarnation_a", + "daemonId": "daemon_a", + "controlConnectionId": "connection_a", + "identityMode": "ephemeral", + "admission": admission, + }) + } + + async fn admit(socket: &mut GatewaySocket, register: &Value, heartbeat_ms: u64) { + send( + socket, + json!({ + "jsonrpc": "2.0", + "id": register["id"], + "result": {"receipt": receipt("created"), "heartbeatIntervalMs": heartbeat_ms}, + }), + ) + .await; + } + + async fn refuse(socket: &mut GatewaySocket, register: &Value, error: ProtocolError) { + send( + socket, + Response::failure( + Some(serde_json::from_value(register["id"].clone()).unwrap()), + error, + ) + .to_value(), + ) + .await; + } + + fn daemon( + gateway: &FakeGateway, + adjust: impl FnOnce(&mut RegistrationConfig), + ) -> (tempfile::TempDir, DaemonRuntime, RegistrationConfig) { + let mut registration = RegistrationConfig::new( + gateway.connect_url.clone(), + crate::upgrade::resolve_discovery_url(Some(&gateway.connect_url), None).unwrap(), + ); + registration.registration_key = Some(SecretString::new("lsrk_test")); + adjust(&mut registration); + let config_registration = registration.clone(); + let (temp, runtime) = runtime_with(move |config| { + config.registration = Some(config_registration); + }); + (temp, runtime, registration) + } + + #[tokio::test(flavor = "current_thread")] + async fn registers_with_an_origin_bound_signature_and_drops_the_key_once_admitted() { + let gateway = FakeGateway::start("/ls").await; + let receipt_dir = tempfile::tempdir().unwrap(); + let receipt_path = receipt_dir.path().join("out").join("registration.json"); + let (_temp, runtime, registration) = daemon(&gateway, |registration| { + registration.receipt_path = Some(receipt_path.clone()); + }); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + + let mut first = gateway.accept().await; + assert_eq!(first.path, format!("/ls{CONNECT_PATH}")); + let nonce = [5u8; 32]; + challenge(&mut first.socket, nonce).await; + let register = recv(&mut first.socket).await; + assert_eq!(register["method"], "registration/register"); + let params = RegisterParams::decode(®ister["params"]).expect("valid register"); + assert_eq!(params.protocol_version, ENVD_PROTOCOL_VERSION); + assert_eq!( + params.registration_key.as_ref().map(SecretString::expose), + Some("lsrk_test") + ); + let origin = canonical_origin(&gateway.connect_url).unwrap(); + let key = + ed25519_dalek::VerifyingKey::from_bytes(params.daemon_public_key.as_bytes()).unwrap(); + let signature = ed25519_dalek::Signature::from_bytes(params.signature.as_bytes()); + key.verify_strict( + &signed_registration_message(&ChallengeNonce::from_bytes(nonce), &origin), + &signature, + ) + .expect("the signature binds the nonce and the dialed origin"); + admit(&mut first.socket, ®ister, 30_000).await; + + // The receipt is written once admitted. + for _ in 0..200 { + if receipt_path.exists() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let written: RegistrationReceipt = + serde_json::from_str(&std::fs::read_to_string(&receipt_path).unwrap()).unwrap(); + assert_eq!(written.daemon_id.as_str(), "daemon_a"); + + // A reconnect proves the identity by its key alone. + drop(first); + let mut second = gateway.accept().await; + challenge(&mut second.socket, [6u8; 32]).await; + let again = recv(&mut second.socket).await; + let again = RegisterParams::decode(&again["params"]).unwrap(); + assert!(again.registration_key.is_none(), "the key is never resent"); + assert_eq!(again.daemon_public_key, params.daemon_public_key); + + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn open_stream_dials_the_derived_data_route_with_the_token() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [1u8; 32]).await; + let register = recv(&mut control.socket).await; + admit(&mut control.socket, ®ister, 30_000).await; + + send( + &mut control.socket, + json!({ + "jsonrpc": "2.0", + "id": "g1", + "method": "registration/openStream", + "params": {"token": "stream-token", "stream": {"kind": "dataProtocol"}}, + }), + ) + .await; + let mut stream = gateway.accept().await; + assert_eq!(stream.path, DATA_PATH); + assert_eq!(stream.authorization.as_deref(), Some("Bearer stream-token")); + let answer = recv(&mut control.socket).await; + assert_eq!(answer["id"], "g1"); + assert_eq!(answer["result"], json!({})); + + // The stream serves the data plane. + send( + &mut stream.socket, + json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { + "protocolVersion": ENVD_PROTOCOL_VERSION, "clientInfo": {"name": "gateway-test"}, + }}), + ) + .await; + let initialized = recv(&mut stream.socket).await; + assert_eq!(initialized["result"]["plane"], "data"); + + // An unknown gateway request is answered, and the session stays. + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "id": "g2", "method": "registration/somethingNew"}), + ) + .await; + let unknown: ProtocolError = + serde_json::from_value(recv(&mut control.socket).await["error"].clone()).unwrap(); + assert_eq!( + unknown.kind(), + environment_protocol::error::ProtocolErrorKind::MethodNotFound + ); + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_silent_gateway_is_dropped_after_three_heartbeat_intervals() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut first = gateway.accept().await; + challenge(&mut first.socket, [2u8; 32]).await; + let register = recv(&mut first.socket).await; + let admitted_at = Instant::now(); + admit(&mut first.socket, ®ister, 1_000).await; + // Keep the socket open but silent: only the deadline can end it. + let _held = first; + let _second = gateway.accept().await; + let elapsed = admitted_at.elapsed(); + assert!( + elapsed >= Duration::from_millis(2_900), + "reconnected after {elapsed:?}, before the deadline" + ); + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn closed_notifications_end_the_process_with_distinct_statuses() { + for (reason, status) in [("superseded", 5), ("environmentClosed", 4)] { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [3u8; 32]).await; + let register = recv(&mut control.socket).await; + admit(&mut control.socket, ®ister, 30_000).await; + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "method": "registration/closed", "params": { + "reason": reason, "message": "another process with this identity connected", + }}), + ) + .await; + let exit = tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), status, "{reason}: {exit}"); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn register_errors_are_terminal_unless_retryable() { + let terminal = [ + ( + ProtocolError::permission_denied( + Some(PermissionReason::InvalidRegistrationKey), + "unknown key", + ), + 3, + ), + ( + ProtocolError::failed_precondition(PreconditionReason::EnvironmentClosed, "closed"), + 4, + ), + ( + ProtocolError::invalid_request("implementation.version is too long"), + 3, + ), + ]; + for (error, status) in terminal { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [4u8; 32]).await; + let register = recv(&mut control.socket).await; + refuse(&mut control.socket, ®ister, error.clone()).await; + let exit = tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), status, "{error}"); + } + + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [4u8; 32]).await; + let register = recv(&mut control.socket).await; + refuse( + &mut control.socket, + ®ister, + ProtocolError::busy(Some(BusyReason::RateLimited), Some(1_500), "slow down"), + ) + .await; + let refused_at = Instant::now(); + let _again = gateway.accept().await; + assert!( + refused_at.elapsed() >= Duration::from_millis(1_400), + "the retry hint is honoured" + ); + assert!(!task.is_finished()); + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_gateway_request_before_admission_breaks_the_session() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [7u8; 32]).await; + let _register = recv(&mut control.socket).await; + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "id": "early", "method": "registration/openStream", "params": { + "token": "t", "stream": {"kind": "dataProtocol"}, + }}), + ) + .await; + let mut ended = false; + while let Ok(Some(frame)) = + tokio::time::timeout(Duration::from_secs(5), control.socket.next()).await + { + match frame { + Ok(Message::Close(_)) | Err(_) => { + ended = true; + break; + } + Ok(Message::Text(text)) => panic!("the daemon answered: {text}"), + Ok(_) => {} + } + } + assert!(ended, "the daemon closed the connection"); + let _reconnected = gateway.accept().await; + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn shutdown_says_goodbye_and_exits_cleanly() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [8u8; 32]).await; + let register = recv(&mut control.socket).await; + admit(&mut control.socket, ®ister, 30_000).await; + // Give the daemon a moment to enter its admitted loop. + tokio::time::sleep(Duration::from_millis(100)).await; + runtime.shutdown().cancel(); + let goodbye = recv(&mut control.socket).await; + assert_eq!(goodbye["method"], "registration/goodbye"); + assert_eq!(goodbye["params"], json!({"reason": "shutdown"})); + assert!(goodbye.get("id").is_none()); + let exit = tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_gateway_of_another_version_without_auto_upgrade_is_terminal() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let discovery = registration.discovery_url.clone(); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "method": "registration/challenge", "params": { + "protocolVersion": ENVD_PROTOCOL_VERSION + 1, "nonce": "not even hex", + }}), + ) + .await; + let exit = tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), 3); + let DaemonExit::Refused(message) = exit else { + panic!("refused"); + }; + assert!(message.contains(&discovery)); + assert!(message.contains("lightspeed-envd upgrade")); + } + + #[test] + fn an_upgrade_runs_once_per_target_version() { + let mut registration = RegistrationConfig::new( + "wss://gateway.example/environment-gateway/connect".to_owned(), + "https://gateway.example/.well-known/lightspeed-envd".to_owned(), + ); + assert!( + upgrade_refusal(®istration, Some(4)).is_some(), + "not opted in" + ); + assert!( + upgrade_refusal(®istration, None).is_some(), + "not opted in" + ); + registration.auto_upgrade = Some(RestartInvocation { + executable: PathBuf::from("/usr/local/bin/lightspeed-envd"), + arg0: "lightspeed-envd".into(), + args: Vec::new(), + }); + assert!(upgrade_refusal(®istration, Some(4)).is_none()); + assert!( + upgrade_refusal(®istration, None).is_none(), + "an unreadable first frame upgrades to whatever discovery publishes" + ); + registration.upgrade_attempted_for = Some(4); + assert!(upgrade_refusal(®istration, Some(4)).is_some()); + assert!( + upgrade_refusal(®istration, Some(5)).is_none(), + "a later bump upgrades again" + ); + assert!( + upgrade_refusal(®istration, None).is_some(), + "a replacement that still cannot read the first frame stops" + ); + } + + #[test] + fn a_replacement_never_inherits_a_registration_key_argument() { + let args: Vec = [ + "--registration-key", + "lsrk_secret", + "--gateway-url", + "wss://g/environment-gateway/connect", + "--registration-key-file=/run/key", + "--registration-name", + "n", + ] + .into_iter() + .map(Into::into) + .collect(); + let kept: Vec = replacement_args(&args) + .into_iter() + .map(|arg| arg.into_string().unwrap()) + .collect(); + assert_eq!( + kept, + [ + "--gateway-url", + "wss://g/environment-gateway/connect", + "--registration-name", + "n" + ] + ); + } + + #[test] + fn retryable_refusals_carry_their_hint() { + let SessionEnd::Retry { retry_after, .. } = refusal(ProtocolError::busy( + Some(BusyReason::RateLimited), + Some(250), + "x", + )) else { + panic!("retry"); + }; + assert_eq!(retry_after, Some(Duration::from_millis(250))); + assert!(matches!( + refusal(ProtocolError::unavailable("store down")), + SessionEnd::Retry { .. } + )); + assert!(matches!( + refusal(ProtocolError::version_mismatch(4, 3)), + SessionEnd::ProtocolMismatch(4) + )); + assert!(matches!( + refusal(ProtocolError::expired("key expired")), + SessionEnd::Exit(DaemonExit::Refused(_)) + )); + } #[test] fn receipts_are_written_atomically_and_contain_only_ids() { let temp = tempfile::tempdir().expect("tempdir"); let path = temp.path().join("out").join("registration.json"); - let receipt = RegistrationReceipt { - environment_id: "environment_a".to_owned(), - incarnation_id: "incarnation_a".to_owned(), - daemon_id: "daemon_a".to_owned(), - connection_id: "connection_a".to_owned(), - identity_mode: "ephemeral".to_owned(), - }; + check_receipt_path(&path).expect("writable"); + let receipt: RegistrationReceipt = serde_json::from_value(receipt("reconnected")).unwrap(); announce_receipt(&receipt, Some(&path)).expect("write"); let written: RegistrationReceipt = serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("json"); assert_eq!(written, receipt); - assert!(std::fs::read_dir(path.parent().unwrap()).unwrap().count() == 1); + assert_eq!( + std::fs::read_dir(path.parent().unwrap()).unwrap().count(), + 1 + ); } + #[cfg(unix)] #[test] - fn terminal_rejections_stop_and_transient_ones_retry() { - assert!(rejection(RegistrationRejectionCode::EnvironmentClosed, "x".to_owned()).is_err()); - assert!(matches!( - rejection(RegistrationRejectionCode::CapacityExhausted, "x".to_owned()), - Ok(SessionEnd::Retryable( - RegistrationRejectionCode::CapacityExhausted, - _ - )) - )); - } - - #[tokio::test(flavor = "current_thread")] - async fn protocol_mismatch_without_auto_upgrade_names_discovery_and_command() { - let registration = RegistrationConfig { - gateway_url: "wss://gateway.example/environment-gateway/connect".to_owned(), - registration_key: None, - display_name: None, - metadata: Default::default(), - receipt_path: None, - ca_file: None, - discovery_url: "https://gateway.example/.well-known/lightspeed-envd".to_owned(), - auto_upgrade: None, - }; - let error = protocol_mismatch_error(®istration, CURRENT_PROTOCOL_VERSION + 1).await; - let rejection = error - .downcast_ref::() - .expect("terminal rejection"); - assert_eq!( - rejection.code, - RegistrationRejectionCode::UnsupportedProtocol - ); - assert!(rejection.message.contains(®istration.discovery_url)); - assert!(rejection.message.contains("lightspeed-envd upgrade")); + fn an_unwritable_receipt_dir_is_refused_at_startup() { + use std::os::unix::fs::PermissionsExt as _; + let temp = tempfile::tempdir().expect("tempdir"); + let locked = temp.path().join("locked"); + std::fs::create_dir(&locked).unwrap(); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o500)).unwrap(); + let refused = check_receipt_path(&locked.join("registration.json")); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700)).unwrap(); + // Root ignores directory permissions. + if !rustix::process::geteuid().is_root() { + assert!(refused.is_err()); + } } /// A self-signed EC certificate, valid 2026-2036, used only as PEM input. diff --git a/crates/environment-daemon/src/rpc.rs b/crates/environment-daemon/src/rpc.rs deleted file mode 100644 index 35d2a370b..000000000 --- a/crates/environment-daemon/src/rpc.rs +++ /dev/null @@ -1,82 +0,0 @@ -use environment_protocol::error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}; -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::{Value, json}; - -#[derive(Debug)] -pub struct RpcRequest { - pub id: Option, - pub method: Option, - pub params: Value, -} - -pub fn parse_request(value: Value) -> Result { - if !value.is_object() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "JSON-RPC request must be an object", - )); - } - let id = value.get("id").cloned(); - let method = value - .get("method") - .and_then(Value::as_str) - .map(ToOwned::to_owned); - let params = value.get("params").cloned().unwrap_or(Value::Null); - Ok(RpcRequest { id, method, params }) -} - -pub fn decode_params(params: Value) -> Result -where - T: DeserializeOwned, -{ - serde_json::from_value(params).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - error.to_string(), - ) - }) -} - -pub fn encode_result(value: T) -> Result -where - T: Serialize, -{ - serde_json::to_value(value).map_err(|error| { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::Internal, error.to_string()) - }) -} - -pub fn success_response(id: Value, result: Value) -> Value { - json!({ - "jsonrpc": "2.0", - "id": id, - "result": result - }) -} - -pub fn error_response(id: Option, error: EnvironmentProtocolError) -> Value { - json!({ - "jsonrpc": "2.0", - "id": id.unwrap_or(Value::Null), - "error": error - }) -} - -pub fn method_not_found(method: &str) -> EnvironmentProtocolError { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Unsupported, - format!("unsupported environment-protocol method: {method}"), - ) -} - -pub fn invalid_request(message: impl Into) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::InvalidRequest, message) -} - -pub fn not_found(message: impl Into) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::NotFound, message) -} - -pub fn unsupported(message: impl Into) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::Unsupported, message) -} diff --git a/crates/environment-daemon/src/server.rs b/crates/environment-daemon/src/server.rs index 21db1d868..2f4d8be54 100644 --- a/crates/environment-daemon/src/server.rs +++ b/crates/environment-daemon/src/server.rs @@ -1,44 +1,111 @@ -use anyhow::Context as _; -use environment_protocol::data::{ - methods::{FS_CAPTURE_METHOD, FS_MATERIALIZE_METHOD}, - transfer::*, +//! Serving the data plane: the passive listener and the connection loop +//! every data socket runs, accepted or reverse-dialed. +//! +//! Per connection, one reader parses frames with the shared envelope and +//! enforces the handshake; one writer task owns the socket sink. Requests +//! after `initialize` run as their own tasks, up to +//! `limits.maxConcurrentRequests` at once, so responses leave in completion +//! order and the reader never waits on the writer. When the reader ends, +//! every pending request of the connection is cancelled and its reply, +//! with any after-write hook, is dropped. + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, }; + +use anyhow::Context as _; use environment_protocol::{ - data::{ - fs::{ - CopyParams, CreateDirectoryParams, GetMetadataParams, GlobFilesParams, - ReadDirectoryParams, ReadFileParams, RemoveParams, SearchTextParams, WriteFileParams, - }, - handshake::{InitializeParams, InitializeResponse, InitializedParams}, - idle::IdleParams, - jobs::{CancelJobsParams, ListJobsParams, ReadJobsParams, StartJobsParams}, - methods::{ - ENV_IDLE_METHOD, FS_COPY_METHOD, FS_CREATE_DIRECTORY_METHOD, FS_GET_METADATA_METHOD, - FS_GLOB_FILES_METHOD, FS_READ_DIRECTORY_METHOD, FS_READ_FILE_METHOD, FS_REMOVE_METHOD, - FS_SEARCH_TEXT_METHOD, FS_WRITE_FILE_METHOD, - INITIALIZE_METHOD as DATA_INITIALIZE_METHOD, INITIALIZED_METHOD, JOB_CANCEL_METHOD, - JOB_LIST_METHOD, JOB_READ_METHOD, JOB_START_METHOD, PROCESS_READ_METHOD, - PROCESS_RESIZE_METHOD, PROCESS_START_METHOD, PROCESS_TERMINATE_METHOD, - PROCESS_WRITE_METHOD, - }, - process::{ - ReadProcessParams, ResizeProcessParams, StartProcessParams, TerminateProcessParams, - WriteProcessParams, - }, + ENVD_PROTOCOL_VERSION, + error::{BusyReason, ProtocolError}, + jsonrpc::{ + self, CANCEL_REQUEST_METHOD, CancelRequestParams, Id, Message as RpcMessage, Request, + Response, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::CURRENT_PROTOCOL_VERSION, + limits::MAX_MESSAGE_BYTES, + manifest::DataInitialize, }; -use futures_util::{SinkExt as _, StreamExt as _}; -use serde_json::Value; -use tokio::net::TcpListener; -use tokio::sync::{Semaphore, mpsc}; -use tokio_tungstenite::{accept_async, tungstenite::Message}; - -/// Serve the daemon until a transport fails terminally: the passive listener -/// when one is configured, the outbound registration loop when a gateway -/// URL is configured, or both. -pub async fn run(runtime: DaemonRuntime) -> anyhow::Result<()> { +use futures_util::{Sink, SinkExt as _, StreamExt as _}; +use tokio::{ + net::TcpListener, + sync::{Semaphore, mpsc}, +}; +use tokio_tungstenite::{ + WebSocketStream, accept_async_with_config, + tungstenite::{Message, protocol::WebSocketConfig}, +}; +use tokio_util::sync::CancellationToken; + +use crate::{ + DaemonRuntime, + config::ListenAddr, + dispatch::{self, ConnectionInfo, RequestContext, WrittenHook}, +}; + +/// How the daemon process ends. Each outcome has its own exit status so an +/// orchestrator can tell a deliberate stop from a refusal or a cloned +/// identity. +#[derive(Debug)] +pub enum DaemonExit { + /// `SIGTERM` or `SIGINT`; a registered daemon said goodbye first. + Shutdown, + /// Local configuration, state or receipt-file error. + Configuration(anyhow::Error), + /// The gateway refused registration for good, or it speaks another + /// protocol and no automatic upgrade succeeded. + Refused(String), + /// The gateway closed the identity's environment. + EnvironmentClosed, + /// Another process with this daemon identity connected and won. + Superseded { daemon_public_key: String }, + /// An unexpected failure. + Failed(anyhow::Error), +} + +impl DaemonExit { + pub fn status(&self) -> u8 { + match self { + Self::Shutdown => 0, + Self::Failed(_) => 1, + Self::Configuration(_) => 2, + Self::Refused(_) => 3, + Self::EnvironmentClosed => 4, + Self::Superseded { .. } => 5, + } + } +} + +impl std::fmt::Display for DaemonExit { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Shutdown => formatter.write_str("shut down"), + Self::Configuration(error) => write!(formatter, "configuration error: {error:#}"), + Self::Refused(message) => write!(formatter, "registration refused: {message}"), + Self::EnvironmentClosed => formatter.write_str("the environment was closed"), + Self::Superseded { daemon_public_key } => write!( + formatter, + "superseded: another process with daemon public key {daemon_public_key} connected; \ + two daemons share one state directory" + ), + Self::Failed(error) => write!(formatter, "failed: {error:#}"), + } + } +} + +/// The WebSocket limits of every envd socket: one message is one frame of +/// at most `MAX_MESSAGE_BYTES`. +pub(crate) fn websocket_config() -> WebSocketConfig { + let limit = usize::try_from(MAX_MESSAGE_BYTES).unwrap_or(usize::MAX); + WebSocketConfig::default() + .max_message_size(Some(limit)) + .max_frame_size(Some(limit)) +} + +/// Serve the daemon until it stops: the passive listener when one is +/// configured, the outbound registration loop when a gateway URL is +/// configured, or both. +pub async fn run(runtime: DaemonRuntime) -> DaemonExit { let listen = runtime.config().listen.clone(); let registration = runtime.config().registration.clone(); match (listen, registration) { @@ -47,18 +114,51 @@ pub async fn run(runtime: DaemonRuntime) -> anyhow::Result<()> { crate::registration::run_outbound(runtime, registration).await } (Some(listen), Some(registration)) => { - let listener = tokio::spawn(run_listener(runtime.clone(), listen)); - let outbound = tokio::spawn(crate::registration::run_outbound(runtime, registration)); + let listener = run_listener(runtime.clone(), listen); + let outbound = crate::registration::run_outbound(runtime, registration); + tokio::pin!(listener, outbound); tokio::select! { - result = listener => result.context("listener task")?, - result = outbound => result.context("registration task")?, + exit = &mut listener => match exit { + // The registration loop says goodbye on the same token. + DaemonExit::Shutdown => outbound.await, + other => other, + }, + exit = &mut outbound => exit, } } - (None, None) => anyhow::bail!("configure a listener or a gateway URL"), + (None, None) => { + DaemonExit::Configuration(anyhow::anyhow!("configure a listener or a gateway URL")) + } } } -async fn run_listener(runtime: DaemonRuntime, listen: ListenAddr) -> anyhow::Result<()> { +/// Cancel the runtime's shutdown token on the first `SIGTERM` or `SIGINT`. +pub fn spawn_signal_listener(runtime: &DaemonRuntime) -> anyhow::Result<()> { + let shutdown = runtime.shutdown().clone(); + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut terminate = signal(SignalKind::terminate()).context("install SIGTERM handler")?; + let mut interrupt = signal(SignalKind::interrupt()).context("install SIGINT handler")?; + tokio::spawn(async move { + tokio::select! { + _ = terminate.recv() => tracing_line("SIGTERM received; shutting down"), + _ = interrupt.recv() => tracing_line("SIGINT received; shutting down"), + } + shutdown.cancel(); + }); + } + #[cfg(not(unix))] + tokio::spawn(async move { + if tokio::signal::ctrl_c().await.is_ok() { + tracing_line("interrupt received; shutting down"); + shutdown.cancel(); + } + }); + Ok(()) +} + +async fn run_listener(runtime: DaemonRuntime, listen: ListenAddr) -> DaemonExit { match listen { ListenAddr::Tcp(addr) => run_tcp_listener(runtime, addr).await, #[cfg(unix)] @@ -66,31 +166,43 @@ async fn run_listener(runtime: DaemonRuntime, listen: ListenAddr) -> anyhow::Res } } -async fn run_tcp_listener( - runtime: DaemonRuntime, - listen: std::net::SocketAddr, -) -> anyhow::Result<()> { - let listener = TcpListener::bind(listen) +async fn run_tcp_listener(runtime: DaemonRuntime, listen: std::net::SocketAddr) -> DaemonExit { + let listener = match TcpListener::bind(listen) .await - .context("bind environment daemon listener")?; + .context("bind environment daemon listener") + { + Ok(listener) => listener, + Err(error) => return DaemonExit::Configuration(error), + }; tracing_line(&format!("listener ready on {listen}")); + let shutdown = runtime.shutdown().clone(); loop { - match listener.accept().await { - Ok((stream, peer)) => serve_accepted(runtime.clone(), stream, peer.to_string()), - Err(error) => accept_failed(error).await, + tokio::select! { + _ = shutdown.cancelled() => return DaemonExit::Shutdown, + accepted = listener.accept() => match accepted { + Ok((stream, peer)) => serve_accepted(runtime.clone(), stream, peer.to_string()), + Err(error) => accept_failed(error).await, + }, } } } #[cfg(unix)] -async fn run_unix_listener(runtime: DaemonRuntime, path: std::path::PathBuf) -> anyhow::Result<()> { - let listener = bind_unix_listener(&path)?; +async fn run_unix_listener(runtime: DaemonRuntime, path: std::path::PathBuf) -> DaemonExit { + let listener = match bind_unix_listener(&path) { + Ok(listener) => listener, + Err(error) => return DaemonExit::Configuration(error), + }; let label = ListenAddr::Unix(path).to_string(); tracing_line(&format!("listener ready on {label}")); + let shutdown = runtime.shutdown().clone(); loop { - match listener.accept().await { - Ok((stream, _)) => serve_accepted(runtime.clone(), stream, label.clone()), - Err(error) => accept_failed(error).await, + tokio::select! { + _ = shutdown.cancelled() => return DaemonExit::Shutdown, + accepted = listener.accept() => match accepted { + Ok((stream, _)) => serve_accepted(runtime.clone(), stream, label.clone()), + Err(error) => accept_failed(error).await, + }, } } } @@ -100,7 +212,7 @@ async fn run_unix_listener(runtime: DaemonRuntime, path: std::path::PathBuf) -> /// The pause stops a persistent error from spinning the loop. async fn accept_failed(error: std::io::Error) { tracing_line(&format!("accept failed: {error}")); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(100)).await; } /// Bind a Unix-socket listener only this daemon's user can connect to. A @@ -154,465 +266,840 @@ where S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, { tokio::spawn(async move { - match accept_async(stream).await { - Ok(socket) => { - if let Err(error) = run_data_connection(&runtime, socket).await { - eprintln!("lightspeed-envd connection {peer} failed: {error:#}"); - } - } - Err(error) => { - eprintln!("lightspeed-envd websocket handshake {peer} failed: {error}") - } + if let Err(error) = serve_stream(runtime, stream).await { + tracing_line(&format!("connection {peer} failed: {error:#}")); } }); } +/// Accept a WebSocket upgrade on an already connected byte stream and serve +/// the data plane on it until the connection ends. In-process callers pair +/// it with `DataClient::connect_initialized_over` on the other end of a +/// `tokio::io::duplex`. +pub async fn serve_stream(runtime: DaemonRuntime, stream: S) -> anyhow::Result<()> +where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + let socket = accept_async_with_config(stream, Some(websocket_config())) + .await + .context("websocket handshake")?; + run_data_connection(&runtime, socket).await; + Ok(()) +} + +/// One frame for the writer task: an encoded response and the hook to run +/// once it is on the socket. +struct Outgoing { + text: String, + on_written: Option, +} + +impl Outgoing { + fn response(response: Response) -> Self { + Self { + text: response.to_text(), + on_written: None, + } + } +} + +/// Pending requests of one connection by id, each with the token that +/// `$/cancelRequest` and the connection's close cancel. +type PendingRequests = Arc>>; + /// Serve the data protocol on one accepted or dialed socket. Outbound data /// sockets and passive connections run this same loop. -pub(crate) async fn run_data_connection( - runtime: &DaemonRuntime, - socket: tokio_tungstenite::WebSocketStream, -) -> anyhow::Result<()> +pub(crate) async fn run_data_connection(runtime: &DaemonRuntime, socket: WebSocketStream) where S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, { - let (mut writer, mut reader) = socket.split(); - let (responses, mut response_rx) = mpsc::channel::(64); - let concurrency = std::sync::Arc::new(Semaphore::new(32)); + let connection = Arc::new(ConnectionInfo { + connection_id: runtime.next_connection_id(), + closed: CancellationToken::new(), + }); + let max_concurrent = runtime.limits().max_concurrent_requests as usize; + // Handler replies are bounded by the request permits; the rest of the + // capacity holds the reader's own answers (framing errors, busy). A + // peer that fills it without reading is closed rather than buffered. + let (outgoing, outgoing_rx) = mpsc::channel::(max_concurrent + 64); + let (sink, mut stream) = socket.split(); + let writer = tokio::spawn(write_loop(sink, outgoing_rx, connection.closed.clone())); + let permits = Arc::new(Semaphore::new(max_concurrent)); + let pending: PendingRequests = Arc::default(); + let shutdown = runtime.shutdown().clone(); + let mut session = Session { + runtime, + connection: &connection, + outgoing: &outgoing, + permits: &permits, + pending: &pending, + initialized: false, + }; loop { - tokio::select! { - response = response_rx.recv() => { - let Some(response) = response else { return Ok(()) }; - writer.send(Message::Text(response.to_string().into())).await?; - } - message = reader.next() => { - let Some(message) = message else { return Ok(()) }; - match message? { - Message::Text(text) => dispatch_gateway_json(runtime, &responses, &concurrency, &text).await?, - Message::Binary(bytes) => { - let text = std::str::from_utf8(&bytes).context("binary provider frame is not UTF-8")?; - dispatch_gateway_json(runtime, &responses, &concurrency, text).await?; - } - Message::Ping(bytes) => writer.send(Message::Pong(bytes)).await?, - Message::Pong(_) | Message::Frame(_) => {} - Message::Close(_) => { - // Tungstenite queues the close reply while reading the - // peer's frame; flush it before dropping the socket. - writer.flush().await?; - return Ok(()) - }, - } + let frame = tokio::select! { + _ = connection.closed.cancelled() => break, + _ = shutdown.cancelled() => break, + frame = stream.next() => frame, + }; + let message = match frame { + Some(Ok(message)) => message, + Some(Err(error)) => { + tracing_line(&format!( + "connection {} ended: {error}", + connection.connection_id + )); + break; } + None => break, + }; + let keep_open = match message { + Message::Text(text) => session.on_text(text.as_str()), + Message::Binary(_) => session.answer(Response::failure( + None, + ProtocolError::invalid_request("binary frames are not part of the protocol"), + )), + // Tungstenite answers pings itself on the read path, ahead of + // anything the writer has queued. + Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => true, + Message::Close(_) => break, + }; + if !keep_open { + break; } } + connection.closed.cancel(); + pending + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .clear(); + drop(outgoing); + let _ = writer.await; } -use crate::{ - DaemonRuntime, - config::ListenAddr, - rpc::{ - decode_params, encode_result, error_response, invalid_request, method_not_found, - parse_request, success_response, - }, -}; +/// The reader's view of one connection. +struct Session<'a> { + runtime: &'a DaemonRuntime, + connection: &'a Arc, + outgoing: &'a mpsc::Sender, + permits: &'a Arc, + pending: &'a PendingRequests, + initialized: bool, +} -async fn dispatch_gateway_json( - runtime: &DaemonRuntime, - responses: &mpsc::Sender, - concurrency: &std::sync::Arc, - text: &str, -) -> anyhow::Result<()> { - let value: Value = serde_json::from_str(text)?; - let request = match parse_request(value) { - Ok(request) => request, - Err(error) => { - responses.send(error_response(None, error)).await?; - return Ok(()); +impl Session<'_> { + /// Handle one text frame; false closes the connection. + fn on_text(&mut self, text: &str) -> bool { + match jsonrpc::parse(text) { + Err(rejection) => match rejection.into_response() { + Some(response) => self.answer(response), + None => true, + }, + Ok(RpcMessage::Request(request)) => self.on_request(request), + Ok(RpcMessage::Notification(notification)) => { + if self.initialized + && notification.method == CANCEL_REQUEST_METHOD + && let Ok(params) = notification.params::() + { + self.cancel(¶ms.id); + } + true + } + // envd never sends requests, so no response can be one it + // waits for. + Ok(RpcMessage::Response(response)) => { + tracing_line(&format!( + "connection {} ignored a response to id {}", + self.connection.connection_id, + response + .id + .map_or_else(|| "null".to_owned(), |id| id.to_string()) + )); + true + } } - }; - let Some(id) = request.id else { - if request.method.as_deref() == Some(INITIALIZED_METHOD) { - let _ = decode_params::(request.params); + } + + fn on_request(&mut self, request: Request) -> bool { + if !self.initialized { + if request.method != DataInitialize::NAME { + return self.answer(Response::failure( + Some(request.id), + ProtocolError::not_initialized(), + )); + } + let result = initialize(self.runtime, &request); + self.initialized = result.is_ok(); + return self.answer(Response::from_result(request.id, result)); } - return Ok(()); - }; - let Ok(permit) = concurrency.clone().try_acquire_owned() else { - responses - .send(error_response( - Some(id), - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Conflict, - "daemon request limit reached", + if request.method == DataInitialize::NAME { + return self.answer(Response::failure( + Some(request.id), + ProtocolError::invalid_request( + "initialize was already answered on this connection", ), - )) - .await?; - return Ok(()); - }; - let runtime = runtime.clone(); - let responses = responses.clone(); - tokio::spawn(async move { - let _permit = permit; - let result = match request.method.as_deref() { - Some(method) => handle_data(&runtime, method, request.params).await, - None => Err(invalid_request("request missing method")), - }; - let response = match result { - Ok(result) => success_response(id, result), - Err(error) => error_response(Some(id), error), - }; - let _ = responses.send(response).await; - }); - Ok(()) -} - -async fn handle_data( - runtime: &DaemonRuntime, - method: &str, - params: Value, -) -> Result { - // Handshakes and idle reports are observation, not use: only real - // filesystem/process/job requests count as activity. - if !matches!(method, DATA_INITIALIZE_METHOD | ENV_IDLE_METHOD) { - runtime.touch_activity(); - } - match method { - ENV_IDLE_METHOD => { - decode_params::(params)?; - encode_result(runtime.idle_report().await) + )); } - DATA_INITIALIZE_METHOD => { - let params = decode_params::(params)?; - if params.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Unsupported, + let Ok(permit) = self.permits.clone().try_acquire_owned() else { + return self.answer(Response::failure( + Some(request.id), + ProtocolError::busy( + Some(BusyReason::RequestLimit), + None, format!( - "unsupported data protocol version {}; expected {CURRENT_PROTOCOL_VERSION}", - params.protocol_version + "{} requests are already in flight on this connection", + self.runtime.limits().max_concurrent_requests ), + ), + )); + }; + let ctx = RequestContext::new(self.connection.clone()); + { + let mut pending = self + .pending + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if pending.contains_key(&request.id) { + drop(pending); + return self.answer(Response::failure( + Some(request.id), + ProtocolError::invalid_request("a request with this id is already in flight"), )); } - encode_result(InitializeResponse { - home_directory: std::env::var("HOME").ok(), - protocol_version: CURRENT_PROTOCOL_VERSION, - connection_id: runtime.next_connection_id(), - capabilities: runtime.capabilities(), - default_cwd: Some(runtime.config().cwd.to_string_lossy().into_owned()), - implementation: runtime.implementation(), - }) + pending.insert(request.id.clone(), ctx.cancel.clone()); } - FS_READ_FILE_METHOD => encode_result( - runtime - .filesystem() - .read_file(decode_params::(params)?) - .await?, - ), - FS_WRITE_FILE_METHOD => encode_result( - runtime - .filesystem() - .write_file(decode_params::(params)?) - .await?, - ), - FS_CREATE_DIRECTORY_METHOD => encode_result( - runtime - .filesystem() - .create_directory(decode_params::(params)?) - .await?, - ), - FS_GET_METADATA_METHOD => encode_result( - runtime - .filesystem() - .get_metadata(decode_params::(params)?) - .await?, - ), - FS_READ_DIRECTORY_METHOD => encode_result( - runtime - .filesystem() - .read_directory(decode_params::(params)?) - .await?, - ), - FS_REMOVE_METHOD => encode_result( - runtime - .filesystem() - .remove(decode_params::(params)?) - .await?, - ), - environment_protocol::data::methods::FS_SCAN_METHOD => { - encode_result(runtime.filesystem().scan(decode_params(params)?).await?) + let runtime = self.runtime.clone(); + let outgoing = self.outgoing.clone(); + let pending = self.pending.clone(); + tokio::spawn(async move { + let _permit = permit; + let Request { id, method, params } = request; + let result = dispatch::handle_request(&runtime, &ctx, &method, params).await; + pending + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .remove(&id); + if ctx.connection.closed.is_cancelled() { + return; + } + let _ = outgoing.send(encode_reply(id, result)).await; + }); + true + } + + fn cancel(&self, id: &Id) { + let pending = self + .pending + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if let Some(token) = pending.get(id) { + token.cancel(); } - environment_protocol::data::methods::FS_TRANSFER_METHOD => encode_result( - runtime - .filesystem() - .transfer(decode_params(params)?) - .await?, - ), - FS_CAPTURE_METHOD => encode_result( - runtime - .filesystem() - .capture(decode_params::(params)?) - .await?, - ), - FS_MATERIALIZE_METHOD => encode_result( - runtime - .filesystem() - .materialize(decode_params::(params)?) - .await?, - ), - FS_COPY_METHOD => encode_result( - runtime - .filesystem() - .copy(decode_params::(params)?) - .await?, - ), - FS_SEARCH_TEXT_METHOD => encode_result( - runtime - .filesystem() - .search_text(decode_params::(params)?) - .await?, - ), - FS_GLOB_FILES_METHOD => encode_result( - runtime - .filesystem() - .glob_files(decode_params::(params)?) - .await?, - ), - PROCESS_START_METHOD => encode_result( - runtime - .processes() - .start_process(decode_params::(params)?) - .await?, - ), - PROCESS_READ_METHOD => encode_result( - runtime - .processes() - .read_process(decode_params::(params)?) - .await?, - ), - PROCESS_WRITE_METHOD => encode_result( - runtime - .processes() - .write_process(decode_params::(params)?) - .await?, - ), - PROCESS_TERMINATE_METHOD => encode_result( - runtime - .processes() - .terminate_process(decode_params::(params)?) - .await?, - ), - PROCESS_RESIZE_METHOD => encode_result( - runtime - .processes() - .resize_process(decode_params::(params)?) - .await?, - ), - JOB_START_METHOD => encode_result( - runtime - .jobs() - .start_jobs(decode_params::(params)?) - .await?, - ), - JOB_LIST_METHOD => encode_result( - runtime - .jobs() - .list_jobs(decode_params::(params)?) - .await?, - ), - JOB_READ_METHOD => encode_result( - runtime - .jobs() - .read_jobs(decode_params::(params)?) - .await?, - ), - JOB_CANCEL_METHOD => encode_result( - runtime - .jobs() - .cancel_jobs(decode_params::(params)?) - .await?, - ), - other => Err(method_not_found(other)), } + + /// Queue a reader-produced answer without waiting; false when the peer + /// stopped reading and the queue is full. + fn answer(&self, response: Response) -> bool { + match self.outgoing.try_send(Outgoing::response(response)) { + Ok(()) => true, + Err(mpsc::error::TrySendError::Full(_)) => { + tracing_line(&format!( + "connection {} closed: the peer does not read its responses", + self.connection.connection_id + )); + false + } + Err(mpsc::error::TrySendError::Closed(_)) => false, + } + } +} + +/// Encode a handler's result. A success whose encoding would break the +/// message limit becomes `payloadTooLarge`, and its after-write hook is +/// dropped with it: nothing the client never received may be committed. +fn encode_reply( + id: Id, + result: Result, ProtocolError>, +) -> Outgoing { + match result { + Ok(reply) => { + let response = Response { + id: Some(id), + result: Ok(reply.value), + }; + let text = response.to_value().to_string(); + if jsonrpc::check_message_size(&text).is_ok() { + Outgoing { + text, + on_written: reply.on_written, + } + } else { + Outgoing::response(response) + } + } + Err(error) => Outgoing::response(Response::failure(Some(id), error)), + } +} + +/// `initialize`: the version is compared before the strict decode, so a +/// client of another version learns that rather than which field it sent +/// that this daemon does not know. +fn initialize( + runtime: &DaemonRuntime, + request: &Request, +) -> Result { + if let Some(received) = request + .params + .get("protocolVersion") + .and_then(serde_json::Value::as_u64) + && received != u64::from(ENVD_PROTOCOL_VERSION) + { + return Err(ProtocolError::version_mismatch( + ENVD_PROTOCOL_VERSION, + u32::try_from(received).unwrap_or(u32::MAX), + )); + } + let _params: environment_protocol::data::handshake::InitializeParams = request.params()?; + Ok(runtime.initialize_response()) +} + +/// The connection's only writer: sends frames in the order handlers finish +/// and runs each after-write hook once its frame is on the socket. Stops, +/// dropping whatever is still queued, when the connection closes. +async fn write_loop( + mut sink: Si, + mut queue: mpsc::Receiver, + closed: CancellationToken, +) where + Si: Sink + Unpin, +{ + loop { + let next = tokio::select! { + biased; + _ = closed.cancelled() => break, + next = queue.recv() => next, + }; + let Some(Outgoing { text, on_written }) = next else { + break; + }; + let sent = tokio::select! { + biased; + _ = closed.cancelled() => break, + sent = sink.send(Message::Text(text.into())) => sent, + }; + match sent { + Ok(()) => { + if let Some(hook) = on_written { + hook(); + } + } + Err(_) => { + closed.cancel(); + break; + } + } + } + // Sends our close frame, or the reply tungstenite queued for the peer's. + let _ = tokio::time::timeout(Duration::from_secs(5), sink.close()).await; } pub(crate) fn tracing_line(message: &str) { eprintln!("lightspeed-envd {message}"); } -#[cfg(all(test, unix))] -mod unix_listener_tests { - use std::os::unix::fs::PermissionsExt as _; +#[cfg(test)] +mod tests { + use std::time::Duration; - use environment_protocol::{data::handshake::InitializeParams, shared::EnvironmentScope}; - use tokio_tungstenite::client_async; + use environment_protocol::{ + error::{ErrorData, ProtocolErrorKind}, + handshake::ImplementationInfo, + }; + use serde_json::{Value, json}; + use tokio::io::DuplexStream; + use tokio_tungstenite::client_async_with_config; use super::*; - use crate::config::DaemonConfig; - - fn runtime(root: &std::path::Path, socket: &std::path::Path) -> DaemonRuntime { - DaemonRuntime::new(DaemonConfig { - listen: Some(ListenAddr::Unix(socket.to_path_buf())), - cwd: root.to_path_buf(), - fs_root: root.to_path_buf(), - state_dir: root.join("state"), - read_only_fs: false, - registration: None, - scrubbed_env: vec![], - }) - .unwrap() + use crate::test_support::runtime; + + type Client = WebSocketStream; + + async fn connect(runtime: &DaemonRuntime) -> Client { + let (client, server) = tokio::io::duplex(1 << 20); + tokio::spawn(serve_stream(runtime.clone(), server)); + client_async_with_config("ws://localhost/", client, Some(websocket_config())) + .await + .expect("upgrade") + .0 } - #[tokio::test(flavor = "current_thread")] - async fn serves_the_data_protocol_on_a_private_unix_socket() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - // The parent does not exist yet: guest tmpfs paths start empty. - let socket = root.join("run").join("envd.sock"); - let server = tokio::spawn(run(runtime(&root, &socket))); - while !socket.exists() { - tokio::task::yield_now().await; - } - let mode = std::fs::metadata(&socket).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o600); - - let stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); - let (mut client, _) = client_async("ws://envd/", stream).await.unwrap(); - let params = InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "unix-listener-test".to_owned(), - scope: EnvironmentScope::Default, - resume_connection_id: None, - }; - let request = serde_json::json!({ - "jsonrpc": "2.0", - "id": 1, - "method": DATA_INITIALIZE_METHOD, - "params": params, - }); + async fn send(client: &mut Client, frame: Value) { client - .send(Message::Text(request.to_string().into())) + .send(Message::Text(frame.to_string().into())) .await - .unwrap(); - let Some(Ok(Message::Text(reply))) = client.next().await else { - panic!("expected an initialize response"); - }; - let reply: Value = serde_json::from_str(reply.as_str()).unwrap(); - assert_eq!(reply["id"], 1); - assert_eq!(reply["result"]["protocolVersion"], CURRENT_PROTOCOL_VERSION); - server.abort(); + .expect("send"); } - #[tokio::test(flavor = "current_thread")] - async fn stale_sockets_are_replaced_but_live_ones_and_files_are_not() { - let temp = tempfile::tempdir().unwrap(); - let socket = temp.path().join("envd.sock"); + async fn next_json(client: &mut Client) -> Value { + loop { + let frame = tokio::time::timeout(Duration::from_secs(10), client.next()) + .await + .expect("a frame in time") + .expect("open") + .expect("frame"); + if let Message::Text(text) = frame { + return serde_json::from_str(text.as_str()).expect("json"); + } + } + } - drop(std::os::unix::net::UnixListener::bind(&socket).unwrap()); - assert!(socket.exists(), "a dropped listener leaves its node behind"); - let listener = bind_unix_listener(&socket).expect("stale socket replaced"); + fn error_kind(response: &Value) -> ProtocolErrorKind { + let error: ProtocolError = + serde_json::from_value(response["error"].clone()).expect("an error response"); + error.kind() + } - assert!(bind_unix_listener(&socket).is_err(), "live listener kept"); - drop(listener); + fn initialize_params(version: u32) -> Value { + json!({ + "protocolVersion": version, + "clientInfo": {"name": "envd-server-test"}, + }) + } - let file = temp.path().join("not-a-socket"); - std::fs::write(&file, b"data").unwrap(); - assert!(bind_unix_listener(&file).is_err()); - assert_eq!(std::fs::read(&file).unwrap(), b"data"); + async fn initialized(runtime: &DaemonRuntime) -> Client { + let mut client = connect(runtime).await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": initialize_params(ENVD_PROTOCOL_VERSION)}), + ) + .await; + let response = next_json(&mut client).await; + assert_eq!(response["result"]["plane"], "data", "{response}"); + client } #[tokio::test(flavor = "current_thread")] - async fn a_live_socket_that_cannot_be_probed_is_left_in_place() { - let temp = tempfile::tempdir().unwrap(); - let socket = temp.path().join("envd.sock"); - let _live = std::os::unix::net::UnixListener::bind(&socket).unwrap(); - // Without write permission, connect fails with EACCES rather than - // ECONNREFUSED (root bypasses this and sees the live listener). - std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o000)).unwrap(); - - assert!(bind_unix_listener(&socket).is_err()); - assert!( - std::fs::symlink_metadata(&socket).is_ok(), - "live socket kept" + async fn the_handshake_is_enforced_and_the_connection_survives_every_refusal() { + let (_temp, runtime) = runtime(); + let mut client = connect(&runtime).await; + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 1, "method": "env/getIdle"}), + ) + .await; + let early = next_json(&mut client).await; + assert_eq!(early["id"], 1); + assert_eq!(error_kind(&early), ProtocolErrorKind::NotInitialized); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 2, "method": "initialize", "params": {"protocolVersion": 4, "clientInfo": {"name": "x"}, "fromTheFuture": true}}), + ) + .await; + let mismatch: ProtocolError = + serde_json::from_value(next_json(&mut client).await["error"].clone()).unwrap(); + assert_eq!( + mismatch.data, + ErrorData::VersionMismatch { + expected: ENVD_PROTOCOL_VERSION, + received: 4 + } ); - } -} -#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] -mod transfer_tests { - use super::*; - use crate::config::DaemonConfig; - use environment_protocol::shared::{ByteChunk, EnvironmentPath}; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 3, "method": "initialize", "params": initialize_params(ENVD_PROTOCOL_VERSION)}), + ) + .await; + let response = next_json(&mut client).await; + assert_eq!(response["id"], 3); + assert_eq!(response["result"]["protocolVersion"], ENVD_PROTOCOL_VERSION); + assert_eq!( + response["result"]["daemonInstanceId"], + runtime.instance_id().as_str() + ); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 4, "method": "initialize", "params": initialize_params(ENVD_PROTOCOL_VERSION)}), + ) + .await; + assert_eq!( + error_kind(&next_json(&mut client).await), + ProtocolErrorKind::InvalidRequest + ); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 5, "method": "env/getIdle"}), + ) + .await; + let idle = next_json(&mut client).await; + assert!(idle["result"]["idleForMs"].is_u64(), "{idle}"); + } #[tokio::test(flavor = "current_thread")] - async fn transfer_dispatch_and_read_only_capabilities() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().to_path_buf(); - for read_only in [false, true] { - let runtime = DaemonRuntime::new(DaemonConfig { - listen: None, - cwd: root.clone(), - fs_root: root.clone(), - state_dir: root.join("state"), - read_only_fs: read_only, - registration: None, - scrubbed_env: vec![], - }) + async fn framing_errors_are_answered_with_null_ids_and_notifications_never() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + + client.send(Message::Text("not json".into())).await.unwrap(); + let parse = next_json(&mut client).await; + assert_eq!(parse["id"], Value::Null); + assert_eq!(error_kind(&parse), ProtocolErrorKind::ParseError); + + client + .send(Message::Binary(b"{}".to_vec().into())) + .await .unwrap(); - assert!(runtime.capabilities().filesystem_capture); - assert_eq!(runtime.capabilities().filesystem_materialize, !read_only); - let limits = TransferLimits { - max_entries: 1, - max_depth: 0, - max_file_bytes: 2, - max_total_bytes: 2, - // This checks dispatch and access, not storage latency. Allow - // shared-runner delays across the transfer's journal fsyncs. - max_duration_ms: MAX_TRANSFER_DURATION_MS, - }; - let entries = vec![TransferEntry { - path: "".into(), - content: TransferContent::File { - data: ByteChunk::from(vec![0, 255]), - executable: true, - }, - }]; - let request = MaterializeParams { - destination: EnvironmentPath::new("binary").unwrap(), - entries: entries.clone(), - limits, - on_existing: TransferOnExisting::Replace, - }; - let result = handle_data( + let binary = next_json(&mut client).await; + assert_eq!(binary["id"], Value::Null); + assert_eq!(error_kind(&binary), ProtocolErrorKind::InvalidRequest); + + // A malformed notification and an unknown one: no answer at all. + send( + &mut client, + json!({"jsonrpc": "2.0", "method": "$/cancelRequest", "params": [1]}), + ) + .await; + send( + &mut client, + json!({"jsonrpc": "2.0", "method": "x/unknown"}), + ) + .await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "after", "method": "nope/nothing"}), + ) + .await; + let next = next_json(&mut client).await; + assert_eq!(next["id"], "after"); + assert_eq!(error_kind(&next), ProtocolErrorKind::MethodNotFound); + } + + #[tokio::test(flavor = "current_thread")] + async fn params_are_decoded_strictly_and_name_the_field() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 9, "method": "env/getIdle", "params": {"unexpected": 1}}), + ) + .await; + let response = next_json(&mut client).await; + assert_eq!(error_kind(&response), ProtocolErrorKind::InvalidParams); + } + + /// Every data request in the manifest reaches a handler: none falls + /// through to `methodNotFound`. + #[tokio::test(flavor = "current_thread")] + async fn every_manifest_request_is_dispatched() { + use environment_protocol::{ + handshake::Plane, + manifest::{self, Direction, MessageKind}, + }; + let (_temp, runtime) = runtime(); + for spec in manifest::plane_methods(Plane::Data).filter(|spec| { + spec.kind == MessageKind::Request && spec.direction == Direction::ClientToServer + }) { + let result = dispatch::handle_request( &runtime, - FS_MATERIALIZE_METHOD, - serde_json::to_value(request).unwrap(), + &RequestContext::detached(), + spec.name, + json!({"__unknownMember": true}), ) .await; - if read_only { - assert_eq!( - result.unwrap_err().code, - EnvironmentProtocolErrorCode::CapabilityUnavailable + if let Err(error) = result { + assert_ne!( + error.kind(), + ProtocolErrorKind::MethodNotFound, + "{} is not dispatched", + spec.name ); - } else { - assert_eq!(result.unwrap()["bytes"], 2); } - let captured: CaptureResponse = serde_json::from_value( - handle_data( - &runtime, - FS_CAPTURE_METHOD, - serde_json::to_value(CaptureParams { - source: EnvironmentPath::new("binary").unwrap(), - limits, - }) - .unwrap(), - ) - .await - .unwrap(), + } + } + + #[tokio::test(flavor = "current_thread")] + async fn requests_are_pipelined_and_abortable_waits_are_cancelled() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "start", "method": "process/start", "params": { + "processId": "quiet", + "command": {"argv": ["/bin/sh", "-c", "sleep 30"], "timeoutMs": 60000}, + }}), + ) + .await; + assert!(next_json(&mut client).await["result"]["pid"].is_u64()); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "wait", "method": "process/read", "params": { + "processId": "quiet", "maxBytes": 1024, "waitMs": 30000, "returnOn": "exit", + }}), + ) + .await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "idle", "method": "env/getIdle"}), + ) + .await; + let first = next_json(&mut client).await; + assert_eq!( + first["id"], "idle", + "a later request answers first: {first}" + ); + + send( + &mut client, + json!({"jsonrpc": "2.0", "method": "$/cancelRequest", "params": {"id": "wait"}}), + ) + .await; + let cancelled = next_json(&mut client).await; + assert_eq!(cancelled["id"], "wait"); + let error: ProtocolError = serde_json::from_value(cancelled["error"].clone()).unwrap(); + assert_eq!(error, ProtocolError::cancelled()); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "kill", "method": "process/terminate", "params": {"processId": "quiet", "signal": "kill"}}), + ) + .await; + assert_eq!(next_json(&mut client).await["id"], "kill"); + } + + #[tokio::test(flavor = "current_thread")] + async fn requests_beyond_the_concurrency_ceiling_are_busy() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "start", "method": "process/start", "params": { + "processId": "quiet", + "command": {"argv": ["/bin/sh", "-c", "sleep 30"], "timeoutMs": 60000}, + }}), + ) + .await; + next_json(&mut client).await; + let ceiling = runtime.limits().max_concurrent_requests; + for index in 0..=ceiling { + send( + &mut client, + json!({"jsonrpc": "2.0", "id": index, "method": "process/read", "params": { + "processId": "quiet", "after": 0, "maxBytes": 0, "waitMs": 30000, "returnOn": "exit", + }}), ) - .unwrap(); - assert_eq!(captured.entries, entries); + .await; } + let busy = next_json(&mut client).await; + assert_eq!(busy["id"], ceiling); + let error: ProtocolError = serde_json::from_value(busy["error"].clone()).unwrap(); + assert!(matches!( + error.data, + ErrorData::Busy { + reason: Some(BusyReason::RequestLimit), + .. + } + )); + send( + &mut client, + json!({"jsonrpc": "2.0", "method": "$/cancelRequest", "params": {"id": 0}}), + ) + .await; + assert_eq!(next_json(&mut client).await["id"], 0); + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "kill", "method": "process/terminate", "params": {"processId": "quiet", "signal": "kill"}}), + ) + .await; } -} -#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] -#[path = "filesystem/transfer/tests.rs"] -mod streaming_transfer_tests; + /// A cursor read whose connection drops is cancelled and consumes + /// nothing: the next connection's cursor read still sees every byte. + #[tokio::test(flavor = "current_thread")] + async fn a_dropped_connection_consumes_no_output() { + let (_temp, runtime) = runtime(); + let mut first = initialized(&runtime).await; + send( + &mut first, + json!({"jsonrpc": "2.0", "id": 1, "method": "process/start", "params": { + "processId": "late", + "command": {"argv": ["/bin/sh", "-c", "printf early; sleep 1; printf late"], "timeoutMs": 60000}, + }}), + ) + .await; + next_json(&mut first).await; + send( + &mut first, + json!({"jsonrpc": "2.0", "id": 2, "method": "process/read", "params": { + "processId": "late", "maxBytes": 1024, "waitMs": 30000, "returnOn": "exit", + }}), + ) + .await; + tokio::time::sleep(Duration::from_millis(200)).await; + drop(first); + + tokio::time::sleep(Duration::from_millis(1500)).await; + let mut second = initialized(&runtime).await; + send( + &mut second, + json!({"jsonrpc": "2.0", "id": 3, "method": "process/read", "params": { + "processId": "late", "maxBytes": 1024, "waitMs": 10000, "returnOn": "exit", + }}), + ) + .await; + let page = next_json(&mut second).await; + let text: String = page["result"]["output"]["chunks"] + .as_array() + .expect("chunks") + .iter() + .map(|chunk| { + let bytes: environment_protocol::bytes::ByteChunk = + serde_json::from_value(chunk["data"].clone()).unwrap(); + String::from_utf8(bytes.into_inner()).unwrap() + }) + .collect(); + assert_eq!(text, "earlylate", "{page}"); + } + + #[tokio::test(flavor = "current_thread")] + async fn the_client_library_completes_the_handshake() { + let (_temp, runtime) = runtime(); + let (client_io, server_io) = tokio::io::duplex(1 << 20); + tokio::spawn(serve_stream(runtime.clone(), server_io)); + let options = environment_client::ConnectOptions::new( + ImplementationInfo::new("envd-server-test").unwrap(), + ); + let (client, initialize) = + environment_client::DataClient::connect_initialized_over(client_io, options) + .await + .expect("initialized"); + assert_eq!(&initialize.daemon_instance_id, runtime.instance_id()); + client + .get_idle(&environment_protocol::data::idle::GetIdleParams {}) + .await + .expect("idle"); + } + + #[tokio::test(flavor = "current_thread")] + async fn shutdown_ends_listeners_and_connections() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + runtime.shutdown().cancel(); + let ended = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match client.next().await { + None | Some(Err(_)) | Some(Ok(Message::Close(_))) => break, + Some(Ok(_)) => continue, + } + } + }) + .await; + assert!(ended.is_ok(), "the daemon closed the connection"); + } + + #[cfg(unix)] + mod unix_listener { + use std::os::unix::fs::PermissionsExt as _; + + use super::*; + use crate::test_support::runtime_with; + + #[tokio::test(flavor = "current_thread")] + async fn serves_the_data_protocol_on_a_private_unix_socket() { + let socket_dir = tempfile::tempdir().unwrap(); + // The parent does not exist yet: guest tmpfs paths start empty. + let socket = socket_dir.path().join("run").join("envd.sock"); + let (_temp, runtime) = runtime_with(|config| { + config.listen = Some(ListenAddr::Unix(socket.clone())); + }); + let server = tokio::spawn(run(runtime.clone())); + while !socket.exists() { + tokio::task::yield_now().await; + } + let mode = std::fs::metadata(&socket).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + + let stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); + let (mut client, _) = + client_async_with_config("ws://envd/", stream, Some(websocket_config())) + .await + .unwrap(); + client + .send(Message::Text( + json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": initialize_params(ENVD_PROTOCOL_VERSION)}) + .to_string() + .into(), + )) + .await + .unwrap(); + let Some(Ok(Message::Text(reply))) = client.next().await else { + panic!("expected an initialize response"); + }; + let reply: Value = serde_json::from_str(reply.as_str()).unwrap(); + assert_eq!(reply["id"], 1); + assert_eq!(reply["result"]["protocolVersion"], ENVD_PROTOCOL_VERSION); + + runtime.shutdown().cancel(); + let exit = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("listener stops") + .unwrap(); + assert_eq!(exit.status(), 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn stale_sockets_are_replaced_but_live_ones_and_files_are_not() { + let temp = tempfile::tempdir().unwrap(); + let socket = temp.path().join("envd.sock"); + + drop(std::os::unix::net::UnixListener::bind(&socket).unwrap()); + assert!(socket.exists(), "a dropped listener leaves its node behind"); + let listener = bind_unix_listener(&socket).expect("stale socket replaced"); + + assert!(bind_unix_listener(&socket).is_err(), "live listener kept"); + drop(listener); + + let file = temp.path().join("not-a-socket"); + std::fs::write(&file, b"data").unwrap(); + assert!(bind_unix_listener(&file).is_err()); + assert_eq!(std::fs::read(&file).unwrap(), b"data"); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_live_socket_that_cannot_be_probed_is_left_in_place() { + let temp = tempfile::tempdir().unwrap(); + let socket = temp.path().join("envd.sock"); + let _live = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + // Without write permission, connect fails with EACCES rather than + // ECONNREFUSED (root bypasses this and sees the live listener). + std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o000)).unwrap(); + + assert!(bind_unix_listener(&socket).is_err()); + assert!( + std::fs::symlink_metadata(&socket).is_ok(), + "live socket kept" + ); + } + } + + #[test] + fn exit_statuses_are_distinct() { + let statuses = [ + DaemonExit::Shutdown.status(), + DaemonExit::Failed(anyhow::anyhow!("x")).status(), + DaemonExit::Configuration(anyhow::anyhow!("x")).status(), + DaemonExit::Refused("x".to_owned()).status(), + DaemonExit::EnvironmentClosed.status(), + DaemonExit::Superseded { + daemon_public_key: "k".to_owned(), + } + .status(), + ]; + let unique: std::collections::BTreeSet<_> = statuses.iter().collect(); + assert_eq!(unique.len(), statuses.len()); + assert_eq!(statuses, [0, 1, 2, 3, 4, 5]); + } +} diff --git a/crates/environment-daemon/src/upgrade.rs b/crates/environment-daemon/src/upgrade.rs index 013539645..331dcfa30 100644 --- a/crates/environment-daemon/src/upgrade.rs +++ b/crates/environment-daemon/src/upgrade.rs @@ -6,21 +6,22 @@ //! before it can replace the running executable. use std::{ - collections::BTreeMap, io::{Read as _, Seek as _, Write as _}, net::IpAddr, path::{Path, PathBuf}, time::Duration, }; -use anyhow::{Context as _, Result, anyhow, bail}; +use anyhow::{Context as _, anyhow}; +use environment_protocol::{ + bootstrap::{ArtifactSha256, BuildFacts, DiscoveryArtifact, DiscoveryDocument}, + routes::{DISCOVERY_PATH, canonical_origin}, +}; use flate2::read::GzDecoder; use futures_util::StreamExt as _; use reqwest::Url; -use serde::Deserialize; use sha2::{Digest as _, Sha256}; -const DISCOVERY_PATH: &str = "/.well-known/lightspeed-envd"; const MAX_DISCOVERY_BYTES: usize = 1_048_576; const MAX_ARCHIVE_BYTES: u64 = 256 * 1024 * 1024; const MAX_BINARY_BYTES: u64 = 256 * 1024 * 1024; @@ -36,6 +37,10 @@ pub struct UpgradeRequest { /// challenge. Manual upgrades install whichever protocol the deployment /// currently publishes. pub expected_protocol: Option, + /// A protocol that installing would not change, refused as terminal: + /// an automatic upgrade after an unreadable first frame installs any + /// protocol but the daemon's own. + pub other_than: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -46,38 +51,61 @@ pub struct InstalledBuild { pub path: PathBuf, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct DiscoveryDocument { - version: String, - git_sha: String, - protocol_version: u32, - artifacts: BTreeMap, +/// Why an upgrade did not install anything. +#[derive(Debug, thiserror::Error)] +pub enum UpgradeError { + /// A network failure, timeout, HTTP 5xx or 429 while fetching the + /// discovery document or the archive: retry later. + #[error("{0:#}")] + Transient(anyhow::Error), + /// Everything else: an integrity failure (checksum, candidate build + /// facts, a discovery protocol other than the challenge's), a document + /// without this target, or a local install failure. Retrying cannot + /// help. + #[error("{0:#}")] + Terminal(anyhow::Error), } -#[derive(Debug, Deserialize)] -struct DiscoveryArtifact { - file: String, - sha256: String, - url: Option, +impl UpgradeError { + pub fn is_transient(&self) -> bool { + matches!(self, Self::Transient(_)) + } +} + +type UpgradeResult = Result; + +fn terminal(error: impl Into) -> UpgradeError { + UpgradeError::Terminal(error.into()) +} + +/// Classify a failed HTTP exchange: transport trouble and server-side +/// statuses are transient, a client-side status is terminal. +fn http_failure(error: reqwest::Error, context: String) -> UpgradeError { + let transient = match error.status() { + Some(status) => status.is_server_error() || status.as_u16() == 429, + None => true, + }; + let error = anyhow::Error::new(error).context(context); + if transient { + UpgradeError::Transient(error) + } else { + UpgradeError::Terminal(error) + } } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct CandidateBuild { - name: String, - version: String, - git_sha: String, - target: String, - protocol_version: u32, +macro_rules! fail { + ($($arg:tt)*) => { + return Err(terminal(anyhow!($($arg)*))) + }; } /// Resolve an explicit override or derive the deployment discovery endpoint -/// from an outbound WebSocket gateway URL. +/// from an outbound WebSocket gateway URL: its canonical origin, as HTTP, +/// with `DISCOVERY_PATH`. pub fn resolve_discovery_url( gateway_url: Option<&str>, override_url: Option<&str>, -) -> Result { +) -> anyhow::Result { if let Some(url) = override_url { validate_discovery_url(url)?; return Ok(url.to_owned()); @@ -85,123 +113,133 @@ pub fn resolve_discovery_url( let gateway_url = gateway_url.ok_or_else(|| { anyhow!("upgrade needs LIGHTSPEED_ENVD_GATEWAY_URL or LIGHTSPEED_ENVD_DISCOVERY_URL") })?; - let mut url = Url::parse(gateway_url).context("parse environment gateway URL")?; - let http_scheme = match url.scheme() { - "wss" => "https", - "ws" => "http", - scheme => bail!("cannot derive discovery URL from gateway scheme {scheme}"), + if !gateway_url.starts_with("ws://") && !gateway_url.starts_with("wss://") { + anyhow::bail!("cannot derive a discovery URL from gateway URL {gateway_url}"); + } + let origin = canonical_origin(gateway_url).context("parse environment gateway URL")?; + let resolved = match origin.as_str().strip_prefix("wss://") { + Some(authority) => format!("https://{authority}{DISCOVERY_PATH}"), + None => format!( + "http://{}{DISCOVERY_PATH}", + origin.as_str().trim_start_matches("ws://") + ), }; - url.set_scheme(http_scheme) - .map_err(|_| anyhow!("derive discovery URL scheme"))?; - url.set_path(DISCOVERY_PATH); - url.set_query(None); - url.set_fragment(None); - let resolved = url.to_string(); validate_discovery_url(&resolved)?; Ok(resolved) } /// Discovery and archive downloads require HTTPS. Plain HTTP exists only for /// loopback development and the local automatic-upgrade acceptance fixture. -pub fn validate_discovery_url(value: &str) -> Result<()> { +pub fn validate_discovery_url(value: &str) -> anyhow::Result<()> { validate_download_url(value, "discovery URL").map(|_| ()) } -pub async fn install(request: UpgradeRequest) -> Result { +pub async fn install(request: UpgradeRequest) -> UpgradeResult { crate::install_crypto_provider(); - validate_discovery_url(&request.discovery_url)?; - let client = http_client(request.ca_file.as_deref()).await?; + validate_discovery_url(&request.discovery_url).map_err(terminal)?; + let client = http_client(request.ca_file.as_deref()) + .await + .map_err(terminal)?; let discovery = fetch_discovery(&client, &request.discovery_url).await?; if let Some(expected) = request.expected_protocol && discovery.protocol_version != expected { - bail!( + fail!( "discovery document {} advertises protocol {}, but the gateway challenged with protocol {expected}", request.discovery_url, discovery.protocol_version ); } - validate_git_sha(&discovery.git_sha, "discovery gitSha")?; - let artifact = discovery.artifacts.get(&request.target).ok_or_else(|| { - anyhow!( + if request.other_than == Some(discovery.protocol_version) { + fail!( + "discovery document {} advertises protocol {}, which this daemon already speaks", + request.discovery_url, + discovery.protocol_version + ); + } + validate_git_sha(&discovery.git_sha, "discovery gitSha").map_err(terminal)?; + let Some(artifact) = discovery.artifacts.get(&request.target) else { + fail!( "discovery document {} has no envd artifact for target {}", request.discovery_url, request.target - ) - })?; - validate_artifact(artifact)?; - let artifact_url = artifact.url.as_deref().ok_or_else(|| { - anyhow!( + ); + }; + validate_artifact(artifact).map_err(terminal)?; + let Some(artifact_url) = artifact.url.as_deref() else { + fail!( "deployment discovery document {} has no public URL for target {}", request.discovery_url, request.target - ) - })?; - let artifact_url = validate_download_url(artifact_url, "envd artifact URL")?; + ); + }; + let artifact_url = + validate_download_url(artifact_url, "envd artifact URL").map_err(terminal)?; if !artifact_url .path() .ends_with(&format!("/{}", artifact.file)) { - bail!("envd artifact URL does not end with its declared filename"); + fail!("envd artifact URL does not end with its declared filename"); } let (mut archive_file, actual_sha) = download_archive(&client, &artifact_url, MAX_ARCHIVE_BYTES).await?; if actual_sha != artifact.sha256 { - bail!( + fail!( "envd archive checksum mismatch: expected {}, downloaded {actual_sha}", artifact.sha256 ); } - let parent = request.install_path.parent().ok_or_else(|| { - anyhow!( + let manual = || { + manual_install_command( + artifact_url.as_str(), + &artifact.sha256.to_hex(), + &request.install_path, + ) + }; + let Some(parent) = request.install_path.parent() else { + fail!( "envd executable has no parent directory: {}", request.install_path.display() - ) - })?; + ); + }; let mut candidate = tempfile::Builder::new() .prefix(".lightspeed-envd-upgrade-") .tempfile_in(parent) .map_err(|error| { - anyhow!( + terminal(anyhow!( "cannot write next to {}: {error}\n{}", request.install_path.display(), - manual_install_command( - artifact_url.as_str(), - &artifact.sha256, - &request.install_path - ) - ) + manual() + )) })?; - extract_binary(&mut archive_file, candidate.as_file_mut())?; - make_executable(candidate.path())?; + extract_binary(&mut archive_file, candidate.as_file_mut()).map_err(terminal)?; + make_executable(candidate.path()).map_err(terminal)?; candidate .as_file_mut() .sync_all() - .context("sync candidate envd executable")?; + .context("sync candidate envd executable") + .map_err(terminal)?; // Close the write handle before executing the candidate: Linux refuses // to run a file that is still open for writing (ETXTBSY). The path stays // reserved and is removed on every failure path. let candidate = candidate.into_temp_path(); - let build = inspect_candidate(&candidate).await?; - verify_candidate(&build, &discovery, &request.target)?; + let build = inspect_candidate(&candidate).await.map_err(terminal)?; + verify_candidate(&build, &discovery, &request.target).map_err(terminal)?; candidate.persist(&request.install_path).map_err(|error| { - anyhow!( + terminal(anyhow!( "replace {} atomically: {}\n{}", request.install_path.display(), error.error, - manual_install_command( - artifact_url.as_str(), - &artifact.sha256, - &request.install_path - ) - ) + manual() + )) })?; std::fs::File::open(&request.install_path) .and_then(|installed| installed.sync_all()) - .context("sync installed envd executable")?; + .context("sync installed envd executable") + .map_err(terminal)?; Ok(InstalledBuild { version: build.version, @@ -211,7 +249,7 @@ pub async fn install(request: UpgradeRequest) -> Result { }) } -async fn http_client(ca_file: Option<&Path>) -> Result { +async fn http_client(ca_file: Option<&Path>) -> anyhow::Result { let mut builder = reqwest::Client::builder() .connect_timeout(Duration::from_secs(15)) .read_timeout(Duration::from_secs(30)) @@ -223,7 +261,7 @@ async fn http_client(ca_file: Option<&Path>) -> Result { let certificates = reqwest::tls::Certificate::from_pem_bundle(&pem) .with_context(|| format!("parse CA file {}", ca_file.display()))?; if certificates.is_empty() { - bail!("CA file {} holds no certificates", ca_file.display()); + anyhow::bail!("CA file {} holds no certificates", ca_file.display()); } for certificate in certificates { builder = builder.add_root_certificate(certificate); @@ -232,79 +270,91 @@ async fn http_client(ca_file: Option<&Path>) -> Result { builder.build().context("build envd upgrade HTTP client") } -async fn fetch_discovery(client: &reqwest::Client, url: &str) -> Result { +/// Fetch and decode the discovery document. It is part of the frozen +/// bootstrap surface: members this daemon does not know are ignored. +async fn fetch_discovery(client: &reqwest::Client, url: &str) -> UpgradeResult { + let context = || format!("fetch envd discovery document {url}"); let response = client .get(url) .send() .await - .with_context(|| format!("fetch envd discovery document {url}"))? - .error_for_status() - .with_context(|| format!("fetch envd discovery document {url}"))?; + .and_then(reqwest::Response::error_for_status) + .map_err(|error| http_failure(error, context()))?; if response .content_length() .is_some_and(|length| length > MAX_DISCOVERY_BYTES as u64) { - bail!("envd discovery document exceeds {MAX_DISCOVERY_BYTES} bytes"); + fail!("envd discovery document exceeds {MAX_DISCOVERY_BYTES} bytes"); } let mut bytes = Vec::new(); let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { - let chunk = chunk.context("read envd discovery document")?; + let chunk = chunk.map_err(|error| http_failure(error, context()))?; if bytes.len().saturating_add(chunk.len()) > MAX_DISCOVERY_BYTES { - bail!("envd discovery document exceeds {MAX_DISCOVERY_BYTES} bytes"); + fail!("envd discovery document exceeds {MAX_DISCOVERY_BYTES} bytes"); } bytes.extend_from_slice(&chunk); } - serde_json::from_slice(&bytes).context("decode envd discovery document") + serde_json::from_slice(&bytes) + .context("decode envd discovery document") + .map_err(terminal) } async fn download_archive( client: &reqwest::Client, url: &Url, max_bytes: u64, -) -> Result<(tempfile::NamedTempFile, String)> { +) -> UpgradeResult<(tempfile::NamedTempFile, ArtifactSha256)> { + let context = || format!("download envd archive {url}"); let response = client .get(url.clone()) .send() .await - .with_context(|| format!("download envd archive {url}"))? - .error_for_status() - .with_context(|| format!("download envd archive {url}"))?; + .and_then(reqwest::Response::error_for_status) + .map_err(|error| http_failure(error, context()))?; if response .content_length() .is_some_and(|length| length > max_bytes) { - bail!("envd archive exceeds {max_bytes} bytes"); + fail!("envd archive exceeds {max_bytes} bytes"); } - let mut file = tempfile::NamedTempFile::new().context("create temporary envd archive")?; + let mut file = tempfile::NamedTempFile::new() + .context("create temporary envd archive") + .map_err(terminal)?; let mut stream = response.bytes_stream(); let mut size = 0_u64; let mut sha = Sha256::new(); while let Some(chunk) = stream.next().await { - let chunk = chunk.context("read envd archive response")?; - size = size - .checked_add(chunk.len() as u64) - .ok_or_else(|| anyhow!("envd archive size overflow"))?; + let chunk = chunk.map_err(|error| http_failure(error, context()))?; + size = size.saturating_add(chunk.len() as u64); if size > max_bytes { - bail!("envd archive exceeds {max_bytes} bytes"); + fail!("envd archive exceeds {max_bytes} bytes"); } sha.update(&chunk); file.write_all(&chunk) - .context("write temporary envd archive")?; + .context("write temporary envd archive") + .map_err(terminal)?; } file.as_file_mut() .sync_all() - .context("sync temporary envd archive")?; + .context("sync temporary envd archive") + .map_err(terminal)?; file.as_file_mut() .rewind() - .context("rewind temporary envd archive")?; - Ok((file, hex::encode(sha.finalize()))) + .context("rewind temporary envd archive") + .map_err(terminal)?; + let digest: [u8; 32] = sha + .finalize() + .as_slice() + .try_into() + .expect("sha256 is 32 bytes"); + Ok((file, ArtifactSha256::from_bytes(digest))) } fn extract_binary( archive_file: &mut tempfile::NamedTempFile, destination: &mut std::fs::File, -) -> Result<()> { +) -> anyhow::Result<()> { archive_file .as_file_mut() .rewind() @@ -316,27 +366,27 @@ fn extract_binary( let mut entry = entry.context("read envd archive entry")?; let path = entry.path().context("read envd archive path")?; if path.as_ref() != Path::new("lightspeed-envd") { - bail!("envd archive contains unexpected entry {}", path.display()); + anyhow::bail!("envd archive contains unexpected entry {}", path.display()); } if found { - bail!("envd archive contains lightspeed-envd more than once"); + anyhow::bail!("envd archive contains lightspeed-envd more than once"); } if !entry.header().entry_type().is_file() { - bail!("envd archive entry lightspeed-envd is not a regular file"); + anyhow::bail!("envd archive entry lightspeed-envd is not a regular file"); } let size = entry.size(); if size == 0 || size > MAX_BINARY_BYTES { - bail!("envd executable has invalid size {size}"); + anyhow::bail!("envd executable has invalid size {size}"); } let copied = std::io::copy(&mut entry.by_ref().take(MAX_BINARY_BYTES + 1), destination) .context("extract envd executable")?; if copied != size { - bail!("envd executable size differs from its archive header"); + anyhow::bail!("envd executable size differs from its archive header"); } found = true; } if !found { - bail!("envd archive does not contain lightspeed-envd"); + anyhow::bail!("envd archive does not contain lightspeed-envd"); } destination .flush() @@ -344,18 +394,20 @@ fn extract_binary( } #[cfg(unix)] -fn make_executable(path: &Path) -> Result<()> { +fn make_executable(path: &Path) -> anyhow::Result<()> { use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)) .with_context(|| format!("make candidate executable {}", path.display())) } #[cfg(not(unix))] -fn make_executable(_path: &Path) -> Result<()> { - bail!("envd self-upgrade is supported only on Unix") +fn make_executable(_path: &Path) -> anyhow::Result<()> { + anyhow::bail!("envd self-upgrade is supported only on Unix") } -async fn inspect_candidate(path: &Path) -> Result { +/// Run the candidate's `--print-build`, decoded as the frozen +/// [`BuildFacts`]. +async fn inspect_candidate(path: &Path) -> anyhow::Result { let output = tokio::time::timeout( CANDIDATE_TIMEOUT, tokio::process::Command::new(path) @@ -367,7 +419,7 @@ async fn inspect_candidate(path: &Path) -> Result { .context("candidate envd --print-build timed out")? .with_context(|| format!("run candidate envd {}", path.display()))?; if !output.status.success() { - bail!( + anyhow::bail!( "candidate envd --print-build exited with {}: {}", output.status, String::from_utf8_lossy(&output.stderr).trim() @@ -377,38 +429,39 @@ async fn inspect_candidate(path: &Path) -> Result { } fn verify_candidate( - build: &CandidateBuild, + build: &BuildFacts, discovery: &DiscoveryDocument, target: &str, -) -> Result<()> { - if build.name != "lightspeed-envd" { - bail!( - "candidate build name is {}, expected lightspeed-envd", - build.name +) -> anyhow::Result<()> { + if build.name != crate::DAEMON_NAME { + anyhow::bail!( + "candidate build name is {}, expected {}", + build.name, + crate::DAEMON_NAME ); } if build.version != discovery.version { - bail!( + anyhow::bail!( "candidate version {} differs from discovery version {}", build.version, discovery.version ); } if build.git_sha != discovery.git_sha { - bail!( + anyhow::bail!( "candidate gitSha {} differs from discovery gitSha {}", build.git_sha, discovery.git_sha ); } if build.target != target { - bail!( + anyhow::bail!( "candidate target {} differs from requested target {target}", build.target ); } if build.protocol_version != discovery.protocol_version { - bail!( + anyhow::bail!( "candidate protocol {} differs from discovery protocol {}", build.protocol_version, discovery.protocol_version @@ -417,55 +470,47 @@ fn verify_candidate( Ok(()) } -fn validate_artifact(artifact: &DiscoveryArtifact) -> Result<()> { +fn validate_artifact(artifact: &DiscoveryArtifact) -> anyhow::Result<()> { if artifact.file.is_empty() || Path::new(&artifact.file) .file_name() .and_then(|name| name.to_str()) != Some(artifact.file.as_str()) { - bail!("envd artifact has an unsafe filename"); + anyhow::bail!("envd artifact has an unsafe filename"); } if !artifact.file.ends_with(".tar.gz") { - bail!("envd artifact is not a .tar.gz archive"); - } - if artifact.sha256.len() != 64 - || !artifact - .sha256 - .bytes() - .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) - { - bail!("envd artifact has an invalid SHA-256 checksum"); + anyhow::bail!("envd artifact is not a .tar.gz archive"); } Ok(()) } -fn validate_git_sha(value: &str, label: &str) -> Result<()> { +fn validate_git_sha(value: &str, label: &str) -> anyhow::Result<()> { if value.len() != 40 || !value .bytes() .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) { - bail!("{label} must be a full lowercase hexadecimal commit"); + anyhow::bail!("{label} must be a full lowercase hexadecimal commit"); } Ok(()) } -fn validate_download_url(value: &str, label: &str) -> Result { +fn validate_download_url(value: &str, label: &str) -> anyhow::Result { let url = Url::parse(value).with_context(|| format!("parse {label}"))?; if !url.username().is_empty() || url.password().is_some() { - bail!("{label} must not contain credentials"); + anyhow::bail!("{label} must not contain credentials"); } if url.fragment().is_some() { - bail!("{label} must not contain a fragment"); + anyhow::bail!("{label} must not contain a fragment"); } match url.scheme() { "https" => {} "http" if is_loopback(&url) => {} - _ => bail!("{label} must use https (or http toward loopback)"), + _ => anyhow::bail!("{label} must use https (or http toward loopback)"), } if url.host_str().is_none() { - bail!("{label} has no host"); + anyhow::bail!("{label} has no host"); } Ok(url) } @@ -474,6 +519,7 @@ fn is_loopback(url: &Url) -> bool { let Some(host) = url.host_str() else { return false; }; + let host = host.trim_start_matches('[').trim_end_matches(']'); host == "localhost" || host .parse::() @@ -539,7 +585,7 @@ mod tests { release_info::TARGET ); let archive = envd_archive(script.as_bytes()); - let archive_sha = hex::encode(Sha256::digest(&archive)); + let archive_sha = environment_protocol::bytes::encode_hex(&Sha256::digest(&archive)); let (origin, server) = serve_upgrade_with_document(archive, |origin| { discovery_json(origin, &archive_sha, &git_sha, 3) }) @@ -551,6 +597,7 @@ mod tests { install_path: install_path.clone(), target: release_info::TARGET.to_owned(), expected_protocol: Some(3), + other_than: None, }) .await .expect("install"); @@ -581,10 +628,12 @@ mod tests { install_path: install_path.clone(), target: release_info::TARGET.to_owned(), expected_protocol: Some(3), + other_than: None, }) .await .expect_err("protocol mismatch"); server.abort(); + assert!(!error.is_transient()); assert!( error .to_string() @@ -596,6 +645,36 @@ mod tests { ); } + #[cfg(unix)] + #[tokio::test(flavor = "current_thread")] + async fn an_upgrade_to_the_daemons_own_protocol_is_refused() { + let temp = tempfile::tempdir().expect("tempdir"); + let install_path = temp.path().join("lightspeed-envd"); + std::fs::write(&install_path, b"old envd").expect("old executable"); + let git_sha = "d".repeat(40); + let archive = envd_archive(b"#!/bin/sh\nexit 1\n"); + let (origin, server) = serve_upgrade_with_document(archive, |origin| { + discovery_json(origin, &"0".repeat(64), &git_sha, 3) + }) + .await; + let error = install(UpgradeRequest { + discovery_url: format!("{origin}/envd.json"), + ca_file: None, + install_path: install_path.clone(), + target: release_info::TARGET.to_owned(), + expected_protocol: None, + other_than: Some(3), + }) + .await + .expect_err("own protocol"); + server.abort(); + assert!(!error.is_transient(), "reinstalling cannot help"); + assert_eq!( + std::fs::read(&install_path).expect("old executable"), + b"old envd" + ); + } + #[cfg(unix)] #[tokio::test(flavor = "current_thread")] async fn checksum_mismatch_leaves_the_executable_untouched() { @@ -614,10 +693,12 @@ mod tests { install_path: install_path.clone(), target: release_info::TARGET.to_owned(), expected_protocol: Some(4), + other_than: None, }) .await .expect_err("checksum mismatch"); server.await.expect("server"); + assert!(!error.is_transient(), "integrity failures are terminal"); assert!(error.to_string().contains("archive checksum mismatch")); assert_eq!( std::fs::read(&install_path).expect("old executable"), @@ -625,6 +706,51 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn server_errors_are_transient_and_client_errors_terminal() { + for (status, transient) in [(503, true), (429, true), (404, false)] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener"); + let origin = format!("http://{}", listener.local_addr().expect("address")); + let server = tokio::spawn(async move { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + let (mut socket, _) = listener.accept().await.expect("accept"); + let mut request = vec![0_u8; 4096]; + let _ = socket.read(&mut request).await; + let response = format!( + "HTTP/1.1 {status} Status\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + socket.write_all(response.as_bytes()).await.expect("write"); + }); + let temp = tempfile::tempdir().expect("tempdir"); + let error = install(UpgradeRequest { + discovery_url: format!("{origin}/envd.json"), + ca_file: None, + install_path: temp.path().join("lightspeed-envd"), + target: release_info::TARGET.to_owned(), + expected_protocol: Some(3), + other_than: None, + }) + .await + .expect_err("no document"); + server.await.expect("server"); + assert_eq!(error.is_transient(), transient, "HTTP {status}: {error}"); + } + + let unreachable = install(UpgradeRequest { + discovery_url: "http://127.0.0.1:9/envd.json".to_owned(), + ca_file: None, + install_path: PathBuf::from("/nonexistent/lightspeed-envd"), + target: release_info::TARGET.to_owned(), + expected_protocol: None, + other_than: None, + }) + .await + .expect_err("nothing listens"); + assert!(unreachable.is_transient(), "{unreachable}"); + } + #[cfg(unix)] fn envd_archive(bytes: &[u8]) -> Vec { use flate2::{Compression, write::GzEncoder}; diff --git a/crates/environment-daemon/tests/client_round_trip.rs b/crates/environment-daemon/tests/client_round_trip.rs new file mode 100644 index 000000000..e7991405e --- /dev/null +++ b/crates/environment-daemon/tests/client_round_trip.rs @@ -0,0 +1,497 @@ +//! envd's data plane driven end to end by its real consumers: the client +//! library over envd's own connection loop, the tools' data conformance +//! suite, the tools' transfer orchestration against a blob store, and the +//! builtin file tools over a remote environment. + +use std::{ + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, +}; + +use environment_client::{ClientError, ConnectOptions, DataClient}; +use environment_daemon::{ + DaemonRuntime, + config::DaemonConfig, + dispatch::{RequestContext, handle_request}, + server, +}; +use environment_protocol::{ + data::transfer::{TransferOnExisting, TransferPhase}, + handshake::ImplementationInfo, + jsonrpc::Response, + paths::EnvironmentPath, +}; +use harness::storage::BlobStore; +use serde_json::json; +use tools::{ + builtin::{ + BuiltinTool, BuiltinToolContext, BuiltinToolOperation as Op, BuiltinToolSurface as Surface, + }, + environment_protocol::{RemoteEnvironmentConnection, conformance}, + transfer, +}; + +fn daemon_config(root: &Path, state_dir: PathBuf) -> DaemonConfig { + DaemonConfig { + listen: None, + cwd: root.to_path_buf(), + fs_root: root.to_path_buf(), + state_dir, + read_only_fs: false, + registration: None, + scrubbed_env: Vec::new(), + } +} + +/// A daemon over a fresh root, its state dir beside the root, and an +/// initialized client connected to it through the real connection loop. +struct Daemon { + _temp: tempfile::TempDir, + base: PathBuf, + root: PathBuf, + client: DataClient, +} + +impl Daemon { + async fn start() -> Self { + let temp = tempfile::tempdir().expect("tempdir"); + let base = temp.path().canonicalize().expect("canonical temp"); + let root = base.join("root"); + std::fs::create_dir_all(&root).expect("root"); + let runtime = + DaemonRuntime::new(daemon_config(&root, base.join("state"))).expect("runtime"); + let (client_io, server_io) = tokio::io::duplex(1 << 20); + tokio::spawn(server::serve_stream(runtime, server_io)); + let options = + ConnectOptions::new(ImplementationInfo::new("envd-round-trip-test").expect("name")); + let (client, _) = DataClient::connect_initialized_over(client_io, options) + .await + .expect("initialized"); + Self { + _temp: temp, + base, + root, + client, + } + } + + fn path(&self, relative: &str) -> EnvironmentPath { + let path = if relative.is_empty() { + self.root.clone() + } else { + self.root.join(relative) + }; + EnvironmentPath::new(path.to_str().expect("UTF-8 temp path")).expect("environment path") + } +} + +#[tokio::test(flavor = "current_thread")] +async fn envd_passes_the_data_conformance_suite() { + let daemon = Daemon::start().await; + let outside = daemon.base.join("outside"); + std::fs::create_dir_all(&outside).expect("outside"); + let mut suite = conformance::DataConformance::new(daemon.client.clone(), daemon.path("")); + suite.outside_root = + Some(EnvironmentPath::new(outside.to_str().expect("UTF-8")).expect("outside path")); + let report = conformance::run(&suite).await; + if let Err(failure) = report.into_result() { + panic!("{failure}"); + } +} + +/// A VFS tree materialized into envd and captured back is the same tree; +/// a repeat reuses the bytes already there, and capturing into an empty +/// store moves them. +#[tokio::test(flavor = "current_thread")] +async fn vfs_trees_round_trip_through_materialize_and_capture() { + let daemon = Daemon::start().await; + let cas = tempfile::tempdir().expect("cas"); + let store = store_fs::FsBlobStore::open(cas.path()) + .await + .expect("store"); + // Larger than one content chunk, so content moves in several writes. + let size = 5 * 1024 * 1024 + 7; + let content: Vec = (0..size).map(|index| (index % 251) as u8).collect(); + let blob = store.put_bytes(content.clone()).await.expect("blob"); + let mut directory = vfs::VfsDirectory::default(); + directory.entries.insert( + "run".into(), + vfs::VfsEntry::File(vfs::VfsFile { + blob_ref: blob.clone(), + size_bytes: size as u64, + media_type: None, + executable: true, + }), + ); + directory.entries.insert( + "empty".into(), + vfs::VfsEntry::Directory(vfs::VfsDirectory::default()), + ); + let selection = vfs::VfsEntry::Directory(directory); + std::fs::write(daemon.root.join("sibling"), b"preserved").expect("sibling"); + + let first = transfer::materialize( + &daemon.client, + &store, + "first", + &selection, + daemon.path("selected"), + TransferOnExisting::Replace, + ) + .await + .expect("materialize"); + assert_eq!(first.phase, TransferPhase::Complete); + assert_eq!(first.total_bytes, size as u64); + assert_eq!(first.transferred_bytes, size as u64); + assert_eq!(first.reused_bytes, 0); + let run = daemon.root.join("selected/run"); + assert_eq!(std::fs::read(&run).expect("run"), content); + assert_eq!( + std::fs::metadata(&run).expect("run").permissions().mode() & 0o111, + 0o111 + ); + assert!(daemon.root.join("selected/empty").is_dir()); + + let again = transfer::materialize( + &daemon.client, + &store, + "again", + &selection, + daemon.path("selected"), + TransferOnExisting::Replace, + ) + .await + .expect("materialize again"); + assert_eq!(again.transferred_bytes, 0); + assert_eq!(again.reused_bytes, size as u64); + + let captured = transfer::capture( + &daemon.client, + &store, + None, + "capture-known", + daemon.path("selected"), + ) + .await + .expect("capture"); + assert_eq!(captured.entry, selection); + assert_eq!(captured.status.transferred_bytes, 0); + + let empty_cas = tempfile::tempdir().expect("empty cas"); + let empty_store = store_fs::FsBlobStore::open(empty_cas.path()) + .await + .expect("empty store"); + let captured = transfer::capture( + &daemon.client, + &empty_store, + None, + "capture-missing", + daemon.path("selected"), + ) + .await + .expect("capture into an empty store"); + assert_eq!(captured.entry, selection); + assert_eq!(captured.status.transferred_bytes, size as u64); + assert!(empty_store.has_blob(&blob).await.expect("has blob")); + assert_eq!( + std::fs::read(daemon.root.join("sibling")).expect("sibling"), + b"preserved" + ); +} + +/// A client whose every request is dispatched straight into `runtime`, +/// recording the largest frame either way, and able to lose the response +/// of the next `transfer/commit` after the daemon acted on it. +fn in_process_client(runtime: DaemonRuntime) -> (DataClient, Arc, Arc) { + let largest = Arc::new(AtomicUsize::new(0)); + let lose_commit = Arc::new(AtomicBool::new(false)); + let initialize = runtime.initialize_response(); + let client = DataClient::in_process(initialize, { + let largest = largest.clone(); + let lose_commit = lose_commit.clone(); + move |request| { + let runtime = runtime.clone(); + let largest = largest.clone(); + let lose_commit = lose_commit.clone(); + Box::pin(async move { + largest.fetch_max(request.to_text().len(), Ordering::Relaxed); + let result = handle_request( + &runtime, + &RequestContext::detached(), + &request.method, + request.params.clone(), + ) + .await; + let response = match result { + Ok(reply) => { + let response = Response::success(request.id.clone(), &reply.value); + if let Some(on_written) = reply.on_written { + on_written(); + } + response + } + Err(error) => Response::failure(Some(request.id.clone()), error), + }; + if request.method == "transfer/commit" && lose_commit.swap(false, Ordering::Relaxed) + { + return Err(ClientError::Transport("response lost".to_owned())); + } + largest.fetch_max(response.to_text().len(), Ordering::Relaxed); + Ok(response) + }) + } + }); + (client, largest, lose_commit) +} + +/// The builtin `write_file` (from a blob reference) and `reference` tools +/// move large binary files through bounded transfer chunks on every tool +/// surface; a retried operation answers its first receipt, even after the +/// source changed, the file was edited, or the commit response was lost. +#[tokio::test(flavor = "current_thread")] +async fn blob_file_tools_stream_and_keep_receipts_across_retries() { + for surface in [ + Surface::Canonical, + Surface::CodexLike, + Surface::ClaudeCodeLike, + ] { + let temp = tempfile::tempdir().expect("tempdir"); + let base = temp.path().canonicalize().expect("canonical temp"); + let root = base.join("root"); + std::fs::create_dir_all(&root).expect("root"); + let runtime = + DaemonRuntime::new(daemon_config(&root, base.join("state"))).expect("runtime"); + let cas = tempfile::tempdir().expect("cas"); + let store = Arc::new( + store_fs::FsBlobStore::open(cas.path()) + .await + .expect("store"), + ); + // Larger than the daemon's content chunk and its whole-file read ceiling. + let size = 9 * 1024 * 1024 + 7; + let content = store.put_bytes(vec![0x83; size]).await.expect("blob"); + let (client, largest, lose_commit) = in_process_client(runtime); + let (_, context) = RemoteEnvironmentConnection::new(client) + .with_cwd(tools::fs::FsPath::new(root.to_str().expect("UTF-8")).expect("cwd")) + .into_contexts(store.clone()); + let context = context.with_environment_id("test-env"); + let key = if surface == Surface::ClaudeCodeLike { + "file_path" + } else { + "path" + }; + let write = BuiltinTool::environment(Op::WriteFile, surface); + let capture = BuiltinTool::environment(Op::Reference, surface); + + let writer = context.clone().with_operation_id("write-binary"); + let output = write + .invoke_json( + BuiltinToolContext::Environment(&writer), + json!({key: "nested/source.bin", "content_ref": content}), + ) + .await + .expect("write from reference"); + assert_eq!(output.output_json["bytes_written"], size); + assert_eq!(output.output_json["receipt"]["total_bytes"], size); + let source = root.join("nested/source.bin"); + assert_eq!( + std::fs::metadata(&source).expect("written").len(), + size as u64 + ); + + let capture_cas = tempfile::tempdir().expect("capture cas"); + let capture_store = Arc::new( + store_fs::FsBlobStore::open(capture_cas.path()) + .await + .expect("capture store"), + ); + let mut capture_context = context.clone().with_operation_id("capture-binary"); + capture_context.blobs = capture_store.clone(); + let captured = capture + .invoke_json( + BuiltinToolContext::Environment(&capture_context), + json!({"path": "nested/source.bin"}), + ) + .await + .expect("capture"); + assert_eq!(captured.output_json["content_ref"], content.to_string()); + assert_eq!(captured.output_json["byte_len"], size); + assert_eq!(captured.output_json["source"]["id"], "test-env"); + assert_eq!(captured.attachments.len(), 1); + assert_eq!( + capture_store + .read_blob_range(&content, size as u64 - 7, 7) + .await + .expect("captured tail"), + vec![0x83; 7] + ); + + std::fs::write(&source, b"changed live file").expect("change source"); + let changed = capture + .invoke_json( + BuiltinToolContext::Environment(&capture_context), + json!({"path": "nested/source.bin"}), + ) + .await + .expect("retry after change"); + assert_eq!( + changed.output_json, captured.output_json, + "a retried capture answers the version it first captured" + ); + std::fs::remove_file(&source).expect("remove source"); + let deleted = capture + .invoke_json( + BuiltinToolContext::Environment(&capture_context), + json!({"path": "nested/source.bin"}), + ) + .await + .expect("retry after delete"); + assert_eq!( + deleted.output_json, captured.output_json, + "a completed capture never revisits the live path" + ); + + // A lost commit response leaves the write's receipt to answer the retry. + let result = root.join("result.bin"); + std::fs::write(&result, b"original").expect("result"); + std::fs::set_permissions(&result, std::fs::Permissions::from_mode(0o751)).expect("mode"); + let writer = context.clone().with_operation_id("lost-write-receipt"); + lose_commit.store(true, Ordering::Relaxed); + assert!( + write + .invoke_json( + BuiltinToolContext::Environment(&writer), + json!({key: "result.bin", "content_ref": content}), + ) + .await + .is_err() + ); + assert_eq!( + std::fs::metadata(&result) + .expect("result") + .permissions() + .mode() + & 0o777, + 0o751 + ); + std::fs::write(&result, b"later local edit").expect("edit"); + let retry = write + .invoke_json( + BuiltinToolContext::Environment(&writer), + json!({key: "result.bin", "content_ref": content}), + ) + .await + .expect("retry the write"); + assert_eq!(retry.output_json["receipt"]["total_bytes"], size); + assert_eq!(std::fs::read(&result).expect("result"), b"later local edit"); + + std::fs::create_dir(root.join("directory")).expect("directory"); + std::fs::write(root.join("directory/keep"), b"keep").expect("keep"); + let writer = context.clone().with_operation_id("reject-directory-write"); + assert!( + write + .invoke_json( + BuiltinToolContext::Environment(&writer), + json!({key: "directory", "content_ref": content}), + ) + .await + .is_err() + ); + assert_eq!( + std::fs::read(root.join("directory/keep")).expect("keep"), + b"keep" + ); + let reader = context + .clone() + .with_operation_id("reject-directory-capture"); + assert!( + capture + .invoke_json( + BuiltinToolContext::Environment(&reader), + json!({"path": "directory"}), + ) + .await + .is_err() + ); + assert!( + largest.load(Ordering::Relaxed) < 8 * 1024 * 1024, + "large binary transfers move in bounded chunks" + ); + } +} + +/// Skill discovery over envd's per-root scans: installer layouts under two +/// roots, per-root fingerprint caching, edits that do not change a skill's +/// metadata, and a malformed skill reported as a warning naming its root. +#[tokio::test(flavor = "current_thread")] +async fn skill_discovery_reads_installer_layouts_per_root() { + use tools::skills::environment::{ + apply_cached_fingerprints, environment_skill_catalog, environment_skill_scan_query, + }; + let daemon = Daemon::start().await; + let project_root = daemon.root.join("project/.agents/skills"); + let home_root = daemon.root.join("home/.codex/skills"); + let review = project_root.join("review"); + for directory in [&review, &home_root.join("copy")] { + std::fs::create_dir_all(directory).expect("skill directory"); + } + let doc = "---\nname: review\ndescription: >-\n Review code\n carefully.\n---\nbody one"; + std::fs::write(review.join("SKILL.md"), doc).expect("review"); + std::fs::write(home_root.join("copy/SKILL.md"), doc).expect("copy"); + let config = harness::EnvironmentSkillsConfig { + roots: Some(vec![ + project_root.to_string_lossy().into_owned(), + home_root.to_string_lossy().into_owned(), + ]), + }; + let cwd = daemon.root.join("project"); + let mut query = environment_skill_scan_query(&config, cwd.to_str(), None).expect("query"); + + let first = daemon.client.scan(&query).await.expect("scan"); + let catalog = environment_skill_catalog("machine-a", &first, None).expect("catalog"); + assert!(catalog.warnings.is_empty(), "{:?}", catalog.warnings); + assert_eq!(catalog.skills.len(), 2, "independent copies stay separate"); + assert!( + catalog + .skills + .iter() + .all(|skill| skill.description == "Review code carefully.") + ); + assert_ne!(catalog.skills[0].skill_id, catalog.skills[1].skill_id); + let other = environment_skill_catalog("machine-b", &first, None).expect("other catalog"); + assert_ne!(catalog.skills[0].skill_id, other.skills[0].skill_id); + assert_eq!(catalog.root_fingerprints.len(), 2); + + apply_cached_fingerprints(&mut query, &catalog); + let unchanged = daemon.client.scan(&query).await.expect("rescan"); + let cached = + environment_skill_catalog("machine-a", &unchanged, Some(&catalog)).expect("cached"); + assert_eq!(cached, catalog); + + std::fs::write(review.join("SKILL.md"), doc.replace("one", "two")).expect("body edit"); + let edited = daemon.client.scan(&query).await.expect("scan after edit"); + let edited = + environment_skill_catalog("machine-a", &edited, Some(&catalog)).expect("edited catalog"); + assert_eq!(edited.skills, catalog.skills); + + std::fs::write( + review.join("SKILL.md"), + "---\nname: broken\ndescription: [invalid]\n---", + ) + .expect("broken"); + apply_cached_fingerprints(&mut query, &edited); + let broken = daemon.client.scan(&query).await.expect("scan broken"); + let broken = + environment_skill_catalog("machine-a", &broken, Some(&edited)).expect("broken catalog"); + assert_eq!(broken.skills.len(), 1, "the other root's skill remains"); + assert_eq!(broken.warnings.len(), 1); + assert!( + broken.warnings[0].starts_with(project_root.to_str().expect("UTF-8")), + "{:?}", + broken.warnings + ); +} diff --git a/crates/environment-protocol/Cargo.toml b/crates/environment-protocol/Cargo.toml index b8eaf45db..13f9407a6 100644 --- a/crates/environment-protocol/Cargo.toml +++ b/crates/environment-protocol/Cargo.toml @@ -10,5 +10,17 @@ schema = ["dep:schemars"] base64 = "0.23" serde = { version = "1", features = ["derive"] } serde_json = "1" +serde_path_to_error = "0.1" +sha2 = "0.11" schemars = { version = "1", optional = true } thiserror = "2" + +[dev-dependencies] +# Integration tests check the committed contract, which needs the schema +# export. +environment-protocol = { path = ".", features = ["schema"] } + +[[bin]] +name = "export-contract" +path = "src/bin/export-contract.rs" +required-features = ["schema"] diff --git a/crates/environment-protocol/contract/controller.openrpc.json b/crates/environment-protocol/contract/controller.openrpc.json new file mode 100644 index 000000000..706c453a7 --- /dev/null +++ b/crates/environment-protocol/contract/controller.openrpc.json @@ -0,0 +1,1701 @@ +{ + "components": { + "schemas": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "AdoptTargetParams": { + "additionalProperties": false, + "description": "`target/adopt` params: take ownership of an existing provider target.\nIdempotent per owner. Requires `capabilities.adoptTarget`. A source that\nbelongs to another owner, or an owner that already holds a different\ntarget, is `failedPrecondition` (`ownerMismatch`); a source the provider\ncannot take over is `failedPrecondition` (`notAdoptable`).", + "properties": { + "owner": { + "$ref": "#/components/schemas/TargetOwner" + }, + "sourceTarget": { + "allOf": [ + { + "$ref": "#/components/schemas/ProviderNativeRef" + } + ], + "description": "Provider-native reference to the existing target; the provider\ndefines its format." + } + }, + "required": [ + "owner", + "sourceTarget" + ], + "type": "object" + }, + "BindingId": { + "description": "`bindingId` of a provider binding within a universe.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "BindingRef": { + "additionalProperties": false, + "description": "The provider binding a call is scoped to, for calls that concern no\nparticular target.", + "properties": { + "bindingId": { + "$ref": "#/components/schemas/BindingId" + }, + "universeId": { + "$ref": "#/components/schemas/UniverseId" + } + }, + "required": [ + "universeId", + "bindingId" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "adoptTarget", + "description": "The controller's `adoptTarget` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "CloseTargetParams": { + "additionalProperties": false, + "description": "`target/close` params: destroy the owner's target and every resource it\nholds, public ingress included. Graceful within the provider's own bound,\nthen forced. Answers `closing` until the target is gone, then `closed`;\n`closed` also when the provider holds no target for the owner.", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "ControllerCapabilities": { + "additionalProperties": false, + "description": "Optional controller methods. Calling an absent one is\n`capabilityUnavailable`.", + "properties": { + "adoptTarget": { + "description": "`target/adopt` is served.", + "type": "boolean" + }, + "ingress": { + "description": "`ingress/ensure` and `ingress/remove` are served.", + "type": "boolean" + } + }, + "type": "object" + }, + "ControllerInitializeParams": { + "additionalProperties": false, + "description": "`initialize` params on the controller plane.", + "properties": { + "clientInfo": { + "$ref": "#/components/schemas/ImplementationInfo" + }, + "protocolVersion": { + "description": "The client's `CONTROLLER_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "clientInfo" + ], + "type": "object" + }, + "ControllerInitializeResponse": { + "additionalProperties": false, + "description": "`initialize` response on the controller plane.", + "properties": { + "capabilities": { + "$ref": "#/components/schemas/ControllerCapabilities" + }, + "plane": { + "allOf": [ + { + "$ref": "#/components/schemas/Plane" + } + ], + "description": "Always `controller`." + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "routes": { + "$ref": "#/components/schemas/ControllerRoutes" + }, + "serverInfo": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + "required": [ + "protocolVersion", + "plane", + "serverInfo", + "capabilities", + "routes" + ], + "type": "object" + }, + "ControllerRoutes": { + "additionalProperties": false, + "description": "Where Lightspeed reaches a target's data plane through this provider.", + "properties": { + "data": { + "allOf": [ + { + "$ref": "#/components/schemas/DataRouteBase" + } + ], + "description": "Base URL of the data routes. A target's data route is this base\nfollowed by\n`/{universeId}/{bindingId}/{environmentId}/{incarnationId}/{targetId}`,\neach segment percent-encoded\n(`provider_data_route`), and is\ndialed with the same bearer credential as the controller. A\ncontroller reached over `wss` must advertise a `wss` base." + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "CreateTargetParams": { + "additionalProperties": false, + "description": "`target/create` params. Idempotent per owner: when the owner already\nhas a target, the provider returns its observation and applies no\ncreation policy (template deprecation, binding or image checks).", + "properties": { + "owner": { + "$ref": "#/components/schemas/TargetOwner" + }, + "templateId": { + "$ref": "#/components/schemas/TemplateId" + } + }, + "required": [ + "owner", + "templateId" + ], + "type": "object" + }, + "DataRouteBase": { + "description": "`routes.data` of a controller's `initialize`: the absolute `ws://` or\n`wss://` base URL of the provider's data routes, validated on decode and\nnever rewritten. Lowercase scheme, a host and optional port, an optional\npath that does not end with `/`, no user information, query or\nfragment, at most 2048 bytes. A controller reached over `wss` must\nadvertise a `wss` base, so the provider credential never travels in\ncleartext; the runtime checks it with\n`DataRouteBase::check_for_controller`.", + "maxLength": 2048, + "minLength": 1, + "pattern": "^wss?://[^/?#@\\s]+(/[^?#\\s]*[^/?#\\s])?$", + "type": "string" + }, + "DescriptiveText": { + "description": "A short human-readable text: 1 to 1024 bytes without control characters\nother than line feed and tab. Never parsed.", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DisplayName": { + "description": "A display name: 1 to 128 bytes without control characters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "EnsureIngressParams": { + "additionalProperties": false, + "description": "`ingress/ensure` params: route the template's public ingress to the\ntarget. Idempotent. A target whose template has no `publicIngress` is\n`failedPrecondition` (`ingressUnsupported`).", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "EnvironmentId": { + "description": "`environmentId` of a Lightspeed environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "EnvironmentTemplate": { + "additionalProperties": false, + "description": "A template targets are created from.", + "properties": { + "deprecated": { + "description": "Creates no new targets; existing targets keep working.", + "type": "boolean" + }, + "description": { + "anyOf": [ + { + "$ref": "#/components/schemas/DescriptiveText" + } + ] + }, + "displayName": { + "$ref": "#/components/schemas/DisplayName" + }, + "metadata": { + "allOf": [ + { + "$ref": "#/components/schemas/Metadata" + } + ], + "description": "Provider-specific display facts, such as an image fingerprint." + }, + "publicIngress": { + "description": "Targets of this template can serve public ingress.", + "type": "boolean" + }, + "resources": { + "anyOf": [ + { + "$ref": "#/components/schemas/TemplateResources" + } + ] + }, + "templateId": { + "$ref": "#/components/schemas/TemplateId" + } + }, + "required": [ + "templateId", + "displayName" + ], + "type": "object" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/components/schemas/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/components/schemas/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "GetTargetParams": { + "additionalProperties": false, + "description": "`target/get` params. A pure read: it never creates or changes anything.\nA target that no longer exists is `notFound`.", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "IncarnationId": { + "description": "`incarnationId` of one incarnation of an environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "IngressObservation": { + "additionalProperties": false, + "description": "Realized public ingress of a target.", + "properties": { + "endpoint": { + "allOf": [ + { + "$ref": "#/components/schemas/PublicUrl" + } + ], + "description": "Public URL that reaches the target's application." + } + }, + "required": [ + "endpoint" + ], + "type": "object" + }, + "ListTemplatesParams": { + "additionalProperties": false, + "description": "`template/list` params.", + "properties": { + "binding": { + "$ref": "#/components/schemas/BindingRef" + } + }, + "required": [ + "binding" + ], + "type": "object" + }, + "ListTemplatesResponse": { + "additionalProperties": false, + "description": "`template/list` response.", + "properties": { + "templates": { + "items": { + "$ref": "#/components/schemas/EnvironmentTemplate" + }, + "type": "array" + } + }, + "type": "object" + }, + "Metadata": { + "additionalProperties": { + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "maxProperties": 32, + "propertyNames": { + "maxLength": 64, + "minLength": 1, + "not": { + "pattern": "^lightspeed\\." + }, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "Plane": { + "description": "The plane a connection serves. Each `initialize` response names its\nplane, so a client that reached the wrong socket fails at the handshake.", + "oneOf": [ + { + "const": "data", + "description": "envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "controller", + "description": "A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "registration", + "description": "The registration channel between a daemon and the gateway, versioned\nby `ENVD_PROTOCOL_VERSION`.", + "type": "string" + } + ] + }, + "PowerState": { + "description": "Steady power state of an active target, in provider-neutral terms.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "paused", + "description": "Execution halted, memory retained; resuming is fast.", + "type": "string" + }, + { + "const": "suspended", + "description": "Execution state persisted, compute released; resuming restores it.", + "type": "string" + }, + { + "const": "stopped", + "description": "Powered off; disks retained.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner, or (`target/adopt`) the owner\nalready holds a different target.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + }, + { + "const": "notAdoptable", + "description": "`target/adopt`: the source target exists but the provider cannot take\nit over, for example an instance of a kind it does not manage.", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/components/schemas/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "ProviderNativeRef": { + "description": "`sourceTarget` of `target/adopt`: a provider-native reference to an\nexisting target, in a format the provider defines.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "ProviderTargetId": { + "description": "`targetId`: the provider's own id for a target, opaque to Lightspeed.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "PublicUrl": { + "description": "A public `http://` or `https://` URL a provider reports, such as a\ntarget's ingress `endpoint`: lowercase scheme, a host and optional port,\noptional path and query, no user information or fragment, at most 2048\nbytes. Validated on decode.", + "maxLength": 2048, + "minLength": 1, + "pattern": "^https?://[^/?#@\\s]+([/?][^#\\s]*)?$", + "type": "string" + }, + "RemoveIngressParams": { + "additionalProperties": false, + "description": "`ingress/remove` params: stop routing public ingress to the target.\nIdempotent.", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "SetTargetPowerParams": { + "additionalProperties": false, + "description": "`target/setPower` params: converge the target toward `power`. A target\nalready in, or moving toward, that state is observed without further\naction. Resuming may replace the underlying host process. A target whose\n`phase` is not `active` is `failedPrecondition` (`targetNotActive`); a\nstate outside `supportedPowerStates` is `invalidParams`.", + "properties": { + "power": { + "allOf": [ + { + "$ref": "#/components/schemas/PowerState" + } + ], + "description": "One of the target's `supportedPowerStates`." + }, + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target", + "power" + ], + "type": "object" + }, + "TargetObservation": { + "additionalProperties": false, + "description": "What the provider observes about a target. Every target method answers\nwith one.", + "properties": { + "displayName": { + "anyOf": [ + { + "$ref": "#/components/schemas/DisplayName" + } + ] + }, + "ingress": { + "anyOf": [ + { + "$ref": "#/components/schemas/IngressObservation" + } + ], + "description": "Public ingress as realized; absent when disabled." + }, + "metadata": { + "allOf": [ + { + "$ref": "#/components/schemas/Metadata" + } + ], + "description": "Provider-specific display facts." + }, + "owner": { + "allOf": [ + { + "$ref": "#/components/schemas/TargetOwner" + } + ], + "description": "Echoed from the request." + }, + "state": { + "$ref": "#/components/schemas/TargetState" + }, + "supportedPowerStates": { + "description": "Power states `target/setPower` can move this target to. Empty when\nthe provider offers no power control beyond create and close.", + "items": { + "$ref": "#/components/schemas/PowerState" + }, + "type": "array" + }, + "targetId": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProviderTargetId" + } + ], + "description": "The provider's id for the target. Absent in two cases only: while\n`phase` is `provisioning` and the provider has not assigned an id\nyet (an asynchronous create answers before it has one; the runtime\nrecords the id from the first observation that carries it), and in a\n`closed` observation of an owner the provider holds no target for." + } + }, + "required": [ + "owner", + "state" + ], + "type": "object" + }, + "TargetOwner": { + "additionalProperties": false, + "description": "The Lightspeed incarnation a target belongs to. A provider owns at most\none target per owner.", + "properties": { + "bindingId": { + "$ref": "#/components/schemas/BindingId" + }, + "environmentId": { + "$ref": "#/components/schemas/EnvironmentId" + }, + "incarnationId": { + "$ref": "#/components/schemas/IncarnationId" + }, + "universeId": { + "$ref": "#/components/schemas/UniverseId" + } + }, + "required": [ + "universeId", + "bindingId", + "environmentId", + "incarnationId" + ], + "type": "object" + }, + "TargetPhase": { + "description": "Lifecycle phase of a target.", + "oneOf": [ + { + "const": "provisioning", + "description": "Being created; not yet usable.", + "type": "string" + }, + { + "const": "active", + "description": "Exists; see `power`.", + "type": "string" + }, + { + "const": "closing", + "description": "Being destroyed.", + "type": "string" + }, + { + "const": "closed", + "description": "Gone, and nothing of it remains.", + "type": "string" + }, + { + "const": "failed", + "description": "Broken beyond the provider's own recovery; see `message`.", + "type": "string" + }, + { + "const": "unknown", + "description": "The provider cannot tell; see `message`.", + "type": "string" + } + ] + }, + "TargetRef": { + "additionalProperties": false, + "description": "A reference to an existing target: its owner and, once known, its id.\nWithout `targetId` the provider resolves the owner's target itself.", + "properties": { + "owner": { + "$ref": "#/components/schemas/TargetOwner" + }, + "targetId": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProviderTargetId" + } + ] + } + }, + "required": [ + "owner" + ], + "type": "object" + }, + "TargetResponse": { + "additionalProperties": false, + "description": "Response of every target method and both ingress methods.", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetObservation" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "TargetState": { + "additionalProperties": false, + "description": "A target's state on two axes: lifecycle `phase`, and `power` while\nactive.", + "properties": { + "daemonReady": { + "description": "envd answers on the target's data route (running targets only).", + "type": "boolean" + }, + "message": { + "anyOf": [ + { + "$ref": "#/components/schemas/DescriptiveText" + } + ], + "description": "Why the target is `failed` or `unknown`, or what a transition waits\nfor. Providers build it with\n`DescriptiveText::lossy` from whatever their backend reports." + }, + "phase": { + "$ref": "#/components/schemas/TargetPhase" + }, + "power": { + "anyOf": [ + { + "$ref": "#/components/schemas/PowerState" + } + ], + "description": "Present exactly when `phase` is `active`." + } + }, + "required": [ + "phase" + ], + "type": "object" + }, + "TemplateId": { + "description": "`templateId`: the provider's own id for a template.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "TemplateResources": { + "additionalProperties": false, + "description": "Compute a template's targets get, in provider-neutral units. Each\nfield is absent when the provider does not fix it.", + "properties": { + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "UniverseId": { + "description": "`universeId` of a Lightspeed universe.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + } + } + }, + "info": { + "description": "JSON-RPC 2.0 over WebSocket text frames of at most 16777216 bytes. Exact-match protocolVersion 3. Errors carry code, message and data.kind; see x-lightspeed-error-kinds.", + "title": "Lightspeed environment controller plane", + "version": "3" + }, + "methods": [ + { + "name": "initialize", + "paramStructure": "by-name", + "params": [ + { + "name": "clientInfo", + "required": true, + "schema": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + { + "name": "protocolVersion", + "required": true, + "schema": { + "description": "The client's `CONTROLLER_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ControllerInitializeResponse" + } + }, + "summary": "Check the protocol version and read the controller's capabilities and routes.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "handshake", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ControllerInitializeParams", + "x-lightspeed-result-type": "ControllerInitializeResponse" + }, + { + "name": "template/list", + "paramStructure": "by-name", + "params": [ + { + "name": "binding", + "required": true, + "schema": { + "$ref": "#/components/schemas/BindingRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ListTemplatesResponse" + } + }, + "summary": "List the templates a binding may create targets from.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "template", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ListTemplatesParams", + "x-lightspeed-result-type": "ListTemplatesResponse" + }, + { + "name": "target/create", + "paramStructure": "by-name", + "params": [ + { + "name": "owner", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetOwner" + } + }, + { + "name": "templateId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TemplateId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Create the owner's target, or observe the one it has.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "CreateTargetParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "target/adopt", + "paramStructure": "by-name", + "params": [ + { + "name": "owner", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetOwner" + } + }, + { + "name": "sourceTarget", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ProviderNativeRef" + } + ], + "description": "Provider-native reference to the existing target; the provider\ndefines its format." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Take ownership of an existing provider target.", + "x-lightspeed-capability": "adoptTarget", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "AdoptTargetParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "target/get", + "paramStructure": "by-name", + "params": [ + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Observe a target without changing it.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "GetTargetParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "target/setPower", + "paramStructure": "by-name", + "params": [ + { + "name": "power", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PowerState" + } + ], + "description": "One of the target's `supportedPowerStates`." + } + }, + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Converge a target toward a power state.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "SetTargetPowerParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "target/close", + "paramStructure": "by-name", + "params": [ + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Destroy the owner's target and everything it holds.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "CloseTargetParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "ingress/ensure", + "paramStructure": "by-name", + "params": [ + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Route public ingress to a target.", + "x-lightspeed-capability": "ingress", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "ingress", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "EnsureIngressParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "ingress/remove", + "paramStructure": "by-name", + "params": [ + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Stop routing public ingress to a target that stays.", + "x-lightspeed-capability": "ingress", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "ingress", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "RemoveIngressParams", + "x-lightspeed-result-type": "TargetResponse" + } + ], + "openrpc": "1.3.2", + "x-lightspeed-error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "x-lightspeed-error-kinds": [ + { + "code": -32700, + "kind": "parseError", + "retryable": false + }, + { + "code": -32600, + "kind": "invalidRequest", + "retryable": false + }, + { + "code": -32601, + "kind": "methodNotFound", + "retryable": false + }, + { + "code": -32602, + "kind": "invalidParams", + "retryable": false + }, + { + "code": -32002, + "kind": "notInitialized", + "retryable": false + }, + { + "code": -32005, + "kind": "versionMismatch", + "retryable": false + }, + { + "code": -32004, + "kind": "notFound", + "retryable": false + }, + { + "code": -32009, + "kind": "alreadyExists", + "retryable": false + }, + { + "code": -32007, + "kind": "failedPrecondition", + "retryable": false + }, + { + "code": -32003, + "kind": "permissionDenied", + "retryable": false + }, + { + "code": -32006, + "kind": "capabilityUnavailable", + "retryable": false + }, + { + "code": -32008, + "kind": "busy", + "retryable": true + }, + { + "code": -32015, + "kind": "deadlineExceeded", + "retryable": false + }, + { + "code": -32016, + "kind": "expired", + "retryable": false + }, + { + "code": -32017, + "kind": "aborted", + "retryable": true + }, + { + "code": -32013, + "kind": "payloadTooLarge", + "retryable": false + }, + { + "code": -32018, + "kind": "processFailed", + "retryable": false + }, + { + "code": -32019, + "kind": "unavailable", + "retryable": true + }, + { + "code": -32603, + "kind": "internal", + "retryable": false + } + ], + "x-lightspeed-plane": "controller" +} diff --git a/crates/environment-protocol/contract/controller.schema.json b/crates/environment-protocol/contract/controller.schema.json new file mode 100644 index 000000000..136d04fa5 --- /dev/null +++ b/crates/environment-protocol/contract/controller.schema.json @@ -0,0 +1,1324 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "AdoptTargetParams": { + "additionalProperties": false, + "description": "`target/adopt` params: take ownership of an existing provider target.\nIdempotent per owner. Requires `capabilities.adoptTarget`. A source that\nbelongs to another owner, or an owner that already holds a different\ntarget, is `failedPrecondition` (`ownerMismatch`); a source the provider\ncannot take over is `failedPrecondition` (`notAdoptable`).", + "properties": { + "owner": { + "$ref": "#/definitions/TargetOwner" + }, + "sourceTarget": { + "allOf": [ + { + "$ref": "#/definitions/ProviderNativeRef" + } + ], + "description": "Provider-native reference to the existing target; the provider\ndefines its format." + } + }, + "required": [ + "owner", + "sourceTarget" + ], + "type": "object" + }, + "BindingId": { + "description": "`bindingId` of a provider binding within a universe.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "BindingRef": { + "additionalProperties": false, + "description": "The provider binding a call is scoped to, for calls that concern no\nparticular target.", + "properties": { + "bindingId": { + "$ref": "#/definitions/BindingId" + }, + "universeId": { + "$ref": "#/definitions/UniverseId" + } + }, + "required": [ + "universeId", + "bindingId" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "adoptTarget", + "description": "The controller's `adoptTarget` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "CloseTargetParams": { + "additionalProperties": false, + "description": "`target/close` params: destroy the owner's target and every resource it\nholds, public ingress included. Graceful within the provider's own bound,\nthen forced. Answers `closing` until the target is gone, then `closed`;\n`closed` also when the provider holds no target for the owner.", + "properties": { + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "ControllerCapabilities": { + "additionalProperties": false, + "description": "Optional controller methods. Calling an absent one is\n`capabilityUnavailable`.", + "properties": { + "adoptTarget": { + "description": "`target/adopt` is served.", + "type": "boolean" + }, + "ingress": { + "description": "`ingress/ensure` and `ingress/remove` are served.", + "type": "boolean" + } + }, + "type": "object" + }, + "ControllerInitializeParams": { + "additionalProperties": false, + "description": "`initialize` params on the controller plane.", + "properties": { + "clientInfo": { + "$ref": "#/definitions/ImplementationInfo" + }, + "protocolVersion": { + "description": "The client's `CONTROLLER_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "clientInfo" + ], + "type": "object" + }, + "ControllerInitializeResponse": { + "additionalProperties": false, + "description": "`initialize` response on the controller plane.", + "properties": { + "capabilities": { + "$ref": "#/definitions/ControllerCapabilities" + }, + "plane": { + "allOf": [ + { + "$ref": "#/definitions/Plane" + } + ], + "description": "Always `controller`." + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "routes": { + "$ref": "#/definitions/ControllerRoutes" + }, + "serverInfo": { + "$ref": "#/definitions/ImplementationInfo" + } + }, + "required": [ + "protocolVersion", + "plane", + "serverInfo", + "capabilities", + "routes" + ], + "type": "object" + }, + "ControllerRoutes": { + "additionalProperties": false, + "description": "Where Lightspeed reaches a target's data plane through this provider.", + "properties": { + "data": { + "allOf": [ + { + "$ref": "#/definitions/DataRouteBase" + } + ], + "description": "Base URL of the data routes. A target's data route is this base\nfollowed by\n`/{universeId}/{bindingId}/{environmentId}/{incarnationId}/{targetId}`,\neach segment percent-encoded\n(`provider_data_route`), and is\ndialed with the same bearer credential as the controller. A\ncontroller reached over `wss` must advertise a `wss` base." + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "CreateTargetParams": { + "additionalProperties": false, + "description": "`target/create` params. Idempotent per owner: when the owner already\nhas a target, the provider returns its observation and applies no\ncreation policy (template deprecation, binding or image checks).", + "properties": { + "owner": { + "$ref": "#/definitions/TargetOwner" + }, + "templateId": { + "$ref": "#/definitions/TemplateId" + } + }, + "required": [ + "owner", + "templateId" + ], + "type": "object" + }, + "DataRouteBase": { + "description": "`routes.data` of a controller's `initialize`: the absolute `ws://` or\n`wss://` base URL of the provider's data routes, validated on decode and\nnever rewritten. Lowercase scheme, a host and optional port, an optional\npath that does not end with `/`, no user information, query or\nfragment, at most 2048 bytes. A controller reached over `wss` must\nadvertise a `wss` base, so the provider credential never travels in\ncleartext; the runtime checks it with\n`DataRouteBase::check_for_controller`.", + "maxLength": 2048, + "minLength": 1, + "pattern": "^wss?://[^/?#@\\s]+(/[^?#\\s]*[^/?#\\s])?$", + "type": "string" + }, + "DescriptiveText": { + "description": "A short human-readable text: 1 to 1024 bytes without control characters\nother than line feed and tab. Never parsed.", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DisplayName": { + "description": "A display name: 1 to 128 bytes without control characters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "EnsureIngressParams": { + "additionalProperties": false, + "description": "`ingress/ensure` params: route the template's public ingress to the\ntarget. Idempotent. A target whose template has no `publicIngress` is\n`failedPrecondition` (`ingressUnsupported`).", + "properties": { + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "EnvironmentId": { + "description": "`environmentId` of a Lightspeed environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "EnvironmentTemplate": { + "additionalProperties": false, + "description": "A template targets are created from.", + "properties": { + "deprecated": { + "description": "Creates no new targets; existing targets keep working.", + "type": "boolean" + }, + "description": { + "anyOf": [ + { + "$ref": "#/definitions/DescriptiveText" + } + ] + }, + "displayName": { + "$ref": "#/definitions/DisplayName" + }, + "metadata": { + "allOf": [ + { + "$ref": "#/definitions/Metadata" + } + ], + "description": "Provider-specific display facts, such as an image fingerprint." + }, + "publicIngress": { + "description": "Targets of this template can serve public ingress.", + "type": "boolean" + }, + "resources": { + "anyOf": [ + { + "$ref": "#/definitions/TemplateResources" + } + ] + }, + "templateId": { + "$ref": "#/definitions/TemplateId" + } + }, + "required": [ + "templateId", + "displayName" + ], + "type": "object" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/definitions/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/definitions/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "GetTargetParams": { + "additionalProperties": false, + "description": "`target/get` params. A pure read: it never creates or changes anything.\nA target that no longer exists is `notFound`.", + "properties": { + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "IncarnationId": { + "description": "`incarnationId` of one incarnation of an environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "IngressObservation": { + "additionalProperties": false, + "description": "Realized public ingress of a target.", + "properties": { + "endpoint": { + "allOf": [ + { + "$ref": "#/definitions/PublicUrl" + } + ], + "description": "Public URL that reaches the target's application." + } + }, + "required": [ + "endpoint" + ], + "type": "object" + }, + "ListTemplatesParams": { + "additionalProperties": false, + "description": "`template/list` params.", + "properties": { + "binding": { + "$ref": "#/definitions/BindingRef" + } + }, + "required": [ + "binding" + ], + "type": "object" + }, + "ListTemplatesResponse": { + "additionalProperties": false, + "description": "`template/list` response.", + "properties": { + "templates": { + "items": { + "$ref": "#/definitions/EnvironmentTemplate" + }, + "type": "array" + } + }, + "type": "object" + }, + "Metadata": { + "additionalProperties": { + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "maxProperties": 32, + "propertyNames": { + "maxLength": 64, + "minLength": 1, + "not": { + "pattern": "^lightspeed\\." + }, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "Plane": { + "description": "The plane a connection serves. Each `initialize` response names its\nplane, so a client that reached the wrong socket fails at the handshake.", + "oneOf": [ + { + "const": "data", + "description": "envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "controller", + "description": "A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "registration", + "description": "The registration channel between a daemon and the gateway, versioned\nby `ENVD_PROTOCOL_VERSION`.", + "type": "string" + } + ] + }, + "PowerState": { + "description": "Steady power state of an active target, in provider-neutral terms.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "paused", + "description": "Execution halted, memory retained; resuming is fast.", + "type": "string" + }, + { + "const": "suspended", + "description": "Execution state persisted, compute released; resuming restores it.", + "type": "string" + }, + { + "const": "stopped", + "description": "Powered off; disks retained.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner, or (`target/adopt`) the owner\nalready holds a different target.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + }, + { + "const": "notAdoptable", + "description": "`target/adopt`: the source target exists but the provider cannot take\nit over, for example an instance of a kind it does not manage.", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "ProviderNativeRef": { + "description": "`sourceTarget` of `target/adopt`: a provider-native reference to an\nexisting target, in a format the provider defines.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "ProviderTargetId": { + "description": "`targetId`: the provider's own id for a target, opaque to Lightspeed.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "PublicUrl": { + "description": "A public `http://` or `https://` URL a provider reports, such as a\ntarget's ingress `endpoint`: lowercase scheme, a host and optional port,\noptional path and query, no user information or fragment, at most 2048\nbytes. Validated on decode.", + "maxLength": 2048, + "minLength": 1, + "pattern": "^https?://[^/?#@\\s]+([/?][^#\\s]*)?$", + "type": "string" + }, + "RemoveIngressParams": { + "additionalProperties": false, + "description": "`ingress/remove` params: stop routing public ingress to the target.\nIdempotent.", + "properties": { + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "SetTargetPowerParams": { + "additionalProperties": false, + "description": "`target/setPower` params: converge the target toward `power`. A target\nalready in, or moving toward, that state is observed without further\naction. Resuming may replace the underlying host process. A target whose\n`phase` is not `active` is `failedPrecondition` (`targetNotActive`); a\nstate outside `supportedPowerStates` is `invalidParams`.", + "properties": { + "power": { + "allOf": [ + { + "$ref": "#/definitions/PowerState" + } + ], + "description": "One of the target's `supportedPowerStates`." + }, + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target", + "power" + ], + "type": "object" + }, + "TargetObservation": { + "additionalProperties": false, + "description": "What the provider observes about a target. Every target method answers\nwith one.", + "properties": { + "displayName": { + "anyOf": [ + { + "$ref": "#/definitions/DisplayName" + } + ] + }, + "ingress": { + "anyOf": [ + { + "$ref": "#/definitions/IngressObservation" + } + ], + "description": "Public ingress as realized; absent when disabled." + }, + "metadata": { + "allOf": [ + { + "$ref": "#/definitions/Metadata" + } + ], + "description": "Provider-specific display facts." + }, + "owner": { + "allOf": [ + { + "$ref": "#/definitions/TargetOwner" + } + ], + "description": "Echoed from the request." + }, + "state": { + "$ref": "#/definitions/TargetState" + }, + "supportedPowerStates": { + "description": "Power states `target/setPower` can move this target to. Empty when\nthe provider offers no power control beyond create and close.", + "items": { + "$ref": "#/definitions/PowerState" + }, + "type": "array" + }, + "targetId": { + "anyOf": [ + { + "$ref": "#/definitions/ProviderTargetId" + } + ], + "description": "The provider's id for the target. Absent in two cases only: while\n`phase` is `provisioning` and the provider has not assigned an id\nyet (an asynchronous create answers before it has one; the runtime\nrecords the id from the first observation that carries it), and in a\n`closed` observation of an owner the provider holds no target for." + } + }, + "required": [ + "owner", + "state" + ], + "type": "object" + }, + "TargetOwner": { + "additionalProperties": false, + "description": "The Lightspeed incarnation a target belongs to. A provider owns at most\none target per owner.", + "properties": { + "bindingId": { + "$ref": "#/definitions/BindingId" + }, + "environmentId": { + "$ref": "#/definitions/EnvironmentId" + }, + "incarnationId": { + "$ref": "#/definitions/IncarnationId" + }, + "universeId": { + "$ref": "#/definitions/UniverseId" + } + }, + "required": [ + "universeId", + "bindingId", + "environmentId", + "incarnationId" + ], + "type": "object" + }, + "TargetPhase": { + "description": "Lifecycle phase of a target.", + "oneOf": [ + { + "const": "provisioning", + "description": "Being created; not yet usable.", + "type": "string" + }, + { + "const": "active", + "description": "Exists; see `power`.", + "type": "string" + }, + { + "const": "closing", + "description": "Being destroyed.", + "type": "string" + }, + { + "const": "closed", + "description": "Gone, and nothing of it remains.", + "type": "string" + }, + { + "const": "failed", + "description": "Broken beyond the provider's own recovery; see `message`.", + "type": "string" + }, + { + "const": "unknown", + "description": "The provider cannot tell; see `message`.", + "type": "string" + } + ] + }, + "TargetRef": { + "additionalProperties": false, + "description": "A reference to an existing target: its owner and, once known, its id.\nWithout `targetId` the provider resolves the owner's target itself.", + "properties": { + "owner": { + "$ref": "#/definitions/TargetOwner" + }, + "targetId": { + "anyOf": [ + { + "$ref": "#/definitions/ProviderTargetId" + } + ] + } + }, + "required": [ + "owner" + ], + "type": "object" + }, + "TargetResponse": { + "additionalProperties": false, + "description": "Response of every target method and both ingress methods.", + "properties": { + "target": { + "$ref": "#/definitions/TargetObservation" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "TargetState": { + "additionalProperties": false, + "description": "A target's state on two axes: lifecycle `phase`, and `power` while\nactive.", + "properties": { + "daemonReady": { + "description": "envd answers on the target's data route (running targets only).", + "type": "boolean" + }, + "message": { + "anyOf": [ + { + "$ref": "#/definitions/DescriptiveText" + } + ], + "description": "Why the target is `failed` or `unknown`, or what a transition waits\nfor. Providers build it with\n`DescriptiveText::lossy` from whatever their backend reports." + }, + "phase": { + "$ref": "#/definitions/TargetPhase" + }, + "power": { + "anyOf": [ + { + "$ref": "#/definitions/PowerState" + } + ], + "description": "Present exactly when `phase` is `active`." + } + }, + "required": [ + "phase" + ], + "type": "object" + }, + "TemplateId": { + "description": "`templateId`: the provider's own id for a template.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "TemplateResources": { + "additionalProperties": false, + "description": "Compute a template's targets get, in provider-neutral units. Each\nfield is absent when the provider does not fix it.", + "properties": { + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "UniverseId": { + "description": "`universeId` of a Lightspeed universe.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + } + }, + "description": "Every wire type of the controller plane, protocol version 3.", + "title": "Lightspeed environment controller plane" +} diff --git a/crates/environment-protocol/contract/data.openrpc.json b/crates/environment-protocol/contract/data.openrpc.json new file mode 100644 index 000000000..8612aa09f --- /dev/null +++ b/crates/environment-protocol/contract/data.openrpc.json @@ -0,0 +1,5768 @@ +{ + "components": { + "schemas": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "AppendTransferInventoryParams": { + "additionalProperties": false, + "description": "`transfer/append` params: add incoming inventory entries in order.\nRetrying an identical page is safe.", + "properties": { + "entries": { + "description": "At most `limits.filesystem.transfer.maxInventoryPageEntries`.", + "items": { + "$ref": "#/components/schemas/InventoryEntry" + }, + "type": "array" + }, + "index": { + "description": "Index of the page's first entry: the number of entries appended\nbefore it.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "last": { + "description": "This page completes the inventory.", + "type": "boolean" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "index", + "entries" + ], + "type": "object" + }, + "BeginTransferParams": { + "additionalProperties": false, + "description": "`transfer/begin` params. Idempotent: a repeat with identical input\nreturns the operation's status; different input under the same id is\n`failedPrecondition` (`operationIdInUse`).", + "properties": { + "limits": { + "$ref": "#/components/schemas/TransferLimits" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + }, + "selection": { + "$ref": "#/components/schemas/TransferSelection" + } + }, + "required": [ + "operationId", + "selection", + "limits" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "ByteChunk": { + "contentEncoding": "base64", + "description": "Payload bytes, always named `data` on the wire, as padded standard\nbase64.", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$", + "type": "string" + }, + "CancelJobsParams": { + "additionalProperties": false, + "description": "`job/cancel` params.", + "properties": { + "force": { + "description": "Kill at once instead of `SIGTERM`, `limits.job.cancelGraceMs`, then\n`SIGKILL`.", + "type": "boolean" + }, + "jobIds": { + "description": "1 to `limits.job.maxRequestJobs` ids.", + "items": { + "$ref": "#/components/schemas/JobId" + }, + "type": "array" + }, + "scope": { + "allOf": [ + { + "$ref": "#/components/schemas/JobCancelScope" + } + ], + "description": "`job` by default." + } + }, + "required": [ + "jobIds" + ], + "type": "object" + }, + "CancelJobsResponse": { + "additionalProperties": false, + "description": "`job/cancel` response: one entry per affected job, named jobs first in\nrequest order, then dependents, at most `limits.job.maxRequestJobs`\nentries in total.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/components/schemas/JobSummaryEntry" + }, + "type": "array" + }, + "unlistedJobCount": { + "description": "Dependents cancelled as well but not listed, beyond the entry limit.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "CancelRequestParams": { + "additionalProperties": false, + "description": "`$/cancelRequest` params.", + "properties": { + "id": { + "allOf": [ + { + "$ref": "#/components/schemas/Id" + } + ], + "description": "Id of the request to abort." + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "adoptTarget", + "description": "The controller's `adoptTarget` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "CommandSpec": { + "additionalProperties": false, + "description": "What to run. Shared by `process/start` and `job/start`.", + "properties": { + "argv": { + "description": "Program and arguments; `argv[0]` is resolved through `PATH`. Not\nempty.", + "items": { + "type": "string" + }, + "type": "array" + }, + "cwd": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "Working directory; the daemon's `defaultCwd` when absent." + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "description": "Variables added to the daemon's scrubbed environment.", + "type": "object" + }, + "input": { + "allOf": [ + { + "$ref": "#/components/schemas/ExecInput" + } + ], + "description": "Standard input; none by default." + }, + "secretEnv": { + "additionalProperties": { + "$ref": "#/components/schemas/SecretString" + }, + "description": "Like `env`, but every value is redacted from the command's output.", + "type": "object" + }, + "timeoutMs": { + "description": "Stop deadline from start. On expiry the daemon terminates the process\ngroup (`SIGTERM`, the advertised grace, then `SIGKILL`) and records\n`stoppedBy: timeout`. Absent means no deadline.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "argv" + ], + "type": "object" + }, + "ContentDigest": { + "description": "`sha256:` followed by 64 lowercase hex digits: the digest of a file's raw\nbytes, independent of its path and mode.", + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + "CreateDirectoryParams": { + "additionalProperties": false, + "description": "`fs/createDirectory` params.", + "properties": { + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "recursive": { + "description": "Create missing parents, and succeed when the directory exists.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "CreateDirectoryResponse": { + "additionalProperties": false, + "description": "`fs/createDirectory` response.", + "type": "object" + }, + "DaemonInstanceId": { + "description": "`daemonInstanceId`: random per daemon start. A change between two\nconnections means the daemon restarted and every process handle from\nthe earlier instance is gone.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DataCapabilities": { + "additionalProperties": false, + "description": "Capability families. A present family means its base methods exist; a\nfamily's fields are facts that vary per instance. A request to an\nabsent family, or one needing an absent fact, is\n`capabilityUnavailable`.", + "properties": { + "filesystem": { + "anyOf": [ + { + "$ref": "#/components/schemas/FilesystemCapabilities" + } + ], + "description": "`fs/*` (search, glob and scan included) and `transfer/*`." + }, + "job": { + "anyOf": [ + { + "$ref": "#/components/schemas/JobCapabilities" + } + ], + "description": "`job/*`." + }, + "process": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProcessCapabilities" + } + ], + "description": "`process/*`." + } + }, + "type": "object" + }, + "DataLimits": { + "additionalProperties": false, + "description": "Operational ceilings this daemon enforces. A request above a ceiling is\n`invalidParams` naming the field and the ceiling; nothing is clamped.", + "properties": { + "filesystem": { + "$ref": "#/components/schemas/FilesystemLimits" + }, + "job": { + "$ref": "#/components/schemas/JobLimits" + }, + "maxConcurrentRequests": { + "description": "Requests in flight per connection before `busy`; at least\n`MIN_CONCURRENT_REQUESTS`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxWaitMs": { + "description": "Largest `waitMs` of `process/read` and `job/read`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "process": { + "$ref": "#/components/schemas/ProcessLimits" + } + }, + "required": [ + "maxConcurrentRequests", + "maxWaitMs", + "process", + "job", + "filesystem" + ], + "type": "object" + }, + "DirectoryCursor": { + "description": "`cursor` and `nextCursor` of `fs/readDirectory`: an opaque position\nthe daemon mints, echoed back unchanged.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DirectoryEntry": { + "additionalProperties": false, + "description": "One `fs/readDirectory` entry.", + "properties": { + "metadata": { + "allOf": [ + { + "$ref": "#/components/schemas/FileMetadata" + } + ], + "description": "The entry itself, not a symlink's target. Entries that vanish while\nthe page is built are left out." + }, + "name": { + "$ref": "#/components/schemas/EntryName" + } + }, + "required": [ + "name", + "metadata" + ], + "type": "object" + }, + "EntryName": { + "description": "One entry name: a single path segment, neither `.` nor `..`, without\n`/` or NUL, at most 4096 bytes. `fs/readDirectory` entries use it.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(?!\\.\\.?$)[^/\\u0000]+$", + "type": "string" + }, + "EnvironmentPath": { + "description": "An absolute, normalized POSIX path in the environment: `/`, or `/` and\none or more segments joined by `/`. Empty, `.` and `..` segments and a\ntrailing `/` are rejected, never rewritten. At most 4096 bytes.\n`defaultCwd`, `homeDirectory`, every `cwd` and every path parameter use\nthis type; see the module's path model and confinement rule.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(/|(/(?!\\.\\.?(/|$))[^/\\u0000]+)+)$", + "type": "string" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/components/schemas/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/components/schemas/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "ExecInput": { + "description": "Standard input of a command.", + "oneOf": [ + { + "additionalProperties": false, + "description": "`/dev/null`: the command reads end of file. Model-facing tools use\nthis.", + "properties": { + "kind": { + "const": "none", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "These bytes, then end of file. Written by a writer task after the\noutput readers, the exit watcher and the deadline are in place.", + "properties": { + "data": { + "$ref": "#/components/schemas/ByteChunk" + }, + "kind": { + "const": "bytes", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A persistent pipe fed by `process/write` until closed. Processes\nonly; `job/start` refuses it with `invalidParams`.", + "properties": { + "kind": { + "const": "pipe", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "ExecState": { + "description": "Lifecycle state of a process.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "exited", + "description": "The root exited and its output settled.", + "type": "string" + } + ] + }, + "ExitStatus": { + "description": "How the root process ended.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "kind": { + "const": "code", + "type": "string" + } + }, + "required": [ + "kind", + "code" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Killed by this signal.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "kind": { + "const": "signal", + "type": "string" + }, + "signal": { + "$ref": "#/components/schemas/SignalName" + } + }, + "required": [ + "kind", + "signal" + ], + "type": "object" + } + ] + }, + "FileKind": { + "description": "Kind of a filesystem entry.", + "oneOf": [ + { + "enum": [ + "file", + "directory", + "symlink" + ], + "type": "string" + }, + { + "const": "other", + "description": "Device, FIFO, socket or another special file.", + "type": "string" + } + ] + }, + "FileMetadata": { + "additionalProperties": false, + "description": "One file-entry shape for `fs/getMetadata` and `fs/readDirectory`.", + "properties": { + "createdAtMs": { + "description": "Absent when the filesystem records no birth time.", + "format": "int64", + "type": "integer" + }, + "executable": { + "description": "A regular file with any execute bit set; always false for\ndirectories, symlinks and special files.", + "type": "boolean" + }, + "kind": { + "$ref": "#/components/schemas/FileKind" + }, + "modifiedAtMs": { + "format": "int64", + "type": "integer" + }, + "sizeBytes": { + "description": "Size in bytes; for a symlink, the length of its target text.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "symlinkTarget": { + "description": "Target text of a symlink, when it is UTF-8.", + "type": "string" + }, + "version": { + "allOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Changes whenever the entry's content or identity changes; compare it\nwith `ifMatch`." + } + }, + "required": [ + "kind", + "sizeBytes", + "modifiedAtMs", + "version" + ], + "type": "object" + }, + "FileVersion": { + "description": "Opaque version token of a file's content and identity. Equal tokens for\none path mean the content is unchanged. 1 to 255 bytes without control\ncharacters.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "FilesystemCapabilities": { + "additionalProperties": false, + "description": "The `filesystem` family: `fs/*` and `transfer/*`.", + "properties": { + "write": { + "description": "Methods that change the filesystem are served; false on a daemon\nstarted read-only.", + "type": "boolean" + } + }, + "type": "object" + }, + "FilesystemLimits": { + "additionalProperties": false, + "description": "Ceilings of `fs/*` and `transfer/*`.", + "properties": { + "glob": { + "$ref": "#/components/schemas/GlobCeilings" + }, + "maxDirectoryEntries": { + "description": "Largest `limit` of `fs/readDirectory`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxReadFileBytes": { + "description": "Largest `maxBytes` of `fs/readFile`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxWriteFileBytes": { + "description": "Largest `data` of `fs/writeFile`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "scan": { + "$ref": "#/components/schemas/ScanCeilings" + }, + "search": { + "$ref": "#/components/schemas/SearchCeilings" + }, + "transfer": { + "$ref": "#/components/schemas/TransferCeilings" + } + }, + "required": [ + "maxReadFileBytes", + "maxWriteFileBytes", + "maxDirectoryEntries", + "search", + "glob", + "scan", + "transfer" + ], + "type": "object" + }, + "GapReason": { + "description": "Why a range of output cannot be returned.", + "oneOf": [ + { + "const": "overflow", + "description": "Never delivered: dropped from the middle because retained output\nexceeded the handle's cap. The head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Already delivered through the daemon cursor (or acknowledged) and\nreleased since.", + "type": "string" + } + ] + }, + "GetIdleParams": { + "additionalProperties": false, + "description": "`env/getIdle` params.", + "type": "object" + }, + "GetIdleResponse": { + "additionalProperties": false, + "description": "`env/getIdle` response.", + "properties": { + "idleForMs": { + "description": "Milliseconds since the last `use` request, or since running work\nlast ended, whichever is later. Zero while any process or job runs.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "leftoverProcessGroupCount": { + "description": "Process groups of exited processes that still have members alive:\nservices a command started and left running. They are not running\nwork and do not hold `idleForMs` at zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "runningJobCount": { + "description": "Jobs that are `running`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "runningProcessCount": { + "description": "Processes started with `process/start` whose root has not exited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "idleForMs" + ], + "type": "object" + }, + "GetMetadataParams": { + "additionalProperties": false, + "description": "`fs/getMetadata` params.", + "properties": { + "followSymlinks": { + "description": "Describe the symlink's target instead of the link (subject to\nconfinement).", + "type": "boolean" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "GetMetadataResponse": { + "additionalProperties": false, + "description": "`fs/getMetadata` response.", + "properties": { + "metadata": { + "$ref": "#/components/schemas/FileMetadata" + } + }, + "required": [ + "metadata" + ], + "type": "object" + }, + "GlobCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/globFiles` quotas, field for field.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxDurationMs" + ], + "type": "object" + }, + "GlobFilesLimits": { + "additionalProperties": false, + "description": "Caller quotas of one enumeration; each at most its advertised ceiling.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "description": "Non-directory entries visited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxDurationMs" + ], + "type": "object" + }, + "GlobFilesParams": { + "additionalProperties": false, + "description": "`fs/globFiles` params: bounded recursive enumeration of non-directory\nentries matching `filter.include`.", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeFilter" + } + ], + "description": "`include` must not be empty." + }, + "limits": { + "$ref": "#/components/schemas/GlobFilesLimits" + }, + "root": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "root", + "filter", + "limits" + ], + "type": "object" + }, + "GlobFilesResponse": { + "additionalProperties": false, + "description": "`fs/globFiles` response.", + "properties": { + "elapsedMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "matches": { + "description": "In walk order (sorted by path).", + "items": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Entries the walk could not represent or list: names that are not\nUTF-8 and directories that could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "stopped": { + "anyOf": [ + { + "$ref": "#/components/schemas/GlobFilesStop" + } + ] + }, + "visitedFileCount": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "visitedFileCount", + "elapsedMs" + ], + "type": "object" + }, + "GlobFilesStop": { + "description": "Why an enumeration stopped before finishing its walk.", + "oneOf": [ + { + "const": "matchLimit", + "description": "`maxMatches` paths were found.", + "type": "string" + }, + { + "const": "fileLimit", + "description": "`maxFiles` non-directory entries were visited.", + "type": "string" + }, + { + "const": "timeLimit", + "description": "`maxDurationMs` ran out.", + "type": "string" + }, + { + "const": "responseLimit", + "description": "The next match would push the encoded matches past\n`MAX_RESULT_BYTES`.", + "type": "string" + } + ] + }, + "Id": { + "anyOf": [ + { + "format": "int64", + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "A request id: an integer or a string, echoed in the response." + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "InitializeParams": { + "additionalProperties": false, + "description": "`initialize` params on the data plane.", + "properties": { + "clientInfo": { + "$ref": "#/components/schemas/ImplementationInfo" + }, + "protocolVersion": { + "description": "The client's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "clientInfo" + ], + "type": "object" + }, + "InitializeResponse": { + "additionalProperties": false, + "description": "`initialize` response on the data plane.", + "properties": { + "capabilities": { + "$ref": "#/components/schemas/DataCapabilities" + }, + "daemonInstanceId": { + "allOf": [ + { + "$ref": "#/components/schemas/DaemonInstanceId" + } + ], + "description": "Random per daemon start; a different value than on an earlier\nconnection means the daemon restarted and the earlier instance's\nprocess handles are gone." + }, + "defaultCwd": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "The daemon's working directory: the default `cwd` of commands." + }, + "homeDirectory": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "Home directory of the user commands run as, when known." + }, + "limits": { + "$ref": "#/components/schemas/DataLimits" + }, + "plane": { + "allOf": [ + { + "$ref": "#/components/schemas/Plane" + } + ], + "description": "Always `data`." + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "serverInfo": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + "required": [ + "protocolVersion", + "plane", + "daemonInstanceId", + "serverInfo", + "capabilities", + "limits", + "defaultCwd" + ], + "type": "object" + }, + "InventoryEntry": { + "additionalProperties": false, + "description": "One inventory entry. Parents precede children.", + "properties": { + "content": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeContent" + } + ], + "description": "A file entry always carries its `digest`." + }, + "path": { + "allOf": [ + { + "$ref": "#/components/schemas/TreePath" + } + ], + "description": "Relative to the selected root; `.` is the root itself." + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + }, + "JobCancelScope": { + "description": "Which jobs a `job/cancel` affects.", + "oneOf": [ + { + "const": "job", + "description": "Only the named jobs.", + "type": "string" + }, + { + "const": "dependents", + "description": "The named jobs and every job that transitively depends on them.", + "type": "string" + } + ] + }, + "JobCapabilities": { + "additionalProperties": false, + "description": "The `job` family: `job/*`. It has no instance facts.", + "type": "object" + }, + "JobDependency": { + "description": "A dependency reference.", + "oneOf": [ + { + "additionalProperties": false, + "description": "An existing job on this daemon, or one in the same request.", + "properties": { + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "kind": { + "const": "jobId", + "type": "string" + } + }, + "required": [ + "kind", + "jobId" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The `name` of a job in the same `job/start`.", + "properties": { + "kind": { + "const": "name", + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/JobName" + } + }, + "required": [ + "kind", + "name" + ], + "type": "object" + } + ] + }, + "JobDependencyPolicy": { + "description": "What a job waits for from its `dependsOn` jobs.", + "oneOf": [ + { + "const": "allSucceeded", + "description": "Run once every dependency `succeeded`; any other terminal status\nmakes this job `dependencyFailed`.", + "type": "string" + }, + { + "const": "allTerminal", + "description": "Run once every dependency is terminal, whatever its status.", + "type": "string" + } + ] + }, + "JobId": { + "description": "`jobId`: caller-chosen name of one job run on one daemon.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "JobLimits": { + "additionalProperties": false, + "description": "Ceilings of `job/*`.", + "properties": { + "cancelGraceMs": { + "description": "Grace between `SIGTERM` and `SIGKILL` for a non-forced cancel and\ntimeouts.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxReadBytes": { + "description": "Largest `maxBytes` of one `job/read`, across its entries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxRequestJobs": { + "description": "Most jobs one `job/start`, `job/read` or `job/cancel` names, and most\nentries a `job/cancel` response lists.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxRetainedOutputBytes": { + "description": "Output a job retains: the head and the tail, each half of this; the\nmiddle is an `overflow` gap.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "recordRetentionMs": { + "description": "How long a terminal job's record and output are kept.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxReadBytes", + "maxRequestJobs", + "maxRetainedOutputBytes", + "recordRetentionMs", + "cancelGraceMs" + ], + "type": "object" + }, + "JobName": { + "description": "`name` of a job and `dependsOn {kind: name}`: a caller-chosen name,\nunique within one `job/start`, 1 to 128 bytes without control\ncharacters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "JobReadEntry": { + "description": "A `job/read` entry: the job's summary and output page, or why it could\nnot be read (`notFound` for an unknown or garbage-collected job).", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "job", + "type": "string" + }, + "output": { + "$ref": "#/components/schemas/OutputPage" + }, + "summary": { + "$ref": "#/components/schemas/JobSummary" + } + }, + "required": [ + "kind", + "summary", + "output" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "kind": { + "const": "error", + "type": "string" + } + }, + "required": [ + "kind", + "jobId", + "error" + ], + "type": "object" + } + ] + }, + "JobReadRequest": { + "additionalProperties": false, + "description": "One entry of a `job/read`. Job reads never consume: without `after` and\n`tailBytes` the page starts at offset 0. One job may be named by several\nentries, such as a head and a tail page.", + "properties": { + "after": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "maxBytes": { + "description": "Most `data` bytes for this entry; the request's `maxBytes` still\nbounds the total. Absent: whatever of the request's budget remains.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "tailBytes": { + "description": "Start at `end - tailBytes`; excludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobId" + ], + "type": "object" + }, + "JobRequestId": { + "description": "`requestId` of a `job/start` call. Part of each job's identity: a\nretried start with the same request id and spec replays the job; any\nother reuse of the job id is refused.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "JobSpec": { + "additionalProperties": false, + "description": "One job of a `job/start`.", + "properties": { + "command": { + "allOf": [ + { + "$ref": "#/components/schemas/CommandSpec" + } + ], + "description": "`input` must be `none` or `bytes`." + }, + "dependencyPolicy": { + "allOf": [ + { + "$ref": "#/components/schemas/JobDependencyPolicy" + } + ], + "description": "What \"finish\" means for `dependsOn`; `allSucceeded` by default." + }, + "dependsOn": { + "description": "Jobs that must finish first.", + "items": { + "$ref": "#/components/schemas/JobDependency" + }, + "type": "array" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "name": { + "anyOf": [ + { + "$ref": "#/components/schemas/JobName" + } + ], + "description": "Local name other jobs of the same `job/start` can depend on; unique\nwithin the request." + }, + "queueKey": { + "anyOf": [ + { + "$ref": "#/components/schemas/QueueKey" + } + ] + } + }, + "required": [ + "jobId", + "command" + ], + "type": "object" + }, + "JobStatus": { + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Accepted; waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its `timeoutMs` expired and the daemon stopped it.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of `job/cancel`.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "JobSummary": { + "additionalProperties": false, + "description": "What a job is and how it ended. Instants are the guest's wall clock.", + "properties": { + "cancelRequestedAtMs": { + "description": "When a `job/cancel` reached the job while it ran.", + "format": "int64", + "type": "integer" + }, + "createdAtMs": { + "format": "int64", + "type": "integer" + }, + "dependsOn": { + "description": "Resolved ids of the jobs this one depends on.", + "items": { + "$ref": "#/components/schemas/JobId" + }, + "type": "array" + }, + "durationMs": { + "description": "Monotonic run time from start to finish, immune to guest clock jumps.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProtocolError" + } + ], + "description": "Why a `failed` or `interrupted` job produced no exit, or an\ninfrastructure failure while it ran." + }, + "exit": { + "anyOf": [ + { + "$ref": "#/components/schemas/ExitStatus" + } + ], + "description": "How the root process ended, once it ran and exited." + }, + "finishedAtMs": { + "format": "int64", + "type": "integer" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "leftovers": { + "anyOf": [ + { + "$ref": "#/components/schemas/LeftoverReport" + } + ], + "description": "What the daemon swept from the job's group at completion\n(`disposition: swept`)." + }, + "name": { + "anyOf": [ + { + "$ref": "#/components/schemas/JobName" + } + ] + }, + "queueKey": { + "anyOf": [ + { + "$ref": "#/components/schemas/QueueKey" + } + ] + }, + "queuedAtMs": { + "description": "When the job became runnable and waited for its queue.", + "format": "int64", + "type": "integer" + }, + "requestId": { + "$ref": "#/components/schemas/JobRequestId" + }, + "startedAtMs": { + "format": "int64", + "type": "integer" + }, + "status": { + "$ref": "#/components/schemas/JobStatus" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/components/schemas/StoppedBy" + } + ], + "description": "Set when the daemon stopped the running job: `timeout` for\n`timedOut`, `terminate` or `kill` for a cancel without or with\n`force`." + } + }, + "required": [ + "jobId", + "requestId", + "status", + "createdAtMs" + ], + "type": "object" + }, + "JobSummaryEntry": { + "description": "A job summary, or why the named job could not be found.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "job", + "type": "string" + }, + "summary": { + "$ref": "#/components/schemas/JobSummary" + } + }, + "required": [ + "kind", + "summary" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "kind": { + "const": "error", + "type": "string" + } + }, + "required": [ + "kind", + "jobId", + "error" + ], + "type": "object" + } + ] + }, + "LeftoverDisposition": { + "description": "What the daemon did with leftovers. The policies differ on purpose.", + "oneOf": [ + { + "const": "kept", + "description": "Left running (processes): they keep the environment's lifetime and\nare reported until they exit.", + "type": "string" + }, + { + "const": "swept", + "description": "Terminated when the job finished (jobs); the submitter should fix\nits script's teardown.", + "type": "string" + } + ] + }, + "LeftoverProcess": { + "additionalProperties": false, + "description": "One leftover process.", + "properties": { + "command": { + "description": "Command line when the platform exposes it, otherwise the executable\nname, cut to\n`MAX_LEFTOVER_COMMAND_BYTES`\non a character boundary; absent when neither could be read.", + "type": "string" + }, + "pid": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "LeftoverReport": { + "additionalProperties": false, + "description": "Processes of a command's group still alive after its root exited.\nPresent only when there is at least one.", + "properties": { + "disposition": { + "$ref": "#/components/schemas/LeftoverDisposition" + }, + "omittedProcessCount": { + "description": "Leftovers beyond the listed ones.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "processes": { + "description": "At most `MAX_LEFTOVER_PROCESSES`,\nlowest pid first; never empty.", + "items": { + "$ref": "#/components/schemas/LeftoverProcess" + }, + "type": "array" + } + }, + "required": [ + "disposition", + "processes" + ], + "type": "object" + }, + "ListTransferMissingParams": { + "additionalProperties": false, + "description": "`transfer/listMissing` params.", + "properties": { + "limit": { + "description": "Most digests to return, 1 to\n`limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "limit" + ], + "type": "object" + }, + "ListTransferMissingResponse": { + "additionalProperties": false, + "description": "`transfer/listMissing` response: digests whose content the daemon still\nneeds. The set shrinks as uploads complete; call again rather than\npaging.", + "properties": { + "digests": { + "items": { + "$ref": "#/components/schemas/ContentDigest" + }, + "type": "array" + }, + "remainingCount": { + "description": "Missing digests beyond this page.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "OutputChunk": { + "additionalProperties": false, + "description": "Contiguous output bytes from one stream.", + "properties": { + "data": { + "$ref": "#/components/schemas/ByteChunk" + }, + "offset": { + "description": "Byte offset of the chunk's first byte in the combined output.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "stream": { + "$ref": "#/components/schemas/OutputStream" + } + }, + "required": [ + "offset", + "stream", + "data" + ], + "type": "object" + }, + "OutputGap": { + "additionalProperties": false, + "description": "Output bytes in `[offset, offset + omittedBytes)` that a page cannot\nreturn.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/components/schemas/GapReason" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, + "OutputPage": { + "additionalProperties": false, + "description": "One page of a handle's output. Returned by `process/read` and in every\n`job/read` entry. A page holds at most\n`MAX_PAGE_CHUNKS` chunks and gaps\ntogether (for `job/read`, across all entries); one that reaches it ends\nearly, and `next` says where to continue.", + "properties": { + "chunks": { + "description": "In offset order, at most `maxBytes` of `data` in total.", + "items": { + "$ref": "#/components/schemas/OutputChunk" + }, + "type": "array" + }, + "end": { + "description": "Bytes produced so far; final once the handle has exited.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "gaps": { + "description": "Unreturnable ranges between the requested start and `next`.", + "items": { + "$ref": "#/components/schemas/OutputGap" + }, + "type": "array" + }, + "next": { + "description": "Offset to read from next: past the last returned byte or gap.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "next", + "end" + ], + "type": "object" + }, + "OutputStream": { + "description": "Which descriptor produced a chunk. Jobs never produce `pty`.", + "oneOf": [ + { + "enum": [ + "stdout", + "stderr" + ], + "type": "string" + }, + { + "const": "pty", + "description": "The PTY master, which merges stdout and stderr.", + "type": "string" + } + ] + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "Plane": { + "description": "The plane a connection serves. Each `initialize` response names its\nplane, so a client that reached the wrong socket fails at the handshake.", + "oneOf": [ + { + "const": "data", + "description": "envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "controller", + "description": "A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "registration", + "description": "The registration channel between a daemon and the gateway, versioned\nby `ENVD_PROTOCOL_VERSION`.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner, or (`target/adopt`) the owner\nalready holds a different target.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + }, + { + "const": "notAdoptable", + "description": "`target/adopt`: the source target exists but the provider cannot take\nit over, for example an instance of a kind it does not manage.", + "type": "string" + } + ] + }, + "ProcessCapabilities": { + "additionalProperties": false, + "description": "The `process` family: `process/*`.", + "properties": { + "pty": { + "description": "`process/start` accepts `pty`.", + "type": "boolean" + } + }, + "type": "object" + }, + "ProcessId": { + "description": "`processId`: caller-chosen handle of a process on one daemon.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ProcessLimits": { + "additionalProperties": false, + "description": "Ceilings of `process/*`.", + "properties": { + "maxExitedRetainedBytes": { + "description": "Output retained across all exited processes; beyond it the oldest\nare evicted early.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxInputQueueBytes": { + "description": "Bytes queued for a process's input before `process/write` accepts\nfewer than it was sent.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxReadBytes": { + "description": "Largest `maxBytes` of one `process/read`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxRetainedOutputBytes": { + "description": "Output a process retains; beyond it the oldest delivered bytes are\nreleased (`expired` gaps), then the undelivered middle (`overflow`\ngaps).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "retentionAfterExitMs": { + "description": "How long an exited process stays readable after a read first\nreturned its settled exit with nothing left unread.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "terminateGraceMs": { + "description": "Grace between `SIGTERM` and `SIGKILL` for `terminate` and timeouts.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "unreadRetentionMs": { + "description": "How long an exited process whose output was never fully read stays\nreadable.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxReadBytes", + "maxRetainedOutputBytes", + "retentionAfterExitMs", + "unreadRetentionMs", + "maxExitedRetainedBytes", + "terminateGraceMs", + "maxInputQueueBytes" + ], + "type": "object" + }, + "ProcessSignal": { + "description": "Signal delivered by `process/terminate`.", + "oneOf": [ + { + "const": "interrupt", + "description": "`SIGINT` to the foreground process group (the terminal's under a\nPTY, otherwise the process group). Records nothing; the next read\nshows what the process did.", + "type": "string" + }, + { + "const": "terminate", + "description": "`SIGTERM` to the group, then `SIGKILL` after\n`limits.process.terminateGraceMs` if members remain. Records\n`stoppedBy: terminate`.", + "type": "string" + }, + { + "const": "kill", + "description": "`SIGKILL` to the group. Records `stoppedBy: kill`.", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/components/schemas/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "PtySize": { + "additionalProperties": false, + "description": "Terminal size of a PTY. Each dimension is 1 to 4096.", + "properties": { + "cols": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "rows": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "rows", + "cols" + ], + "type": "object" + }, + "QueueKey": { + "description": "`queueKey`: jobs sharing a key run one at a time, in start order.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ReadDirectoryParams": { + "additionalProperties": false, + "description": "`fs/readDirectory` params. Pages in name order.", + "properties": { + "cursor": { + "anyOf": [ + { + "$ref": "#/components/schemas/DirectoryCursor" + } + ], + "description": "`nextCursor` of the previous page; a cursor this daemon did not mint\nis `invalidParams`." + }, + "limit": { + "description": "Most entries per page, 1 to `limits.filesystem.maxDirectoryEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "path": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "A directory, or a symlink to one inside the root." + } + }, + "required": [ + "path", + "limit" + ], + "type": "object" + }, + "ReadDirectoryResponse": { + "additionalProperties": false, + "description": "`fs/readDirectory` response.", + "properties": { + "entries": { + "items": { + "$ref": "#/components/schemas/DirectoryEntry" + }, + "type": "array" + }, + "nextCursor": { + "anyOf": [ + { + "$ref": "#/components/schemas/DirectoryCursor" + } + ], + "description": "Opaque cursor of the next page; absent on the last page." + }, + "skippedEntryCount": { + "description": "Entries in this page's range skipped because their names are not\nUTF-8. Entries that vanish while the page is built are left out and\nnot counted.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "ReadFileParams": { + "additionalProperties": false, + "description": "`fs/readFile` params. Reads a regular file only; anything else is\n`failedPrecondition` (`notRegularFile`) and is never opened for a\nblocking read.", + "properties": { + "maxBytes": { + "description": "Most bytes to return, at most `limits.filesystem.maxReadFileBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "offset": { + "description": "First byte to return; 0 by default.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path", + "maxBytes" + ], + "type": "object" + }, + "ReadFileResponse": { + "additionalProperties": false, + "description": "`fs/readFile` response.", + "properties": { + "data": { + "$ref": "#/components/schemas/ByteChunk" + }, + "fileSize": { + "description": "Size of the whole file when it was read.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "truncated": { + "description": "Bytes exist after `offset` plus the length of `data`.", + "type": "boolean" + }, + "version": { + "$ref": "#/components/schemas/FileVersion" + } + }, + "required": [ + "data", + "fileSize", + "version" + ], + "type": "object" + }, + "ReadJobsParams": { + "additionalProperties": false, + "description": "`job/read` params.", + "properties": { + "jobs": { + "description": "1 to `limits.job.maxRequestJobs` entries, each with its own position.", + "items": { + "$ref": "#/components/schemas/JobReadRequest" + }, + "type": "array" + }, + "maxBytes": { + "description": "Most `data` bytes across all entries, at most\n`limits.job.maxReadBytes`, filled in request order: an entry reached\nafter the budget is spent returns its summary and an empty page at\nits start position. 0 reads only summaries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "returnOn": { + "allOf": [ + { + "$ref": "#/components/schemas/ReturnOn" + } + ], + "description": "`anyOutput`: return when any entry has output past its position or\nchanged status; `exit`: when every entry is terminal; `window`: wait\nthe whole window." + }, + "waitMs": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobs", + "maxBytes", + "waitMs", + "returnOn" + ], + "type": "object" + }, + "ReadJobsResponse": { + "additionalProperties": false, + "description": "`job/read` response: one entry per requested job, in request order.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/components/schemas/JobReadEntry" + }, + "type": "array" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "ReadProcessParams": { + "additionalProperties": false, + "description": "`process/read` params.", + "properties": { + "ack": { + "description": "Acknowledge every byte before this offset first: the cursor moves to\nit (never backwards) whether or not this read's response arrives. A\ncaller that tracks positions reads with `after` and acknowledges\nwith `ack` for exactly-once delivery.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "after": { + "description": "Start offset. Absent: the daemon cursor, which this read advances to\n`next` once its response is written. Present: a non-consuming read\nthat leaves the cursor alone.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxBytes": { + "description": "Most `data` bytes to return, at most `limits.process.maxReadBytes`.\n0 reads only the state.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "processId": { + "$ref": "#/components/schemas/ProcessId" + }, + "returnOn": { + "$ref": "#/components/schemas/ReturnOn" + }, + "tailBytes": { + "description": "Start at `end - tailBytes` instead (clamped at 0). Non-consuming;\nexcludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "waitMs": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "processId", + "maxBytes", + "waitMs", + "returnOn" + ], + "type": "object" + }, + "ReadProcessResponse": { + "additionalProperties": false, + "description": "`process/read` response.", + "properties": { + "error": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProtocolError" + } + ], + "description": "Infrastructure failure after start (output capture or wait failed)." + }, + "exit": { + "anyOf": [ + { + "$ref": "#/components/schemas/ExitStatus" + } + ], + "description": "How the root ended; present once `state` is `exited`, unless `error`\nexplains why it could not be observed." + }, + "leftovers": { + "anyOf": [ + { + "$ref": "#/components/schemas/LeftoverReport" + } + ], + "description": "Group members alive after the root exited (`disposition: kept`),\nsampled when this response was built. Absent while running, when\nnone remain, and on platforms without process enumeration." + }, + "output": { + "$ref": "#/components/schemas/OutputPage" + }, + "pid": { + "description": "OS pid of the root process.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "state": { + "$ref": "#/components/schemas/ExecState" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/components/schemas/StoppedBy" + } + ], + "description": "Set when the daemon stopped the process; absent on a natural exit." + } + }, + "required": [ + "output", + "state", + "pid" + ], + "type": "object" + }, + "ReadTransferContentParams": { + "additionalProperties": false, + "description": "`transfer/read` params: read a captured file's content by digest.", + "properties": { + "digest": { + "$ref": "#/components/schemas/ContentDigest" + }, + "maxBytes": { + "description": "1 to `limits.filesystem.transfer.maxContentChunkBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "digest", + "offset", + "maxBytes" + ], + "type": "object" + }, + "ReadTransferContentResponse": { + "additionalProperties": false, + "description": "`transfer/read` response.", + "properties": { + "data": { + "$ref": "#/components/schemas/ByteChunk" + }, + "truncated": { + "description": "Bytes exist after `offset` plus the length of `data`, as for\n`fs/readFile`.", + "type": "boolean" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "ReadTransferInventoryParams": { + "additionalProperties": false, + "description": "`transfer/readInventory` params: page a sealed inventory by index.", + "properties": { + "index": { + "description": "Index of the first entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "limit": { + "description": "1 to `limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "index", + "limit" + ], + "type": "object" + }, + "ReadTransferInventoryResponse": { + "additionalProperties": false, + "description": "`transfer/readInventory` response.", + "properties": { + "entries": { + "items": { + "$ref": "#/components/schemas/InventoryEntry" + }, + "type": "array" + }, + "nextIndex": { + "description": "Index of the next page; absent after the last entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "RemoveParams": { + "additionalProperties": false, + "description": "`fs/remove` params.", + "properties": { + "force": { + "description": "Succeed when the path does not exist.", + "type": "boolean" + }, + "path": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "A symlink is removed, never followed." + }, + "recursive": { + "description": "Remove a directory and its contents.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "RemoveResponse": { + "additionalProperties": false, + "description": "`fs/remove` response.", + "type": "object" + }, + "RenameParams": { + "additionalProperties": false, + "description": "`fs/rename` params. Atomic within one filesystem; across filesystems it\nis `failedPrecondition` (`crossDevice`). The entry keeps its mode.", + "properties": { + "destination": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Rename only when the source's version equals this token." + }, + "overwrite": { + "description": "Replace an existing destination file; otherwise an existing\ndestination is `alreadyExists`. A directory destination is never\nreplaced.", + "type": "boolean" + }, + "source": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "source", + "destination" + ], + "type": "object" + }, + "RenameResponse": { + "additionalProperties": false, + "description": "`fs/rename` response.", + "type": "object" + }, + "ReturnOn": { + "description": "When a waiting read returns. Every mode also returns when `waitMs`\nelapses, when the handle reaches a settled exit, or at once when there\nis nothing left to wait for.", + "oneOf": [ + { + "const": "window", + "description": "Collect for the whole window; return early only when unread output\nreaches `maxBytes`.", + "type": "string" + }, + { + "const": "anyOutput", + "description": "Return as soon as any output past the start position exists.", + "type": "string" + }, + { + "const": "exit", + "description": "Return only at the settled exit (for `job/read`: once every entry is\nterminal), even if more than `maxBytes` of output accumulates.", + "type": "string" + } + ] + }, + "ScanCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/scan`: root and pattern counts, then its quotas.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxPatterns": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxRoots": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxRoots", + "maxPatterns", + "maxEntries", + "maxFileBytes", + "maxTotalBytes", + "maxDurationMs" + ], + "type": "object" + }, + "ScanEntry": { + "additionalProperties": false, + "description": "One selected entry below a root.", + "properties": { + "content": { + "$ref": "#/components/schemas/TreeContent" + }, + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "The file's bytes when `readContent` was set." + }, + "path": { + "allOf": [ + { + "$ref": "#/components/schemas/TreePath" + } + ], + "description": "Relative to the root as requested." + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + }, + "ScanFingerprint": { + "description": "Opaque fingerprint of one root's selection, prefixed `scan:v1:`, at most\n255 bytes without control characters.", + "maxLength": 255, + "minLength": 9, + "pattern": "^scan:v1:[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "ScanLimits": { + "additionalProperties": false, + "description": "Caller quotas of one scan.", + "properties": { + "maxDurationMs": { + "description": "Walk time per root; exhausting it fails that root\n(`deadlineExceeded`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "description": "Selected entries per root; more fail that root (`payloadTooLarge`\nwith `limitEntries`).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "description": "Larger selected files are skipped (`fileTooLarge`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "description": "Content bytes returned across all roots, in root order: a root whose\ncontent would exceed the rest fails (`payloadTooLarge` with\n`limitBytes`) and later roots are still scanned. Independently, a\nroot whose encoded result (entries, skips and base64 content) would\npush the scan's results past\n`MAX_RESULT_BYTES` fails the same\nway.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxEntries", + "maxFileBytes", + "maxTotalBytes", + "maxDurationMs" + ], + "type": "object" + }, + "ScanParams": { + "additionalProperties": false, + "description": "`fs/scan` params.", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeFilter" + } + ], + "description": "Applied below every root; at most `limits.filesystem.scan.maxPatterns`\npatterns in total." + }, + "followSymlinks": { + "description": "Follow symlinks whose targets stay inside the filesystem root; others\nare skipped (`symlinkOutsideRoot`). Without it symlinks are skipped\n(`symlink`).", + "type": "boolean" + }, + "includeDigests": { + "description": "Return each selected file's `digest`.", + "type": "boolean" + }, + "limits": { + "allOf": [ + { + "$ref": "#/components/schemas/ScanLimits" + } + ], + "description": "Caller quotas; each at most its ceiling." + }, + "readContent": { + "description": "Return the bytes of every selected file in `data`.", + "type": "boolean" + }, + "roots": { + "description": "1 to `limits.filesystem.scan.maxRoots` roots.", + "items": { + "$ref": "#/components/schemas/ScanRoot" + }, + "type": "array" + } + }, + "required": [ + "roots", + "limits" + ], + "type": "object" + }, + "ScanResponse": { + "additionalProperties": false, + "description": "`fs/scan` response: one result per requested root, in request order.", + "properties": { + "roots": { + "description": "Never empty.", + "items": { + "$ref": "#/components/schemas/ScanRootResult" + }, + "type": "array" + } + }, + "required": [ + "roots" + ], + "type": "object" + }, + "ScanRoot": { + "additionalProperties": false, + "description": "One root of a scan.", + "properties": { + "ifNoneMatch": { + "anyOf": [ + { + "$ref": "#/components/schemas/ScanFingerprint" + } + ], + "description": "The root's previous `fingerprint`: when the selection is unchanged\nthe result is `unchanged` and carries no entries." + }, + "maxDepth": { + "description": "Overrides `filter.maxDepth` for this root. `0` examines only the\nroot itself: a `complete` result without entries proves it is an\nexisting directory, which is how discovery checks a working\ndirectory in the same call.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "ScanRootResult": { + "description": "The outcome of one root.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Every selected entry, plus what was skipped and why.", + "properties": { + "canonicalPath": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "Resolved absolute path of the root when it differs from `path`\n(the root is a symlink)." + }, + "entries": { + "items": { + "$ref": "#/components/schemas/ScanEntry" + }, + "type": "array" + }, + "fingerprint": { + "$ref": "#/components/schemas/ScanFingerprint" + }, + "kind": { + "const": "complete", + "type": "string" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "skipped": { + "description": "The first\n`MAX_LISTED_SKIPS` skipped\nentries, in walk order.", + "items": { + "$ref": "#/components/schemas/ScanSkip" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Total number of skipped entries, listed or not; present whenever\nit is not zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "path", + "fingerprint" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The selection matches `ifNoneMatch`.", + "properties": { + "fingerprint": { + "$ref": "#/components/schemas/ScanFingerprint" + }, + "kind": { + "const": "unchanged", + "type": "string" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "kind", + "path", + "fingerprint" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The root does not exist.", + "properties": { + "kind": { + "const": "missing", + "type": "string" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "kind", + "path" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The root could not be scanned; no entries are returned for it.", + "properties": { + "error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "kind": { + "const": "failed", + "type": "string" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "kind", + "path", + "error" + ], + "type": "object" + } + ] + }, + "ScanSkip": { + "additionalProperties": false, + "description": "An entry a scan left out.", + "properties": { + "path": { + "anyOf": [ + { + "$ref": "#/components/schemas/TreePath" + } + ], + "description": "Absent when the entry's name is not UTF-8." + }, + "reason": { + "$ref": "#/components/schemas/ScanSkipReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "ScanSkipReason": { + "description": "Why a scan left an entry out.", + "oneOf": [ + { + "const": "fileTooLarge", + "description": "Larger than `maxFileBytes`.", + "type": "string" + }, + { + "const": "symlink", + "description": "A symlink, and `followSymlinks` is not set.", + "type": "string" + }, + { + "const": "symlinkOutsideRoot", + "description": "A symlink whose target leaves the filesystem root, or a link cycle.", + "type": "string" + }, + { + "const": "specialFile", + "description": "A device, FIFO, socket or another special file.", + "type": "string" + }, + { + "const": "nonUtf8Name", + "description": "The name is not UTF-8.", + "type": "string" + }, + { + "const": "unreadable", + "description": "The entry vanished or could not be read during the walk.", + "type": "string" + } + ] + }, + "SearchCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/searchText` quotas, field for field.", + "properties": { + "maxBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxLineBytes": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxBytes", + "maxDurationMs", + "maxLineBytes" + ], + "type": "object" + }, + "SearchTextLimits": { + "additionalProperties": false, + "description": "Caller quotas of one search; each at most its advertised ceiling.", + "properties": { + "maxBytes": { + "description": "Bytes actually read across files.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "description": "Files searched, not merely visited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxLineBytes": { + "description": "Longer matching lines are cut and marked `lineTruncated`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxBytes", + "maxDurationMs", + "maxLineBytes" + ], + "type": "object" + }, + "SearchTextMatch": { + "additionalProperties": false, + "description": "One matching line.", + "properties": { + "line": { + "description": "The matching line without its terminator, cut at `maxLineBytes` on a\ncharacter boundary.", + "type": "string" + }, + "lineNumber": { + "description": "1-based.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "lineTruncated": { + "type": "boolean" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path", + "lineNumber", + "line" + ], + "type": "object" + }, + "SearchTextParams": { + "additionalProperties": false, + "description": "`fs/searchText` params: a bounded recursive regex search run by the\ndaemon. A file larger than the remaining byte budget is skipped and\ncounted; it never ends the walk.", + "properties": { + "filter": { + "$ref": "#/components/schemas/TreeFilter" + }, + "limits": { + "$ref": "#/components/schemas/SearchTextLimits" + }, + "pattern": { + "description": "Regular expression in the Rust `regex` crate's syntax, matched per\nline.", + "type": "string" + }, + "patternIgnoreCase": { + "description": "Match `pattern` case-insensitively. `filter.ignoreCase` governs the\npath patterns instead.", + "type": "boolean" + }, + "root": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "root", + "pattern", + "limits" + ], + "type": "object" + }, + "SearchTextResponse": { + "additionalProperties": false, + "description": "`fs/searchText` response.", + "properties": { + "elapsedMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "matches": { + "items": { + "$ref": "#/components/schemas/SearchTextMatch" + }, + "type": "array" + }, + "searchedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "searchedFileCount": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "skippedEntryCount": { + "description": "Entries the walk could not represent or list: names that are not\nUTF-8 and directories that could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "skippedFileCount": { + "description": "Files skipped because they exceeded the remaining byte budget, are\nbinary or could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "stopped": { + "anyOf": [ + { + "$ref": "#/components/schemas/SearchTextStop" + } + ], + "description": "Absent when the walk finished within its limits." + } + }, + "required": [ + "searchedFileCount", + "searchedBytes", + "elapsedMs" + ], + "type": "object" + }, + "SearchTextStop": { + "description": "Why a search stopped before finishing its walk.", + "oneOf": [ + { + "const": "matchLimit", + "description": "`maxMatches` matches were found.", + "type": "string" + }, + { + "const": "fileLimit", + "description": "`maxFiles` files were searched.", + "type": "string" + }, + { + "const": "byteLimit", + "description": "`maxBytes` bytes were read.", + "type": "string" + }, + { + "const": "timeLimit", + "description": "`maxDurationMs` ran out.", + "type": "string" + }, + { + "const": "responseLimit", + "description": "The next match would push the encoded matches past\n`MAX_RESULT_BYTES`.", + "type": "string" + } + ] + }, + "SecretString": { + "description": "A secret value carried on the wire, such as a `secretEnv` value, a\nregistration key or a stream token. It serializes as a plain string;\n`Debug` is redacted so structured logging never prints it.", + "type": "string" + }, + "SignalName": { + "description": "A POSIX signal name such as `SIGSEGV` or `SIGRTMIN+3`: `SIG` followed by\n1 to 16 uppercase letters, digits or `+`. Names, not numbers, because\nnumbers differ between platforms.", + "pattern": "^SIG[A-Z0-9+]{1,16}$", + "type": "string" + }, + "SpecialFilePolicy": { + "description": "Devices, FIFOs and sockets met during a capture.", + "oneOf": [ + { + "const": "reject", + "description": "Fail the operation (`notRegularFile`).", + "type": "string" + }, + { + "const": "skip", + "description": "Leave them out and report them in `skipped`.", + "type": "string" + } + ] + }, + "StartJobsParams": { + "additionalProperties": false, + "description": "`job/start` params. All-or-nothing: either every job is accepted (or\nreplayed) or none is.", + "properties": { + "jobs": { + "description": "1 to `limits.job.maxRequestJobs` jobs.", + "items": { + "$ref": "#/components/schemas/JobSpec" + }, + "type": "array" + }, + "requestId": { + "allOf": [ + { + "$ref": "#/components/schemas/JobRequestId" + } + ], + "description": "Idempotency key of this start. Retrying with the same request id and\nidentical jobs replays the existing jobs; reusing a job id with\nanother request id or spec is `failedPrecondition` (`jobIdInUse`)." + } + }, + "required": [ + "requestId", + "jobs" + ], + "type": "object" + }, + "StartJobsResponse": { + "additionalProperties": false, + "description": "`job/start` response: one summary per requested job, in request order.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/components/schemas/JobSummary" + }, + "type": "array" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "StartProcessParams": { + "additionalProperties": false, + "description": "`process/start` params.", + "properties": { + "command": { + "$ref": "#/components/schemas/CommandSpec" + }, + "processId": { + "allOf": [ + { + "$ref": "#/components/schemas/ProcessId" + } + ], + "description": "Caller-chosen handle, reserved before any side effect: a start whose\nid is already in use (running, retained or remembered by a tombstone)\nis `alreadyExists` and runs nothing." + }, + "pty": { + "anyOf": [ + { + "$ref": "#/components/schemas/PtySize" + } + ], + "description": "Run under a PTY of this size. The PTY master is the input: `input`\nmust then be `none` or `bytes`, and `process/write` is always\naccepted while the process runs. Requires\n`capabilities.process.pty`." + } + }, + "required": [ + "processId", + "command" + ], + "type": "object" + }, + "StartProcessResponse": { + "additionalProperties": false, + "description": "`process/start` response, sent once the process runs.", + "properties": { + "pid": { + "description": "OS pid of the root process, which is also its process group id.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "StoppedBy": { + "description": "Why the daemon stopped a process or job that did not exit on its own.\nSet only when the daemon signalled a root that was still running.", + "oneOf": [ + { + "const": "timeout", + "description": "`timeoutMs` expired: `SIGTERM`, the grace, then `SIGKILL`.", + "type": "string" + }, + { + "const": "terminate", + "description": "`process/terminate` with `terminate`, or `job/cancel` without\n`force`: `SIGTERM`, the grace, then `SIGKILL`.", + "type": "string" + }, + { + "const": "kill", + "description": "`process/terminate` with `kill`, or `job/cancel` with `force`:\n`SIGKILL` at once.", + "type": "string" + } + ] + }, + "SymlinkPolicy": { + "description": "Symlinks met during a capture.", + "oneOf": [ + { + "const": "expandInside", + "description": "Capture a link whose target stays inside the selection as the entry\nit points to; a link leaving the selection or a cycle fails the\noperation (`symlinkRejected`). A dangling link inside the selection\nis left out and reported in `skipped` as `symlink`.", + "type": "string" + }, + { + "const": "skip", + "description": "Leave every link out and report it in `skipped`.", + "type": "string" + }, + { + "const": "reject", + "description": "Fail on the first link.", + "type": "string" + } + ] + }, + "TerminateProcessParams": { + "additionalProperties": false, + "description": "`process/terminate` params.", + "properties": { + "processId": { + "$ref": "#/components/schemas/ProcessId" + }, + "signal": { + "$ref": "#/components/schemas/ProcessSignal" + } + }, + "required": [ + "processId", + "signal" + ], + "type": "object" + }, + "TerminateProcessResponse": { + "additionalProperties": false, + "description": "`process/terminate` response, sent once the signal was delivered (any\nescalation continues in the background).", + "properties": { + "running": { + "description": "True when the root process was still running when the signal was\nsent.", + "type": "boolean" + } + }, + "type": "object" + }, + "TransferCeilings": { + "additionalProperties": false, + "description": "Ceilings of `transfer/*`: capacity, page and chunk sizes, receipt\nretention, then the per-operation quotas.", + "properties": { + "maxActiveOperations": { + "description": "Operations not yet complete, aborted or failed, across connections;\na begin beyond it is `busy` (`transferCapacity`).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxContentChunkBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDepth": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxInventoryPageEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxManifestBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "receiptRetentionMs": { + "description": "How long a finished operation's receipt is answered.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxActiveOperations", + "maxContentChunkBytes", + "maxInventoryPageEntries", + "receiptRetentionMs", + "maxEntries", + "maxDepth", + "maxFileBytes", + "maxTotalBytes", + "maxManifestBytes", + "maxDurationMs" + ], + "type": "object" + }, + "TransferLimits": { + "additionalProperties": false, + "description": "Caller quotas of one operation; each at most its advertised ceiling.", + "properties": { + "maxDepth": { + "description": "The selected root has depth 0.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "description": "From begin to commit.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "description": "Entries, the selected root included.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxManifestBytes": { + "description": "Encoded inventory size.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxEntries", + "maxDepth", + "maxFileBytes", + "maxTotalBytes", + "maxManifestBytes", + "maxDurationMs" + ], + "type": "object" + }, + "TransferOnExisting": { + "description": "What a materialize does with an existing destination.", + "oneOf": [ + { + "const": "error", + "description": "Refuse at begin (`alreadyExists`).", + "type": "string" + }, + { + "const": "replace", + "description": "Replace it atomically at commit.", + "type": "string" + } + ] + }, + "TransferOperationId": { + "description": "`operationId` of one transfer operation.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "TransferOperationParams": { + "additionalProperties": false, + "description": "Params of `transfer/advance`, `transfer/getStatus`, `transfer/commit`\nand `transfer/abort`: each names only the operation.", + "properties": { + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId" + ], + "type": "object" + }, + "TransferPhase": { + "description": "Where an operation stands; see the module docs for the order per\nselection kind.", + "oneOf": [ + { + "enum": [ + "complete", + "aborted" + ], + "type": "string" + }, + { + "const": "scanning", + "description": "The daemon walks a tree: the capture source, or the destination of a\nmaterialize. Advance until the phase changes.", + "type": "string" + }, + { + "const": "inventory", + "description": "Capture: the inventory is sealed and pageable. Materialize: waiting\nfor appended entries.", + "type": "string" + }, + { + "const": "content", + "description": "Waiting for the content of `transfer/listMissing` digests.", + "type": "string" + }, + { + "const": "staging", + "description": "The daemon assembles the staged tree. Advance until `ready`.", + "type": "string" + }, + { + "const": "ready", + "description": "Ready for `transfer/commit`.", + "type": "string" + }, + { + "const": "failed", + "description": "Stopped by an error; see `error`.", + "type": "string" + } + ] + }, + "TransferSelection": { + "description": "What one operation transfers.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read a file or tree out of the environment.", + "properties": { + "filter": { + "$ref": "#/components/schemas/TreeFilter" + }, + "kind": { + "const": "capture", + "type": "string" + }, + "source": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "specialFiles": { + "$ref": "#/components/schemas/SpecialFilePolicy" + }, + "symlinks": { + "$ref": "#/components/schemas/SymlinkPolicy" + } + }, + "required": [ + "kind", + "source" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Write a tree into the environment, replacing `destination` atomically\nat commit.", + "properties": { + "destination": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "kind": { + "const": "materialize", + "type": "string" + }, + "onExisting": { + "$ref": "#/components/schemas/TransferOnExisting" + } + }, + "required": [ + "kind", + "destination" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Write exactly one file with `fs/writeFile` semantics: staged,\natomic, an existing target keeps its permissions, an in-root symlink\nat `destination` is followed.", + "properties": { + "create": { + "$ref": "#/components/schemas/WriteCondition" + }, + "destination": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "executable": { + "type": "boolean" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ] + }, + "kind": { + "const": "writeFile", + "type": "string" + } + }, + "required": [ + "kind", + "destination" + ], + "type": "object" + } + ] + }, + "TransferSkip": { + "additionalProperties": false, + "description": "An entry a capture left out.", + "properties": { + "path": { + "anyOf": [ + { + "$ref": "#/components/schemas/TreePath" + } + ], + "description": "Absent when the entry's name is not UTF-8." + }, + "reason": { + "$ref": "#/components/schemas/TransferSkipReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "TransferSkipReason": { + "description": "Why a capture left an entry out.", + "oneOf": [ + { + "const": "symlink", + "description": "`symlinks: skip`, or a dangling link under `symlinks: expandInside`.", + "type": "string" + }, + { + "const": "specialFile", + "description": "`specialFiles: skip`.", + "type": "string" + }, + { + "const": "nonUtf8Name", + "description": "The name is not UTF-8; such entries are always skipped.", + "type": "string" + } + ] + }, + "TransferStatus": { + "additionalProperties": false, + "description": "The operation's status: the response of `transfer/begin`,\n`transfer/advance`, `transfer/getStatus`, `transfer/append`,\n`transfer/commit` and `transfer/abort`.", + "properties": { + "deadlineAtMs": { + "description": "When `maxDurationMs` runs out for an active operation.", + "format": "int64", + "type": "integer" + }, + "entryCount": { + "description": "Entries in the inventory so far.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProtocolError" + } + ], + "description": "Why the operation failed; present exactly when `phase` is `failed`." + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + }, + "phase": { + "$ref": "#/components/schemas/TransferPhase" + }, + "receiptExpiresAtMs": { + "description": "When the receipt of a finished (complete, aborted or failed)\noperation stops being answered; afterwards calls are `notFound`\n(`expired`).", + "format": "int64", + "type": "integer" + }, + "reusedBytes": { + "description": "Content bytes satisfied from files already at the destination.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "skipped": { + "description": "The first\n`MAX_LISTED_SKIPS` entries a\ncapture left out under its policies.", + "items": { + "$ref": "#/components/schemas/TransferSkip" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Total number of entries left out, listed or not; present whenever it\nis not zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "totalBytes": { + "description": "File bytes in the inventory so far.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "transferredBytes": { + "description": "Content bytes moved over the wire.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "operationId", + "phase", + "entryCount", + "totalBytes", + "transferredBytes", + "reusedBytes" + ], + "type": "object" + }, + "TreeContent": { + "description": "What a tree entry is: a scan entry or a transfer inventory entry.\nDigests identify raw file bytes, not paths or modes.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "directory", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "digest": { + "anyOf": [ + { + "$ref": "#/components/schemas/ContentDigest" + } + ], + "description": "Scans: present when `includeDigests` was set. Transfer\ninventories: always present; an appended file entry without it\nis `invalidParams`." + }, + "executable": { + "type": "boolean" + }, + "kind": { + "const": "file", + "type": "string" + }, + "sizeBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "sizeBytes" + ], + "type": "object" + } + ] + }, + "TreeFilter": { + "additionalProperties": false, + "description": "Which entries a walk visits. The default visits everything except\nhidden and ignored entries.", + "properties": { + "exclude": { + "description": "Drop entries matching any pattern; a matching directory is pruned.", + "items": { + "type": "string" + }, + "type": "array" + }, + "ignoreCase": { + "description": "Match patterns case-insensitively.", + "type": "boolean" + }, + "include": { + "description": "Select entries matching any pattern; empty selects every entry.", + "items": { + "type": "string" + }, + "type": "array" + }, + "includeHidden": { + "description": "Visit entries whose name starts with `.`.", + "type": "boolean" + }, + "includeIgnored": { + "description": "Visit entries that ignore files exclude.", + "type": "boolean" + }, + "maxDepth": { + "description": "Deepest entry depth visited: the root's direct children are depth\n1, so `1` selects direct children only. Unbounded when absent.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "TreePath": { + "description": "A relative, normalized path below a selected root: segments joined by\n`/`, without empty, `.` or `..` segments. The single segment `.` names\nthe root itself. At most 4096 bytes. Used for scan, inventory and\ntransfer entries.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(\\.|(?!\\.\\.?(/|$))[^/\\u0000]+(/(?!\\.\\.?(/|$))[^/\\u0000]+)*)$", + "type": "string" + }, + "WriteCondition": { + "description": "When `fs/writeFile` may write.", + "oneOf": [ + { + "const": "always", + "description": "Create or replace.", + "type": "string" + }, + { + "const": "ifAbsent", + "description": "Only create; an existing entry is `failedPrecondition`\n(`createCondition`).", + "type": "string" + }, + { + "const": "ifPresent", + "description": "Only replace; a missing file is `failedPrecondition`\n(`createCondition`).", + "type": "string" + } + ] + }, + "WriteFileParams": { + "additionalProperties": false, + "description": "`fs/writeFile` params. The write is staged in the target's directory and\npublished by an atomic rename: readers see the old or the new content,\nnever a mix. An existing target keeps its permissions. A symlink at\n`path` whose target stays inside the root is followed like any path: the\nfinal target is replaced and the link itself is kept (a dangling link\ncreates its target, subject to `create`); a link that leaves the root is\n`permissionDenied` (`outsideRoot`). A special file is\n`failedPrecondition` (`notRegularFile`), a directory\n`failedPrecondition` (`isDirectory`). The parent of the final target\nmust exist.", + "properties": { + "create": { + "$ref": "#/components/schemas/WriteCondition" + }, + "data": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "At most `limits.filesystem.maxWriteFileBytes`; larger files go\nthrough `transfer/*` with a `writeFile` selection." + }, + "executable": { + "description": "Set the execute bits (the read bits' pattern). Without it a new file\ngets the default mode and an existing file keeps its mode.", + "type": "boolean" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Write only when the current file's version equals this token;\notherwise `failedPrecondition` (`ifMatchFailed`)." + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path", + "data" + ], + "type": "object" + }, + "WriteFileResponse": { + "additionalProperties": false, + "description": "`fs/writeFile` response.", + "properties": { + "version": { + "allOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Version of the written file." + } + }, + "required": [ + "version" + ], + "type": "object" + }, + "WriteProcessParams": { + "additionalProperties": false, + "description": "`process/write` params.", + "properties": { + "closeInput": { + "description": "Close the input after `data`: end of file on a pipe, `VEOF` on a\nPTY. Applied only when every byte of `data` was accepted.", + "type": "boolean" + }, + "data": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "Bytes for the process's input pipe or PTY." + }, + "processId": { + "$ref": "#/components/schemas/ProcessId" + } + }, + "required": [ + "processId" + ], + "type": "object" + }, + "WriteProcessResponse": { + "additionalProperties": false, + "description": "`process/write` response.", + "properties": { + "acceptedBytes": { + "description": "Leading bytes of `data` queued to the process's writer. Fewer than\nsent (possibly 0) means the input queue is full: resend the rest\nlater.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "acceptedBytes" + ], + "type": "object" + }, + "WriteTransferContentParams": { + "additionalProperties": false, + "description": "`transfer/write` params: upload content for a missing digest. Chunks for\none digest arrive in order; an identical chunk may be retried.\nDifferent digests may be uploaded concurrently.", + "properties": { + "data": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "At most `limits.filesystem.transfer.maxContentChunkBytes`." + }, + "digest": { + "$ref": "#/components/schemas/ContentDigest" + }, + "offset": { + "description": "Must equal the digest's next expected offset.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "digest", + "offset", + "data" + ], + "type": "object" + }, + "WriteTransferContentResponse": { + "additionalProperties": false, + "description": "`transfer/write` response.", + "properties": { + "nextOffset": { + "description": "The digest's next expected offset; equal to its size once complete\nand verified.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "nextOffset" + ], + "type": "object" + } + } + }, + "info": { + "description": "JSON-RPC 2.0 over WebSocket text frames of at most 16777216 bytes. Exact-match protocolVersion 3. Errors carry code, message and data.kind; see x-lightspeed-error-kinds.", + "title": "Lightspeed environment data plane", + "version": "3" + }, + "methods": [ + { + "name": "initialize", + "paramStructure": "by-name", + "params": [ + { + "name": "clientInfo", + "required": true, + "schema": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + { + "name": "protocolVersion", + "required": true, + "schema": { + "description": "The client's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/InitializeResponse" + } + }, + "summary": "Check the protocol version and read the daemon's instance id, capabilities and ceilings.", + "x-lightspeed-activity": "observation", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "handshake", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "InitializeParams", + "x-lightspeed-result-type": "InitializeResponse" + }, + { + "name": "env/getIdle", + "paramStructure": "by-name", + "params": [], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/GetIdleResponse" + } + }, + "summary": "Report how long the environment has been idle.", + "x-lightspeed-activity": "observation", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "env", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "GetIdleParams", + "x-lightspeed-result-type": "GetIdleResponse" + }, + { + "name": "fs/readFile", + "paramStructure": "by-name", + "params": [ + { + "name": "maxBytes", + "required": true, + "schema": { + "description": "Most bytes to return, at most `limits.filesystem.maxReadFileBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "offset", + "required": false, + "schema": { + "description": "First byte to return; 0 by default.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadFileResponse" + } + }, + "summary": "Read a byte range of a regular file.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadFileParams", + "x-lightspeed-result-type": "ReadFileResponse" + }, + { + "name": "fs/writeFile", + "paramStructure": "by-name", + "params": [ + { + "name": "create", + "required": false, + "schema": { + "$ref": "#/components/schemas/WriteCondition" + } + }, + { + "name": "data", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "At most `limits.filesystem.maxWriteFileBytes`; larger files go\nthrough `transfer/*` with a `writeFile` selection." + } + }, + { + "name": "executable", + "required": false, + "schema": { + "description": "Set the execute bits (the read bits' pattern). Without it a new file\ngets the default mode and an existing file keeps its mode.", + "type": "boolean" + } + }, + { + "name": "ifMatch", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Write only when the current file's version equals this token;\notherwise `failedPrecondition` (`ifMatchFailed`)." + } + }, + { + "name": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/WriteFileResponse" + } + }, + "summary": "Write a file atomically, optionally conditioned on its version.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystemWrite", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "WriteFileParams", + "x-lightspeed-result-type": "WriteFileResponse" + }, + { + "name": "fs/createDirectory", + "paramStructure": "by-name", + "params": [ + { + "name": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + { + "name": "recursive", + "required": false, + "schema": { + "description": "Create missing parents, and succeed when the directory exists.", + "type": "boolean" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/CreateDirectoryResponse" + } + }, + "summary": "Create a directory.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystemWrite", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "CreateDirectoryParams", + "x-lightspeed-result-type": "CreateDirectoryResponse" + }, + { + "name": "fs/getMetadata", + "paramStructure": "by-name", + "params": [ + { + "name": "followSymlinks", + "required": false, + "schema": { + "description": "Describe the symlink's target instead of the link (subject to\nconfinement).", + "type": "boolean" + } + }, + { + "name": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/GetMetadataResponse" + } + }, + "summary": "Describe one entry.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "GetMetadataParams", + "x-lightspeed-result-type": "GetMetadataResponse" + }, + { + "name": "fs/readDirectory", + "paramStructure": "by-name", + "params": [ + { + "name": "cursor", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/DirectoryCursor" + } + ], + "description": "`nextCursor` of the previous page; a cursor this daemon did not mint\nis `invalidParams`." + } + }, + { + "name": "limit", + "required": true, + "schema": { + "description": "Most entries per page, 1 to `limits.filesystem.maxDirectoryEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "path", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "A directory, or a symlink to one inside the root." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadDirectoryResponse" + } + }, + "summary": "List one page of a directory's entries.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadDirectoryParams", + "x-lightspeed-result-type": "ReadDirectoryResponse" + }, + { + "name": "fs/remove", + "paramStructure": "by-name", + "params": [ + { + "name": "force", + "required": false, + "schema": { + "description": "Succeed when the path does not exist.", + "type": "boolean" + } + }, + { + "name": "path", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "A symlink is removed, never followed." + } + }, + { + "name": "recursive", + "required": false, + "schema": { + "description": "Remove a directory and its contents.", + "type": "boolean" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/RemoveResponse" + } + }, + "summary": "Remove a file, symlink or directory.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystemWrite", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "RemoveParams", + "x-lightspeed-result-type": "RemoveResponse" + }, + { + "name": "fs/rename", + "paramStructure": "by-name", + "params": [ + { + "name": "destination", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + { + "name": "ifMatch", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Rename only when the source's version equals this token." + } + }, + { + "name": "overwrite", + "required": false, + "schema": { + "description": "Replace an existing destination file; otherwise an existing\ndestination is `alreadyExists`. A directory destination is never\nreplaced.", + "type": "boolean" + } + }, + { + "name": "source", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/RenameResponse" + } + }, + "summary": "Rename an entry atomically.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystemWrite", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "RenameParams", + "x-lightspeed-result-type": "RenameResponse" + }, + { + "name": "fs/searchText", + "paramStructure": "by-name", + "params": [ + { + "name": "filter", + "required": false, + "schema": { + "$ref": "#/components/schemas/TreeFilter" + } + }, + { + "name": "limits", + "required": true, + "schema": { + "$ref": "#/components/schemas/SearchTextLimits" + } + }, + { + "name": "pattern", + "required": true, + "schema": { + "description": "Regular expression in the Rust `regex` crate's syntax, matched per\nline.", + "type": "string" + } + }, + { + "name": "patternIgnoreCase", + "required": false, + "schema": { + "description": "Match `pattern` case-insensitively. `filter.ignoreCase` governs the\npath patterns instead.", + "type": "boolean" + } + }, + { + "name": "root", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/SearchTextResponse" + } + }, + "summary": "Search file contents below a root with a regular expression.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "SearchTextParams", + "x-lightspeed-result-type": "SearchTextResponse" + }, + { + "name": "fs/globFiles", + "paramStructure": "by-name", + "params": [ + { + "name": "filter", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeFilter" + } + ], + "description": "`include` must not be empty." + } + }, + { + "name": "limits", + "required": true, + "schema": { + "$ref": "#/components/schemas/GlobFilesLimits" + } + }, + { + "name": "root", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/GlobFilesResponse" + } + }, + "summary": "List files below a root that match patterns.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "GlobFilesParams", + "x-lightspeed-result-type": "GlobFilesResponse" + }, + { + "name": "fs/scan", + "paramStructure": "by-name", + "params": [ + { + "name": "filter", + "required": false, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeFilter" + } + ], + "description": "Applied below every root; at most `limits.filesystem.scan.maxPatterns`\npatterns in total." + } + }, + { + "name": "followSymlinks", + "required": false, + "schema": { + "description": "Follow symlinks whose targets stay inside the filesystem root; others\nare skipped (`symlinkOutsideRoot`). Without it symlinks are skipped\n(`symlink`).", + "type": "boolean" + } + }, + { + "name": "includeDigests", + "required": false, + "schema": { + "description": "Return each selected file's `digest`.", + "type": "boolean" + } + }, + { + "name": "limits", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ScanLimits" + } + ], + "description": "Caller quotas; each at most its ceiling." + } + }, + { + "name": "readContent", + "required": false, + "schema": { + "description": "Return the bytes of every selected file in `data`.", + "type": "boolean" + } + }, + { + "name": "roots", + "required": true, + "schema": { + "description": "1 to `limits.filesystem.scan.maxRoots` roots.", + "items": { + "$ref": "#/components/schemas/ScanRoot" + }, + "type": "array" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ScanResponse" + } + }, + "summary": "Observe small trees below several roots, each reported on its own.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "observation", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ScanParams", + "x-lightspeed-result-type": "ScanResponse" + }, + { + "name": "transfer/begin", + "paramStructure": "by-name", + "params": [ + { + "name": "limits", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferLimits" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + { + "name": "selection", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferSelection" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Start or resume a transfer operation.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "BeginTransferParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/advance", + "paramStructure": "by-name", + "params": [ + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Do a bounded step of scanning or staging work.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TransferOperationParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/getStatus", + "paramStructure": "by-name", + "params": [ + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Read an operation's status or receipt.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TransferOperationParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/readInventory", + "paramStructure": "by-name", + "params": [ + { + "name": "index", + "required": true, + "schema": { + "description": "Index of the first entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "limit", + "required": true, + "schema": { + "description": "1 to `limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadTransferInventoryResponse" + } + }, + "summary": "Read one page of a sealed inventory.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadTransferInventoryParams", + "x-lightspeed-result-type": "ReadTransferInventoryResponse" + }, + { + "name": "transfer/append", + "paramStructure": "by-name", + "params": [ + { + "name": "entries", + "required": true, + "schema": { + "description": "At most `limits.filesystem.transfer.maxInventoryPageEntries`.", + "items": { + "$ref": "#/components/schemas/InventoryEntry" + }, + "type": "array" + } + }, + { + "name": "index", + "required": true, + "schema": { + "description": "Index of the page's first entry: the number of entries appended\nbefore it.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "last", + "required": false, + "schema": { + "description": "This page completes the inventory.", + "type": "boolean" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Append incoming inventory entries.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "AppendTransferInventoryParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/listMissing", + "paramStructure": "by-name", + "params": [ + { + "name": "limit", + "required": true, + "schema": { + "description": "Most digests to return, 1 to\n`limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ListTransferMissingResponse" + } + }, + "summary": "List digests whose content the daemon still needs.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ListTransferMissingParams", + "x-lightspeed-result-type": "ListTransferMissingResponse" + }, + { + "name": "transfer/read", + "paramStructure": "by-name", + "params": [ + { + "name": "digest", + "required": true, + "schema": { + "$ref": "#/components/schemas/ContentDigest" + } + }, + { + "name": "maxBytes", + "required": true, + "schema": { + "description": "1 to `limits.filesystem.transfer.maxContentChunkBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "offset", + "required": true, + "schema": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadTransferContentResponse" + } + }, + "summary": "Read captured content by digest.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadTransferContentParams", + "x-lightspeed-result-type": "ReadTransferContentResponse" + }, + { + "name": "transfer/write", + "paramStructure": "by-name", + "params": [ + { + "name": "data", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "At most `limits.filesystem.transfer.maxContentChunkBytes`." + } + }, + { + "name": "digest", + "required": true, + "schema": { + "$ref": "#/components/schemas/ContentDigest" + } + }, + { + "name": "offset", + "required": true, + "schema": { + "description": "Must equal the digest's next expected offset.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/WriteTransferContentResponse" + } + }, + "summary": "Upload content for a missing digest.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "WriteTransferContentParams", + "x-lightspeed-result-type": "WriteTransferContentResponse" + }, + { + "name": "transfer/commit", + "paramStructure": "by-name", + "params": [ + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Finish a ready operation: publish a staged tree, or release a capture.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TransferOperationParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/abort", + "paramStructure": "by-name", + "params": [ + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Abandon an operation and discard its staging.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TransferOperationParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "process/start", + "paramStructure": "by-name", + "params": [ + { + "name": "command", + "required": true, + "schema": { + "$ref": "#/components/schemas/CommandSpec" + } + }, + { + "name": "processId", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ProcessId" + } + ], + "description": "Caller-chosen handle, reserved before any side effect: a start whose\nid is already in use (running, retained or remembered by a tombstone)\nis `alreadyExists` and runs nothing." + } + }, + { + "name": "pty", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/PtySize" + } + ], + "description": "Run under a PTY of this size. The PTY master is the input: `input`\nmust then be `none` or `bytes`, and `process/write` is always\naccepted while the process runs. Requires\n`capabilities.process.pty`." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/StartProcessResponse" + } + }, + "summary": "Start a process under a caller-chosen handle.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "process", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "process", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "StartProcessParams", + "x-lightspeed-result-type": "StartProcessResponse" + }, + { + "name": "process/read", + "paramStructure": "by-name", + "params": [ + { + "name": "ack", + "required": false, + "schema": { + "description": "Acknowledge every byte before this offset first: the cursor moves to\nit (never backwards) whether or not this read's response arrives. A\ncaller that tracks positions reads with `after` and acknowledges\nwith `ack` for exactly-once delivery.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "after", + "required": false, + "schema": { + "description": "Start offset. Absent: the daemon cursor, which this read advances to\n`next` once its response is written. Present: a non-consuming read\nthat leaves the cursor alone.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "maxBytes", + "required": true, + "schema": { + "description": "Most `data` bytes to return, at most `limits.process.maxReadBytes`.\n0 reads only the state.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "processId", + "required": true, + "schema": { + "$ref": "#/components/schemas/ProcessId" + } + }, + { + "name": "returnOn", + "required": true, + "schema": { + "$ref": "#/components/schemas/ReturnOn" + } + }, + { + "name": "tailBytes", + "required": false, + "schema": { + "description": "Start at `end - tailBytes` instead (clamped at 0). Non-consuming;\nexcludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "waitMs", + "required": true, + "schema": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadProcessResponse" + } + }, + "summary": "Wait for and read a page of a process's output and its state.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "process", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "process", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadProcessParams", + "x-lightspeed-result-type": "ReadProcessResponse" + }, + { + "name": "process/write", + "paramStructure": "by-name", + "params": [ + { + "name": "closeInput", + "required": false, + "schema": { + "description": "Close the input after `data`: end of file on a pipe, `VEOF` on a\nPTY. Applied only when every byte of `data` was accepted.", + "type": "boolean" + } + }, + { + "name": "data", + "required": false, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "Bytes for the process's input pipe or PTY." + } + }, + { + "name": "processId", + "required": true, + "schema": { + "$ref": "#/components/schemas/ProcessId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/WriteProcessResponse" + } + }, + "summary": "Queue bytes for a process's input pipe or PTY.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "process", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "process", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "WriteProcessParams", + "x-lightspeed-result-type": "WriteProcessResponse" + }, + { + "name": "process/terminate", + "paramStructure": "by-name", + "params": [ + { + "name": "processId", + "required": true, + "schema": { + "$ref": "#/components/schemas/ProcessId" + } + }, + { + "name": "signal", + "required": true, + "schema": { + "$ref": "#/components/schemas/ProcessSignal" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TerminateProcessResponse" + } + }, + "summary": "Interrupt, terminate or kill a process's group.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "process", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "process", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TerminateProcessParams", + "x-lightspeed-result-type": "TerminateProcessResponse" + }, + { + "name": "job/start", + "paramStructure": "by-name", + "params": [ + { + "name": "jobs", + "required": true, + "schema": { + "description": "1 to `limits.job.maxRequestJobs` jobs.", + "items": { + "$ref": "#/components/schemas/JobSpec" + }, + "type": "array" + } + }, + { + "name": "requestId", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/JobRequestId" + } + ], + "description": "Idempotency key of this start. Retrying with the same request id and\nidentical jobs replays the existing jobs; reusing a job id with\nanother request id or spec is `failedPrecondition` (`jobIdInUse`)." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/StartJobsResponse" + } + }, + "summary": "Start jobs, or replay an identical earlier start.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "job", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "job", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "StartJobsParams", + "x-lightspeed-result-type": "StartJobsResponse" + }, + { + "name": "job/read", + "paramStructure": "by-name", + "params": [ + { + "name": "jobs", + "required": true, + "schema": { + "description": "1 to `limits.job.maxRequestJobs` entries, each with its own position.", + "items": { + "$ref": "#/components/schemas/JobReadRequest" + }, + "type": "array" + } + }, + { + "name": "maxBytes", + "required": true, + "schema": { + "description": "Most `data` bytes across all entries, at most\n`limits.job.maxReadBytes`, filled in request order: an entry reached\nafter the budget is spent returns its summary and an empty page at\nits start position. 0 reads only summaries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "returnOn", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ReturnOn" + } + ], + "description": "`anyOutput`: return when any entry has output past its position or\nchanged status; `exit`: when every entry is terminal; `window`: wait\nthe whole window." + } + }, + { + "name": "waitMs", + "required": true, + "schema": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadJobsResponse" + } + }, + "summary": "Wait for and read jobs' summaries and output pages.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "observation", + "x-lightspeed-capability": "job", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "job", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadJobsParams", + "x-lightspeed-result-type": "ReadJobsResponse" + }, + { + "name": "job/cancel", + "paramStructure": "by-name", + "params": [ + { + "name": "force", + "required": false, + "schema": { + "description": "Kill at once instead of `SIGTERM`, `limits.job.cancelGraceMs`, then\n`SIGKILL`.", + "type": "boolean" + } + }, + { + "name": "jobIds", + "required": true, + "schema": { + "description": "1 to `limits.job.maxRequestJobs` ids.", + "items": { + "$ref": "#/components/schemas/JobId" + }, + "type": "array" + } + }, + { + "name": "scope", + "required": false, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/JobCancelScope" + } + ], + "description": "`job` by default." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/CancelJobsResponse" + } + }, + "summary": "Cancel jobs and optionally their dependents.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "job", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "job", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "CancelJobsParams", + "x-lightspeed-result-type": "CancelJobsResponse" + }, + { + "name": "$/cancelRequest", + "paramStructure": "by-name", + "params": [ + { + "name": "id", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/Id" + } + ], + "description": "Id of the request to abort." + } + } + ], + "summary": "Abort a pending abortable request.", + "x-lightspeed-activity": "observation", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "envelope", + "x-lightspeed-kind": "notification", + "x-lightspeed-params-type": "CancelRequestParams" + } + ], + "openrpc": "1.3.2", + "x-lightspeed-error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "x-lightspeed-error-kinds": [ + { + "code": -32700, + "kind": "parseError", + "retryable": false + }, + { + "code": -32600, + "kind": "invalidRequest", + "retryable": false + }, + { + "code": -32601, + "kind": "methodNotFound", + "retryable": false + }, + { + "code": -32602, + "kind": "invalidParams", + "retryable": false + }, + { + "code": -32002, + "kind": "notInitialized", + "retryable": false + }, + { + "code": -32005, + "kind": "versionMismatch", + "retryable": false + }, + { + "code": -32004, + "kind": "notFound", + "retryable": false + }, + { + "code": -32009, + "kind": "alreadyExists", + "retryable": false + }, + { + "code": -32007, + "kind": "failedPrecondition", + "retryable": false + }, + { + "code": -32003, + "kind": "permissionDenied", + "retryable": false + }, + { + "code": -32006, + "kind": "capabilityUnavailable", + "retryable": false + }, + { + "code": -32008, + "kind": "busy", + "retryable": true + }, + { + "code": -32015, + "kind": "deadlineExceeded", + "retryable": false + }, + { + "code": -32016, + "kind": "expired", + "retryable": false + }, + { + "code": -32017, + "kind": "aborted", + "retryable": true + }, + { + "code": -32013, + "kind": "payloadTooLarge", + "retryable": false + }, + { + "code": -32018, + "kind": "processFailed", + "retryable": false + }, + { + "code": -32019, + "kind": "unavailable", + "retryable": true + }, + { + "code": -32603, + "kind": "internal", + "retryable": false + } + ], + "x-lightspeed-plane": "data" +} diff --git a/crates/environment-protocol/contract/data.schema.json b/crates/environment-protocol/contract/data.schema.json new file mode 100644 index 000000000..40e77831e --- /dev/null +++ b/crates/environment-protocol/contract/data.schema.json @@ -0,0 +1,4285 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "AppendTransferInventoryParams": { + "additionalProperties": false, + "description": "`transfer/append` params: add incoming inventory entries in order.\nRetrying an identical page is safe.", + "properties": { + "entries": { + "description": "At most `limits.filesystem.transfer.maxInventoryPageEntries`.", + "items": { + "$ref": "#/definitions/InventoryEntry" + }, + "type": "array" + }, + "index": { + "description": "Index of the page's first entry: the number of entries appended\nbefore it.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "last": { + "description": "This page completes the inventory.", + "type": "boolean" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "index", + "entries" + ], + "type": "object" + }, + "BeginTransferParams": { + "additionalProperties": false, + "description": "`transfer/begin` params. Idempotent: a repeat with identical input\nreturns the operation's status; different input under the same id is\n`failedPrecondition` (`operationIdInUse`).", + "properties": { + "limits": { + "$ref": "#/definitions/TransferLimits" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + }, + "selection": { + "$ref": "#/definitions/TransferSelection" + } + }, + "required": [ + "operationId", + "selection", + "limits" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "ByteChunk": { + "contentEncoding": "base64", + "description": "Payload bytes, always named `data` on the wire, as padded standard\nbase64.", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$", + "type": "string" + }, + "CancelJobsParams": { + "additionalProperties": false, + "description": "`job/cancel` params.", + "properties": { + "force": { + "description": "Kill at once instead of `SIGTERM`, `limits.job.cancelGraceMs`, then\n`SIGKILL`.", + "type": "boolean" + }, + "jobIds": { + "description": "1 to `limits.job.maxRequestJobs` ids.", + "items": { + "$ref": "#/definitions/JobId" + }, + "type": "array" + }, + "scope": { + "allOf": [ + { + "$ref": "#/definitions/JobCancelScope" + } + ], + "description": "`job` by default." + } + }, + "required": [ + "jobIds" + ], + "type": "object" + }, + "CancelJobsResponse": { + "additionalProperties": false, + "description": "`job/cancel` response: one entry per affected job, named jobs first in\nrequest order, then dependents, at most `limits.job.maxRequestJobs`\nentries in total.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/definitions/JobSummaryEntry" + }, + "type": "array" + }, + "unlistedJobCount": { + "description": "Dependents cancelled as well but not listed, beyond the entry limit.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "CancelRequestParams": { + "additionalProperties": false, + "description": "`$/cancelRequest` params.", + "properties": { + "id": { + "allOf": [ + { + "$ref": "#/definitions/Id" + } + ], + "description": "Id of the request to abort." + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "adoptTarget", + "description": "The controller's `adoptTarget` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "CommandSpec": { + "additionalProperties": false, + "description": "What to run. Shared by `process/start` and `job/start`.", + "properties": { + "argv": { + "description": "Program and arguments; `argv[0]` is resolved through `PATH`. Not\nempty.", + "items": { + "type": "string" + }, + "type": "array" + }, + "cwd": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "Working directory; the daemon's `defaultCwd` when absent." + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "description": "Variables added to the daemon's scrubbed environment.", + "type": "object" + }, + "input": { + "allOf": [ + { + "$ref": "#/definitions/ExecInput" + } + ], + "description": "Standard input; none by default." + }, + "secretEnv": { + "additionalProperties": { + "$ref": "#/definitions/SecretString" + }, + "description": "Like `env`, but every value is redacted from the command's output.", + "type": "object" + }, + "timeoutMs": { + "description": "Stop deadline from start. On expiry the daemon terminates the process\ngroup (`SIGTERM`, the advertised grace, then `SIGKILL`) and records\n`stoppedBy: timeout`. Absent means no deadline.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "argv" + ], + "type": "object" + }, + "ContentDigest": { + "description": "`sha256:` followed by 64 lowercase hex digits: the digest of a file's raw\nbytes, independent of its path and mode.", + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + "CreateDirectoryParams": { + "additionalProperties": false, + "description": "`fs/createDirectory` params.", + "properties": { + "path": { + "$ref": "#/definitions/EnvironmentPath" + }, + "recursive": { + "description": "Create missing parents, and succeed when the directory exists.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "CreateDirectoryResponse": { + "additionalProperties": false, + "description": "`fs/createDirectory` response.", + "type": "object" + }, + "DaemonInstanceId": { + "description": "`daemonInstanceId`: random per daemon start. A change between two\nconnections means the daemon restarted and every process handle from\nthe earlier instance is gone.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DataCapabilities": { + "additionalProperties": false, + "description": "Capability families. A present family means its base methods exist; a\nfamily's fields are facts that vary per instance. A request to an\nabsent family, or one needing an absent fact, is\n`capabilityUnavailable`.", + "properties": { + "filesystem": { + "anyOf": [ + { + "$ref": "#/definitions/FilesystemCapabilities" + } + ], + "description": "`fs/*` (search, glob and scan included) and `transfer/*`." + }, + "job": { + "anyOf": [ + { + "$ref": "#/definitions/JobCapabilities" + } + ], + "description": "`job/*`." + }, + "process": { + "anyOf": [ + { + "$ref": "#/definitions/ProcessCapabilities" + } + ], + "description": "`process/*`." + } + }, + "type": "object" + }, + "DataLimits": { + "additionalProperties": false, + "description": "Operational ceilings this daemon enforces. A request above a ceiling is\n`invalidParams` naming the field and the ceiling; nothing is clamped.", + "properties": { + "filesystem": { + "$ref": "#/definitions/FilesystemLimits" + }, + "job": { + "$ref": "#/definitions/JobLimits" + }, + "maxConcurrentRequests": { + "description": "Requests in flight per connection before `busy`; at least\n`MIN_CONCURRENT_REQUESTS`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxWaitMs": { + "description": "Largest `waitMs` of `process/read` and `job/read`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "process": { + "$ref": "#/definitions/ProcessLimits" + } + }, + "required": [ + "maxConcurrentRequests", + "maxWaitMs", + "process", + "job", + "filesystem" + ], + "type": "object" + }, + "DirectoryCursor": { + "description": "`cursor` and `nextCursor` of `fs/readDirectory`: an opaque position\nthe daemon mints, echoed back unchanged.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DirectoryEntry": { + "additionalProperties": false, + "description": "One `fs/readDirectory` entry.", + "properties": { + "metadata": { + "allOf": [ + { + "$ref": "#/definitions/FileMetadata" + } + ], + "description": "The entry itself, not a symlink's target. Entries that vanish while\nthe page is built are left out." + }, + "name": { + "$ref": "#/definitions/EntryName" + } + }, + "required": [ + "name", + "metadata" + ], + "type": "object" + }, + "EntryName": { + "description": "One entry name: a single path segment, neither `.` nor `..`, without\n`/` or NUL, at most 4096 bytes. `fs/readDirectory` entries use it.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(?!\\.\\.?$)[^/\\u0000]+$", + "type": "string" + }, + "EnvironmentPath": { + "description": "An absolute, normalized POSIX path in the environment: `/`, or `/` and\none or more segments joined by `/`. Empty, `.` and `..` segments and a\ntrailing `/` are rejected, never rewritten. At most 4096 bytes.\n`defaultCwd`, `homeDirectory`, every `cwd` and every path parameter use\nthis type; see the module's path model and confinement rule.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(/|(/(?!\\.\\.?(/|$))[^/\\u0000]+)+)$", + "type": "string" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/definitions/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/definitions/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "ExecInput": { + "description": "Standard input of a command.", + "oneOf": [ + { + "additionalProperties": false, + "description": "`/dev/null`: the command reads end of file. Model-facing tools use\nthis.", + "properties": { + "kind": { + "const": "none", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "These bytes, then end of file. Written by a writer task after the\noutput readers, the exit watcher and the deadline are in place.", + "properties": { + "data": { + "$ref": "#/definitions/ByteChunk" + }, + "kind": { + "const": "bytes", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A persistent pipe fed by `process/write` until closed. Processes\nonly; `job/start` refuses it with `invalidParams`.", + "properties": { + "kind": { + "const": "pipe", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "ExecState": { + "description": "Lifecycle state of a process.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "exited", + "description": "The root exited and its output settled.", + "type": "string" + } + ] + }, + "ExitStatus": { + "description": "How the root process ended.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "kind": { + "const": "code", + "type": "string" + } + }, + "required": [ + "kind", + "code" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Killed by this signal.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "kind": { + "const": "signal", + "type": "string" + }, + "signal": { + "$ref": "#/definitions/SignalName" + } + }, + "required": [ + "kind", + "signal" + ], + "type": "object" + } + ] + }, + "FileKind": { + "description": "Kind of a filesystem entry.", + "oneOf": [ + { + "enum": [ + "file", + "directory", + "symlink" + ], + "type": "string" + }, + { + "const": "other", + "description": "Device, FIFO, socket or another special file.", + "type": "string" + } + ] + }, + "FileMetadata": { + "additionalProperties": false, + "description": "One file-entry shape for `fs/getMetadata` and `fs/readDirectory`.", + "properties": { + "createdAtMs": { + "description": "Absent when the filesystem records no birth time.", + "format": "int64", + "type": "integer" + }, + "executable": { + "description": "A regular file with any execute bit set; always false for\ndirectories, symlinks and special files.", + "type": "boolean" + }, + "kind": { + "$ref": "#/definitions/FileKind" + }, + "modifiedAtMs": { + "format": "int64", + "type": "integer" + }, + "sizeBytes": { + "description": "Size in bytes; for a symlink, the length of its target text.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "symlinkTarget": { + "description": "Target text of a symlink, when it is UTF-8.", + "type": "string" + }, + "version": { + "allOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ], + "description": "Changes whenever the entry's content or identity changes; compare it\nwith `ifMatch`." + } + }, + "required": [ + "kind", + "sizeBytes", + "modifiedAtMs", + "version" + ], + "type": "object" + }, + "FileVersion": { + "description": "Opaque version token of a file's content and identity. Equal tokens for\none path mean the content is unchanged. 1 to 255 bytes without control\ncharacters.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "FilesystemCapabilities": { + "additionalProperties": false, + "description": "The `filesystem` family: `fs/*` and `transfer/*`.", + "properties": { + "write": { + "description": "Methods that change the filesystem are served; false on a daemon\nstarted read-only.", + "type": "boolean" + } + }, + "type": "object" + }, + "FilesystemLimits": { + "additionalProperties": false, + "description": "Ceilings of `fs/*` and `transfer/*`.", + "properties": { + "glob": { + "$ref": "#/definitions/GlobCeilings" + }, + "maxDirectoryEntries": { + "description": "Largest `limit` of `fs/readDirectory`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxReadFileBytes": { + "description": "Largest `maxBytes` of `fs/readFile`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxWriteFileBytes": { + "description": "Largest `data` of `fs/writeFile`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "scan": { + "$ref": "#/definitions/ScanCeilings" + }, + "search": { + "$ref": "#/definitions/SearchCeilings" + }, + "transfer": { + "$ref": "#/definitions/TransferCeilings" + } + }, + "required": [ + "maxReadFileBytes", + "maxWriteFileBytes", + "maxDirectoryEntries", + "search", + "glob", + "scan", + "transfer" + ], + "type": "object" + }, + "GapReason": { + "description": "Why a range of output cannot be returned.", + "oneOf": [ + { + "const": "overflow", + "description": "Never delivered: dropped from the middle because retained output\nexceeded the handle's cap. The head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Already delivered through the daemon cursor (or acknowledged) and\nreleased since.", + "type": "string" + } + ] + }, + "GetIdleParams": { + "additionalProperties": false, + "description": "`env/getIdle` params.", + "type": "object" + }, + "GetIdleResponse": { + "additionalProperties": false, + "description": "`env/getIdle` response.", + "properties": { + "idleForMs": { + "description": "Milliseconds since the last `use` request, or since running work\nlast ended, whichever is later. Zero while any process or job runs.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "leftoverProcessGroupCount": { + "description": "Process groups of exited processes that still have members alive:\nservices a command started and left running. They are not running\nwork and do not hold `idleForMs` at zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "runningJobCount": { + "description": "Jobs that are `running`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "runningProcessCount": { + "description": "Processes started with `process/start` whose root has not exited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "idleForMs" + ], + "type": "object" + }, + "GetMetadataParams": { + "additionalProperties": false, + "description": "`fs/getMetadata` params.", + "properties": { + "followSymlinks": { + "description": "Describe the symlink's target instead of the link (subject to\nconfinement).", + "type": "boolean" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "GetMetadataResponse": { + "additionalProperties": false, + "description": "`fs/getMetadata` response.", + "properties": { + "metadata": { + "$ref": "#/definitions/FileMetadata" + } + }, + "required": [ + "metadata" + ], + "type": "object" + }, + "GlobCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/globFiles` quotas, field for field.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxDurationMs" + ], + "type": "object" + }, + "GlobFilesLimits": { + "additionalProperties": false, + "description": "Caller quotas of one enumeration; each at most its advertised ceiling.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "description": "Non-directory entries visited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxDurationMs" + ], + "type": "object" + }, + "GlobFilesParams": { + "additionalProperties": false, + "description": "`fs/globFiles` params: bounded recursive enumeration of non-directory\nentries matching `filter.include`.", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/definitions/TreeFilter" + } + ], + "description": "`include` must not be empty." + }, + "limits": { + "$ref": "#/definitions/GlobFilesLimits" + }, + "root": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "root", + "filter", + "limits" + ], + "type": "object" + }, + "GlobFilesResponse": { + "additionalProperties": false, + "description": "`fs/globFiles` response.", + "properties": { + "elapsedMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "matches": { + "description": "In walk order (sorted by path).", + "items": { + "$ref": "#/definitions/EnvironmentPath" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Entries the walk could not represent or list: names that are not\nUTF-8 and directories that could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "stopped": { + "anyOf": [ + { + "$ref": "#/definitions/GlobFilesStop" + } + ] + }, + "visitedFileCount": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "visitedFileCount", + "elapsedMs" + ], + "type": "object" + }, + "GlobFilesStop": { + "description": "Why an enumeration stopped before finishing its walk.", + "oneOf": [ + { + "const": "matchLimit", + "description": "`maxMatches` paths were found.", + "type": "string" + }, + { + "const": "fileLimit", + "description": "`maxFiles` non-directory entries were visited.", + "type": "string" + }, + { + "const": "timeLimit", + "description": "`maxDurationMs` ran out.", + "type": "string" + }, + { + "const": "responseLimit", + "description": "The next match would push the encoded matches past\n`MAX_RESULT_BYTES`.", + "type": "string" + } + ] + }, + "Id": { + "anyOf": [ + { + "format": "int64", + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "A request id: an integer or a string, echoed in the response." + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "InitializeParams": { + "additionalProperties": false, + "description": "`initialize` params on the data plane.", + "properties": { + "clientInfo": { + "$ref": "#/definitions/ImplementationInfo" + }, + "protocolVersion": { + "description": "The client's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "clientInfo" + ], + "type": "object" + }, + "InitializeResponse": { + "additionalProperties": false, + "description": "`initialize` response on the data plane.", + "properties": { + "capabilities": { + "$ref": "#/definitions/DataCapabilities" + }, + "daemonInstanceId": { + "allOf": [ + { + "$ref": "#/definitions/DaemonInstanceId" + } + ], + "description": "Random per daemon start; a different value than on an earlier\nconnection means the daemon restarted and the earlier instance's\nprocess handles are gone." + }, + "defaultCwd": { + "allOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "The daemon's working directory: the default `cwd` of commands." + }, + "homeDirectory": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "Home directory of the user commands run as, when known." + }, + "limits": { + "$ref": "#/definitions/DataLimits" + }, + "plane": { + "allOf": [ + { + "$ref": "#/definitions/Plane" + } + ], + "description": "Always `data`." + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "serverInfo": { + "$ref": "#/definitions/ImplementationInfo" + } + }, + "required": [ + "protocolVersion", + "plane", + "daemonInstanceId", + "serverInfo", + "capabilities", + "limits", + "defaultCwd" + ], + "type": "object" + }, + "InventoryEntry": { + "additionalProperties": false, + "description": "One inventory entry. Parents precede children.", + "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/TreeContent" + } + ], + "description": "A file entry always carries its `digest`." + }, + "path": { + "allOf": [ + { + "$ref": "#/definitions/TreePath" + } + ], + "description": "Relative to the selected root; `.` is the root itself." + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + }, + "JobCancelScope": { + "description": "Which jobs a `job/cancel` affects.", + "oneOf": [ + { + "const": "job", + "description": "Only the named jobs.", + "type": "string" + }, + { + "const": "dependents", + "description": "The named jobs and every job that transitively depends on them.", + "type": "string" + } + ] + }, + "JobCapabilities": { + "additionalProperties": false, + "description": "The `job` family: `job/*`. It has no instance facts.", + "type": "object" + }, + "JobDependency": { + "description": "A dependency reference.", + "oneOf": [ + { + "additionalProperties": false, + "description": "An existing job on this daemon, or one in the same request.", + "properties": { + "jobId": { + "$ref": "#/definitions/JobId" + }, + "kind": { + "const": "jobId", + "type": "string" + } + }, + "required": [ + "kind", + "jobId" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The `name` of a job in the same `job/start`.", + "properties": { + "kind": { + "const": "name", + "type": "string" + }, + "name": { + "$ref": "#/definitions/JobName" + } + }, + "required": [ + "kind", + "name" + ], + "type": "object" + } + ] + }, + "JobDependencyPolicy": { + "description": "What a job waits for from its `dependsOn` jobs.", + "oneOf": [ + { + "const": "allSucceeded", + "description": "Run once every dependency `succeeded`; any other terminal status\nmakes this job `dependencyFailed`.", + "type": "string" + }, + { + "const": "allTerminal", + "description": "Run once every dependency is terminal, whatever its status.", + "type": "string" + } + ] + }, + "JobId": { + "description": "`jobId`: caller-chosen name of one job run on one daemon.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "JobLimits": { + "additionalProperties": false, + "description": "Ceilings of `job/*`.", + "properties": { + "cancelGraceMs": { + "description": "Grace between `SIGTERM` and `SIGKILL` for a non-forced cancel and\ntimeouts.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxReadBytes": { + "description": "Largest `maxBytes` of one `job/read`, across its entries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxRequestJobs": { + "description": "Most jobs one `job/start`, `job/read` or `job/cancel` names, and most\nentries a `job/cancel` response lists.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxRetainedOutputBytes": { + "description": "Output a job retains: the head and the tail, each half of this; the\nmiddle is an `overflow` gap.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "recordRetentionMs": { + "description": "How long a terminal job's record and output are kept.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxReadBytes", + "maxRequestJobs", + "maxRetainedOutputBytes", + "recordRetentionMs", + "cancelGraceMs" + ], + "type": "object" + }, + "JobName": { + "description": "`name` of a job and `dependsOn {kind: name}`: a caller-chosen name,\nunique within one `job/start`, 1 to 128 bytes without control\ncharacters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "JobReadEntry": { + "description": "A `job/read` entry: the job's summary and output page, or why it could\nnot be read (`notFound` for an unknown or garbage-collected job).", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "job", + "type": "string" + }, + "output": { + "$ref": "#/definitions/OutputPage" + }, + "summary": { + "$ref": "#/definitions/JobSummary" + } + }, + "required": [ + "kind", + "summary", + "output" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "$ref": "#/definitions/ProtocolError" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "kind": { + "const": "error", + "type": "string" + } + }, + "required": [ + "kind", + "jobId", + "error" + ], + "type": "object" + } + ] + }, + "JobReadRequest": { + "additionalProperties": false, + "description": "One entry of a `job/read`. Job reads never consume: without `after` and\n`tailBytes` the page starts at offset 0. One job may be named by several\nentries, such as a head and a tail page.", + "properties": { + "after": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "maxBytes": { + "description": "Most `data` bytes for this entry; the request's `maxBytes` still\nbounds the total. Absent: whatever of the request's budget remains.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "tailBytes": { + "description": "Start at `end - tailBytes`; excludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobId" + ], + "type": "object" + }, + "JobRequestId": { + "description": "`requestId` of a `job/start` call. Part of each job's identity: a\nretried start with the same request id and spec replays the job; any\nother reuse of the job id is refused.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "JobSpec": { + "additionalProperties": false, + "description": "One job of a `job/start`.", + "properties": { + "command": { + "allOf": [ + { + "$ref": "#/definitions/CommandSpec" + } + ], + "description": "`input` must be `none` or `bytes`." + }, + "dependencyPolicy": { + "allOf": [ + { + "$ref": "#/definitions/JobDependencyPolicy" + } + ], + "description": "What \"finish\" means for `dependsOn`; `allSucceeded` by default." + }, + "dependsOn": { + "description": "Jobs that must finish first.", + "items": { + "$ref": "#/definitions/JobDependency" + }, + "type": "array" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "name": { + "anyOf": [ + { + "$ref": "#/definitions/JobName" + } + ], + "description": "Local name other jobs of the same `job/start` can depend on; unique\nwithin the request." + }, + "queueKey": { + "anyOf": [ + { + "$ref": "#/definitions/QueueKey" + } + ] + } + }, + "required": [ + "jobId", + "command" + ], + "type": "object" + }, + "JobStatus": { + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Accepted; waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its `timeoutMs` expired and the daemon stopped it.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of `job/cancel`.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "JobSummary": { + "additionalProperties": false, + "description": "What a job is and how it ended. Instants are the guest's wall clock.", + "properties": { + "cancelRequestedAtMs": { + "description": "When a `job/cancel` reached the job while it ran.", + "format": "int64", + "type": "integer" + }, + "createdAtMs": { + "format": "int64", + "type": "integer" + }, + "dependsOn": { + "description": "Resolved ids of the jobs this one depends on.", + "items": { + "$ref": "#/definitions/JobId" + }, + "type": "array" + }, + "durationMs": { + "description": "Monotonic run time from start to finish, immune to guest clock jumps.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/definitions/ProtocolError" + } + ], + "description": "Why a `failed` or `interrupted` job produced no exit, or an\ninfrastructure failure while it ran." + }, + "exit": { + "anyOf": [ + { + "$ref": "#/definitions/ExitStatus" + } + ], + "description": "How the root process ended, once it ran and exited." + }, + "finishedAtMs": { + "format": "int64", + "type": "integer" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "leftovers": { + "anyOf": [ + { + "$ref": "#/definitions/LeftoverReport" + } + ], + "description": "What the daemon swept from the job's group at completion\n(`disposition: swept`)." + }, + "name": { + "anyOf": [ + { + "$ref": "#/definitions/JobName" + } + ] + }, + "queueKey": { + "anyOf": [ + { + "$ref": "#/definitions/QueueKey" + } + ] + }, + "queuedAtMs": { + "description": "When the job became runnable and waited for its queue.", + "format": "int64", + "type": "integer" + }, + "requestId": { + "$ref": "#/definitions/JobRequestId" + }, + "startedAtMs": { + "format": "int64", + "type": "integer" + }, + "status": { + "$ref": "#/definitions/JobStatus" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/definitions/StoppedBy" + } + ], + "description": "Set when the daemon stopped the running job: `timeout` for\n`timedOut`, `terminate` or `kill` for a cancel without or with\n`force`." + } + }, + "required": [ + "jobId", + "requestId", + "status", + "createdAtMs" + ], + "type": "object" + }, + "JobSummaryEntry": { + "description": "A job summary, or why the named job could not be found.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "job", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/JobSummary" + } + }, + "required": [ + "kind", + "summary" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "$ref": "#/definitions/ProtocolError" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "kind": { + "const": "error", + "type": "string" + } + }, + "required": [ + "kind", + "jobId", + "error" + ], + "type": "object" + } + ] + }, + "LeftoverDisposition": { + "description": "What the daemon did with leftovers. The policies differ on purpose.", + "oneOf": [ + { + "const": "kept", + "description": "Left running (processes): they keep the environment's lifetime and\nare reported until they exit.", + "type": "string" + }, + { + "const": "swept", + "description": "Terminated when the job finished (jobs); the submitter should fix\nits script's teardown.", + "type": "string" + } + ] + }, + "LeftoverProcess": { + "additionalProperties": false, + "description": "One leftover process.", + "properties": { + "command": { + "description": "Command line when the platform exposes it, otherwise the executable\nname, cut to\n`MAX_LEFTOVER_COMMAND_BYTES`\non a character boundary; absent when neither could be read.", + "type": "string" + }, + "pid": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "LeftoverReport": { + "additionalProperties": false, + "description": "Processes of a command's group still alive after its root exited.\nPresent only when there is at least one.", + "properties": { + "disposition": { + "$ref": "#/definitions/LeftoverDisposition" + }, + "omittedProcessCount": { + "description": "Leftovers beyond the listed ones.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "processes": { + "description": "At most `MAX_LEFTOVER_PROCESSES`,\nlowest pid first; never empty.", + "items": { + "$ref": "#/definitions/LeftoverProcess" + }, + "type": "array" + } + }, + "required": [ + "disposition", + "processes" + ], + "type": "object" + }, + "ListTransferMissingParams": { + "additionalProperties": false, + "description": "`transfer/listMissing` params.", + "properties": { + "limit": { + "description": "Most digests to return, 1 to\n`limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "limit" + ], + "type": "object" + }, + "ListTransferMissingResponse": { + "additionalProperties": false, + "description": "`transfer/listMissing` response: digests whose content the daemon still\nneeds. The set shrinks as uploads complete; call again rather than\npaging.", + "properties": { + "digests": { + "items": { + "$ref": "#/definitions/ContentDigest" + }, + "type": "array" + }, + "remainingCount": { + "description": "Missing digests beyond this page.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "OutputChunk": { + "additionalProperties": false, + "description": "Contiguous output bytes from one stream.", + "properties": { + "data": { + "$ref": "#/definitions/ByteChunk" + }, + "offset": { + "description": "Byte offset of the chunk's first byte in the combined output.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "stream": { + "$ref": "#/definitions/OutputStream" + } + }, + "required": [ + "offset", + "stream", + "data" + ], + "type": "object" + }, + "OutputGap": { + "additionalProperties": false, + "description": "Output bytes in `[offset, offset + omittedBytes)` that a page cannot\nreturn.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/definitions/GapReason" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, + "OutputPage": { + "additionalProperties": false, + "description": "One page of a handle's output. Returned by `process/read` and in every\n`job/read` entry. A page holds at most\n`MAX_PAGE_CHUNKS` chunks and gaps\ntogether (for `job/read`, across all entries); one that reaches it ends\nearly, and `next` says where to continue.", + "properties": { + "chunks": { + "description": "In offset order, at most `maxBytes` of `data` in total.", + "items": { + "$ref": "#/definitions/OutputChunk" + }, + "type": "array" + }, + "end": { + "description": "Bytes produced so far; final once the handle has exited.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "gaps": { + "description": "Unreturnable ranges between the requested start and `next`.", + "items": { + "$ref": "#/definitions/OutputGap" + }, + "type": "array" + }, + "next": { + "description": "Offset to read from next: past the last returned byte or gap.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "next", + "end" + ], + "type": "object" + }, + "OutputStream": { + "description": "Which descriptor produced a chunk. Jobs never produce `pty`.", + "oneOf": [ + { + "enum": [ + "stdout", + "stderr" + ], + "type": "string" + }, + { + "const": "pty", + "description": "The PTY master, which merges stdout and stderr.", + "type": "string" + } + ] + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "Plane": { + "description": "The plane a connection serves. Each `initialize` response names its\nplane, so a client that reached the wrong socket fails at the handshake.", + "oneOf": [ + { + "const": "data", + "description": "envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "controller", + "description": "A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "registration", + "description": "The registration channel between a daemon and the gateway, versioned\nby `ENVD_PROTOCOL_VERSION`.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner, or (`target/adopt`) the owner\nalready holds a different target.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + }, + { + "const": "notAdoptable", + "description": "`target/adopt`: the source target exists but the provider cannot take\nit over, for example an instance of a kind it does not manage.", + "type": "string" + } + ] + }, + "ProcessCapabilities": { + "additionalProperties": false, + "description": "The `process` family: `process/*`.", + "properties": { + "pty": { + "description": "`process/start` accepts `pty`.", + "type": "boolean" + } + }, + "type": "object" + }, + "ProcessId": { + "description": "`processId`: caller-chosen handle of a process on one daemon.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ProcessLimits": { + "additionalProperties": false, + "description": "Ceilings of `process/*`.", + "properties": { + "maxExitedRetainedBytes": { + "description": "Output retained across all exited processes; beyond it the oldest\nare evicted early.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxInputQueueBytes": { + "description": "Bytes queued for a process's input before `process/write` accepts\nfewer than it was sent.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxReadBytes": { + "description": "Largest `maxBytes` of one `process/read`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxRetainedOutputBytes": { + "description": "Output a process retains; beyond it the oldest delivered bytes are\nreleased (`expired` gaps), then the undelivered middle (`overflow`\ngaps).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "retentionAfterExitMs": { + "description": "How long an exited process stays readable after a read first\nreturned its settled exit with nothing left unread.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "terminateGraceMs": { + "description": "Grace between `SIGTERM` and `SIGKILL` for `terminate` and timeouts.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "unreadRetentionMs": { + "description": "How long an exited process whose output was never fully read stays\nreadable.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxReadBytes", + "maxRetainedOutputBytes", + "retentionAfterExitMs", + "unreadRetentionMs", + "maxExitedRetainedBytes", + "terminateGraceMs", + "maxInputQueueBytes" + ], + "type": "object" + }, + "ProcessSignal": { + "description": "Signal delivered by `process/terminate`.", + "oneOf": [ + { + "const": "interrupt", + "description": "`SIGINT` to the foreground process group (the terminal's under a\nPTY, otherwise the process group). Records nothing; the next read\nshows what the process did.", + "type": "string" + }, + { + "const": "terminate", + "description": "`SIGTERM` to the group, then `SIGKILL` after\n`limits.process.terminateGraceMs` if members remain. Records\n`stoppedBy: terminate`.", + "type": "string" + }, + { + "const": "kill", + "description": "`SIGKILL` to the group. Records `stoppedBy: kill`.", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "PtySize": { + "additionalProperties": false, + "description": "Terminal size of a PTY. Each dimension is 1 to 4096.", + "properties": { + "cols": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "rows": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "rows", + "cols" + ], + "type": "object" + }, + "QueueKey": { + "description": "`queueKey`: jobs sharing a key run one at a time, in start order.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ReadDirectoryParams": { + "additionalProperties": false, + "description": "`fs/readDirectory` params. Pages in name order.", + "properties": { + "cursor": { + "anyOf": [ + { + "$ref": "#/definitions/DirectoryCursor" + } + ], + "description": "`nextCursor` of the previous page; a cursor this daemon did not mint\nis `invalidParams`." + }, + "limit": { + "description": "Most entries per page, 1 to `limits.filesystem.maxDirectoryEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "path": { + "allOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "A directory, or a symlink to one inside the root." + } + }, + "required": [ + "path", + "limit" + ], + "type": "object" + }, + "ReadDirectoryResponse": { + "additionalProperties": false, + "description": "`fs/readDirectory` response.", + "properties": { + "entries": { + "items": { + "$ref": "#/definitions/DirectoryEntry" + }, + "type": "array" + }, + "nextCursor": { + "anyOf": [ + { + "$ref": "#/definitions/DirectoryCursor" + } + ], + "description": "Opaque cursor of the next page; absent on the last page." + }, + "skippedEntryCount": { + "description": "Entries in this page's range skipped because their names are not\nUTF-8. Entries that vanish while the page is built are left out and\nnot counted.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "ReadFileParams": { + "additionalProperties": false, + "description": "`fs/readFile` params. Reads a regular file only; anything else is\n`failedPrecondition` (`notRegularFile`) and is never opened for a\nblocking read.", + "properties": { + "maxBytes": { + "description": "Most bytes to return, at most `limits.filesystem.maxReadFileBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "offset": { + "description": "First byte to return; 0 by default.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path", + "maxBytes" + ], + "type": "object" + }, + "ReadFileResponse": { + "additionalProperties": false, + "description": "`fs/readFile` response.", + "properties": { + "data": { + "$ref": "#/definitions/ByteChunk" + }, + "fileSize": { + "description": "Size of the whole file when it was read.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "truncated": { + "description": "Bytes exist after `offset` plus the length of `data`.", + "type": "boolean" + }, + "version": { + "$ref": "#/definitions/FileVersion" + } + }, + "required": [ + "data", + "fileSize", + "version" + ], + "type": "object" + }, + "ReadJobsParams": { + "additionalProperties": false, + "description": "`job/read` params.", + "properties": { + "jobs": { + "description": "1 to `limits.job.maxRequestJobs` entries, each with its own position.", + "items": { + "$ref": "#/definitions/JobReadRequest" + }, + "type": "array" + }, + "maxBytes": { + "description": "Most `data` bytes across all entries, at most\n`limits.job.maxReadBytes`, filled in request order: an entry reached\nafter the budget is spent returns its summary and an empty page at\nits start position. 0 reads only summaries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "returnOn": { + "allOf": [ + { + "$ref": "#/definitions/ReturnOn" + } + ], + "description": "`anyOutput`: return when any entry has output past its position or\nchanged status; `exit`: when every entry is terminal; `window`: wait\nthe whole window." + }, + "waitMs": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobs", + "maxBytes", + "waitMs", + "returnOn" + ], + "type": "object" + }, + "ReadJobsResponse": { + "additionalProperties": false, + "description": "`job/read` response: one entry per requested job, in request order.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/definitions/JobReadEntry" + }, + "type": "array" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "ReadProcessParams": { + "additionalProperties": false, + "description": "`process/read` params.", + "properties": { + "ack": { + "description": "Acknowledge every byte before this offset first: the cursor moves to\nit (never backwards) whether or not this read's response arrives. A\ncaller that tracks positions reads with `after` and acknowledges\nwith `ack` for exactly-once delivery.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "after": { + "description": "Start offset. Absent: the daemon cursor, which this read advances to\n`next` once its response is written. Present: a non-consuming read\nthat leaves the cursor alone.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxBytes": { + "description": "Most `data` bytes to return, at most `limits.process.maxReadBytes`.\n0 reads only the state.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "processId": { + "$ref": "#/definitions/ProcessId" + }, + "returnOn": { + "$ref": "#/definitions/ReturnOn" + }, + "tailBytes": { + "description": "Start at `end - tailBytes` instead (clamped at 0). Non-consuming;\nexcludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "waitMs": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "processId", + "maxBytes", + "waitMs", + "returnOn" + ], + "type": "object" + }, + "ReadProcessResponse": { + "additionalProperties": false, + "description": "`process/read` response.", + "properties": { + "error": { + "anyOf": [ + { + "$ref": "#/definitions/ProtocolError" + } + ], + "description": "Infrastructure failure after start (output capture or wait failed)." + }, + "exit": { + "anyOf": [ + { + "$ref": "#/definitions/ExitStatus" + } + ], + "description": "How the root ended; present once `state` is `exited`, unless `error`\nexplains why it could not be observed." + }, + "leftovers": { + "anyOf": [ + { + "$ref": "#/definitions/LeftoverReport" + } + ], + "description": "Group members alive after the root exited (`disposition: kept`),\nsampled when this response was built. Absent while running, when\nnone remain, and on platforms without process enumeration." + }, + "output": { + "$ref": "#/definitions/OutputPage" + }, + "pid": { + "description": "OS pid of the root process.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "state": { + "$ref": "#/definitions/ExecState" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/definitions/StoppedBy" + } + ], + "description": "Set when the daemon stopped the process; absent on a natural exit." + } + }, + "required": [ + "output", + "state", + "pid" + ], + "type": "object" + }, + "ReadTransferContentParams": { + "additionalProperties": false, + "description": "`transfer/read` params: read a captured file's content by digest.", + "properties": { + "digest": { + "$ref": "#/definitions/ContentDigest" + }, + "maxBytes": { + "description": "1 to `limits.filesystem.transfer.maxContentChunkBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "digest", + "offset", + "maxBytes" + ], + "type": "object" + }, + "ReadTransferContentResponse": { + "additionalProperties": false, + "description": "`transfer/read` response.", + "properties": { + "data": { + "$ref": "#/definitions/ByteChunk" + }, + "truncated": { + "description": "Bytes exist after `offset` plus the length of `data`, as for\n`fs/readFile`.", + "type": "boolean" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "ReadTransferInventoryParams": { + "additionalProperties": false, + "description": "`transfer/readInventory` params: page a sealed inventory by index.", + "properties": { + "index": { + "description": "Index of the first entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "limit": { + "description": "1 to `limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "index", + "limit" + ], + "type": "object" + }, + "ReadTransferInventoryResponse": { + "additionalProperties": false, + "description": "`transfer/readInventory` response.", + "properties": { + "entries": { + "items": { + "$ref": "#/definitions/InventoryEntry" + }, + "type": "array" + }, + "nextIndex": { + "description": "Index of the next page; absent after the last entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "RemoveParams": { + "additionalProperties": false, + "description": "`fs/remove` params.", + "properties": { + "force": { + "description": "Succeed when the path does not exist.", + "type": "boolean" + }, + "path": { + "allOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "A symlink is removed, never followed." + }, + "recursive": { + "description": "Remove a directory and its contents.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "RemoveResponse": { + "additionalProperties": false, + "description": "`fs/remove` response.", + "type": "object" + }, + "RenameParams": { + "additionalProperties": false, + "description": "`fs/rename` params. Atomic within one filesystem; across filesystems it\nis `failedPrecondition` (`crossDevice`). The entry keeps its mode.", + "properties": { + "destination": { + "$ref": "#/definitions/EnvironmentPath" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ], + "description": "Rename only when the source's version equals this token." + }, + "overwrite": { + "description": "Replace an existing destination file; otherwise an existing\ndestination is `alreadyExists`. A directory destination is never\nreplaced.", + "type": "boolean" + }, + "source": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "source", + "destination" + ], + "type": "object" + }, + "RenameResponse": { + "additionalProperties": false, + "description": "`fs/rename` response.", + "type": "object" + }, + "ReturnOn": { + "description": "When a waiting read returns. Every mode also returns when `waitMs`\nelapses, when the handle reaches a settled exit, or at once when there\nis nothing left to wait for.", + "oneOf": [ + { + "const": "window", + "description": "Collect for the whole window; return early only when unread output\nreaches `maxBytes`.", + "type": "string" + }, + { + "const": "anyOutput", + "description": "Return as soon as any output past the start position exists.", + "type": "string" + }, + { + "const": "exit", + "description": "Return only at the settled exit (for `job/read`: once every entry is\nterminal), even if more than `maxBytes` of output accumulates.", + "type": "string" + } + ] + }, + "ScanCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/scan`: root and pattern counts, then its quotas.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxPatterns": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxRoots": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxRoots", + "maxPatterns", + "maxEntries", + "maxFileBytes", + "maxTotalBytes", + "maxDurationMs" + ], + "type": "object" + }, + "ScanEntry": { + "additionalProperties": false, + "description": "One selected entry below a root.", + "properties": { + "content": { + "$ref": "#/definitions/TreeContent" + }, + "data": { + "anyOf": [ + { + "$ref": "#/definitions/ByteChunk" + } + ], + "description": "The file's bytes when `readContent` was set." + }, + "path": { + "allOf": [ + { + "$ref": "#/definitions/TreePath" + } + ], + "description": "Relative to the root as requested." + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + }, + "ScanFingerprint": { + "description": "Opaque fingerprint of one root's selection, prefixed `scan:v1:`, at most\n255 bytes without control characters.", + "maxLength": 255, + "minLength": 9, + "pattern": "^scan:v1:[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "ScanLimits": { + "additionalProperties": false, + "description": "Caller quotas of one scan.", + "properties": { + "maxDurationMs": { + "description": "Walk time per root; exhausting it fails that root\n(`deadlineExceeded`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "description": "Selected entries per root; more fail that root (`payloadTooLarge`\nwith `limitEntries`).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "description": "Larger selected files are skipped (`fileTooLarge`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "description": "Content bytes returned across all roots, in root order: a root whose\ncontent would exceed the rest fails (`payloadTooLarge` with\n`limitBytes`) and later roots are still scanned. Independently, a\nroot whose encoded result (entries, skips and base64 content) would\npush the scan's results past\n`MAX_RESULT_BYTES` fails the same\nway.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxEntries", + "maxFileBytes", + "maxTotalBytes", + "maxDurationMs" + ], + "type": "object" + }, + "ScanParams": { + "additionalProperties": false, + "description": "`fs/scan` params.", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/definitions/TreeFilter" + } + ], + "description": "Applied below every root; at most `limits.filesystem.scan.maxPatterns`\npatterns in total." + }, + "followSymlinks": { + "description": "Follow symlinks whose targets stay inside the filesystem root; others\nare skipped (`symlinkOutsideRoot`). Without it symlinks are skipped\n(`symlink`).", + "type": "boolean" + }, + "includeDigests": { + "description": "Return each selected file's `digest`.", + "type": "boolean" + }, + "limits": { + "allOf": [ + { + "$ref": "#/definitions/ScanLimits" + } + ], + "description": "Caller quotas; each at most its ceiling." + }, + "readContent": { + "description": "Return the bytes of every selected file in `data`.", + "type": "boolean" + }, + "roots": { + "description": "1 to `limits.filesystem.scan.maxRoots` roots.", + "items": { + "$ref": "#/definitions/ScanRoot" + }, + "type": "array" + } + }, + "required": [ + "roots", + "limits" + ], + "type": "object" + }, + "ScanResponse": { + "additionalProperties": false, + "description": "`fs/scan` response: one result per requested root, in request order.", + "properties": { + "roots": { + "description": "Never empty.", + "items": { + "$ref": "#/definitions/ScanRootResult" + }, + "type": "array" + } + }, + "required": [ + "roots" + ], + "type": "object" + }, + "ScanRoot": { + "additionalProperties": false, + "description": "One root of a scan.", + "properties": { + "ifNoneMatch": { + "anyOf": [ + { + "$ref": "#/definitions/ScanFingerprint" + } + ], + "description": "The root's previous `fingerprint`: when the selection is unchanged\nthe result is `unchanged` and carries no entries." + }, + "maxDepth": { + "description": "Overrides `filter.maxDepth` for this root. `0` examines only the\nroot itself: a `complete` result without entries proves it is an\nexisting directory, which is how discovery checks a working\ndirectory in the same call.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "ScanRootResult": { + "description": "The outcome of one root.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Every selected entry, plus what was skipped and why.", + "properties": { + "canonicalPath": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "Resolved absolute path of the root when it differs from `path`\n(the root is a symlink)." + }, + "entries": { + "items": { + "$ref": "#/definitions/ScanEntry" + }, + "type": "array" + }, + "fingerprint": { + "$ref": "#/definitions/ScanFingerprint" + }, + "kind": { + "const": "complete", + "type": "string" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + }, + "skipped": { + "description": "The first\n`MAX_LISTED_SKIPS` skipped\nentries, in walk order.", + "items": { + "$ref": "#/definitions/ScanSkip" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Total number of skipped entries, listed or not; present whenever\nit is not zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "path", + "fingerprint" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The selection matches `ifNoneMatch`.", + "properties": { + "fingerprint": { + "$ref": "#/definitions/ScanFingerprint" + }, + "kind": { + "const": "unchanged", + "type": "string" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "kind", + "path", + "fingerprint" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The root does not exist.", + "properties": { + "kind": { + "const": "missing", + "type": "string" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "kind", + "path" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The root could not be scanned; no entries are returned for it.", + "properties": { + "error": { + "$ref": "#/definitions/ProtocolError" + }, + "kind": { + "const": "failed", + "type": "string" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "kind", + "path", + "error" + ], + "type": "object" + } + ] + }, + "ScanSkip": { + "additionalProperties": false, + "description": "An entry a scan left out.", + "properties": { + "path": { + "anyOf": [ + { + "$ref": "#/definitions/TreePath" + } + ], + "description": "Absent when the entry's name is not UTF-8." + }, + "reason": { + "$ref": "#/definitions/ScanSkipReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "ScanSkipReason": { + "description": "Why a scan left an entry out.", + "oneOf": [ + { + "const": "fileTooLarge", + "description": "Larger than `maxFileBytes`.", + "type": "string" + }, + { + "const": "symlink", + "description": "A symlink, and `followSymlinks` is not set.", + "type": "string" + }, + { + "const": "symlinkOutsideRoot", + "description": "A symlink whose target leaves the filesystem root, or a link cycle.", + "type": "string" + }, + { + "const": "specialFile", + "description": "A device, FIFO, socket or another special file.", + "type": "string" + }, + { + "const": "nonUtf8Name", + "description": "The name is not UTF-8.", + "type": "string" + }, + { + "const": "unreadable", + "description": "The entry vanished or could not be read during the walk.", + "type": "string" + } + ] + }, + "SearchCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/searchText` quotas, field for field.", + "properties": { + "maxBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxLineBytes": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxBytes", + "maxDurationMs", + "maxLineBytes" + ], + "type": "object" + }, + "SearchTextLimits": { + "additionalProperties": false, + "description": "Caller quotas of one search; each at most its advertised ceiling.", + "properties": { + "maxBytes": { + "description": "Bytes actually read across files.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "description": "Files searched, not merely visited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxLineBytes": { + "description": "Longer matching lines are cut and marked `lineTruncated`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxBytes", + "maxDurationMs", + "maxLineBytes" + ], + "type": "object" + }, + "SearchTextMatch": { + "additionalProperties": false, + "description": "One matching line.", + "properties": { + "line": { + "description": "The matching line without its terminator, cut at `maxLineBytes` on a\ncharacter boundary.", + "type": "string" + }, + "lineNumber": { + "description": "1-based.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "lineTruncated": { + "type": "boolean" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path", + "lineNumber", + "line" + ], + "type": "object" + }, + "SearchTextParams": { + "additionalProperties": false, + "description": "`fs/searchText` params: a bounded recursive regex search run by the\ndaemon. A file larger than the remaining byte budget is skipped and\ncounted; it never ends the walk.", + "properties": { + "filter": { + "$ref": "#/definitions/TreeFilter" + }, + "limits": { + "$ref": "#/definitions/SearchTextLimits" + }, + "pattern": { + "description": "Regular expression in the Rust `regex` crate's syntax, matched per\nline.", + "type": "string" + }, + "patternIgnoreCase": { + "description": "Match `pattern` case-insensitively. `filter.ignoreCase` governs the\npath patterns instead.", + "type": "boolean" + }, + "root": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "root", + "pattern", + "limits" + ], + "type": "object" + }, + "SearchTextResponse": { + "additionalProperties": false, + "description": "`fs/searchText` response.", + "properties": { + "elapsedMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "matches": { + "items": { + "$ref": "#/definitions/SearchTextMatch" + }, + "type": "array" + }, + "searchedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "searchedFileCount": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "skippedEntryCount": { + "description": "Entries the walk could not represent or list: names that are not\nUTF-8 and directories that could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "skippedFileCount": { + "description": "Files skipped because they exceeded the remaining byte budget, are\nbinary or could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "stopped": { + "anyOf": [ + { + "$ref": "#/definitions/SearchTextStop" + } + ], + "description": "Absent when the walk finished within its limits." + } + }, + "required": [ + "searchedFileCount", + "searchedBytes", + "elapsedMs" + ], + "type": "object" + }, + "SearchTextStop": { + "description": "Why a search stopped before finishing its walk.", + "oneOf": [ + { + "const": "matchLimit", + "description": "`maxMatches` matches were found.", + "type": "string" + }, + { + "const": "fileLimit", + "description": "`maxFiles` files were searched.", + "type": "string" + }, + { + "const": "byteLimit", + "description": "`maxBytes` bytes were read.", + "type": "string" + }, + { + "const": "timeLimit", + "description": "`maxDurationMs` ran out.", + "type": "string" + }, + { + "const": "responseLimit", + "description": "The next match would push the encoded matches past\n`MAX_RESULT_BYTES`.", + "type": "string" + } + ] + }, + "SecretString": { + "description": "A secret value carried on the wire, such as a `secretEnv` value, a\nregistration key or a stream token. It serializes as a plain string;\n`Debug` is redacted so structured logging never prints it.", + "type": "string" + }, + "SignalName": { + "description": "A POSIX signal name such as `SIGSEGV` or `SIGRTMIN+3`: `SIG` followed by\n1 to 16 uppercase letters, digits or `+`. Names, not numbers, because\nnumbers differ between platforms.", + "pattern": "^SIG[A-Z0-9+]{1,16}$", + "type": "string" + }, + "SpecialFilePolicy": { + "description": "Devices, FIFOs and sockets met during a capture.", + "oneOf": [ + { + "const": "reject", + "description": "Fail the operation (`notRegularFile`).", + "type": "string" + }, + { + "const": "skip", + "description": "Leave them out and report them in `skipped`.", + "type": "string" + } + ] + }, + "StartJobsParams": { + "additionalProperties": false, + "description": "`job/start` params. All-or-nothing: either every job is accepted (or\nreplayed) or none is.", + "properties": { + "jobs": { + "description": "1 to `limits.job.maxRequestJobs` jobs.", + "items": { + "$ref": "#/definitions/JobSpec" + }, + "type": "array" + }, + "requestId": { + "allOf": [ + { + "$ref": "#/definitions/JobRequestId" + } + ], + "description": "Idempotency key of this start. Retrying with the same request id and\nidentical jobs replays the existing jobs; reusing a job id with\nanother request id or spec is `failedPrecondition` (`jobIdInUse`)." + } + }, + "required": [ + "requestId", + "jobs" + ], + "type": "object" + }, + "StartJobsResponse": { + "additionalProperties": false, + "description": "`job/start` response: one summary per requested job, in request order.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/definitions/JobSummary" + }, + "type": "array" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "StartProcessParams": { + "additionalProperties": false, + "description": "`process/start` params.", + "properties": { + "command": { + "$ref": "#/definitions/CommandSpec" + }, + "processId": { + "allOf": [ + { + "$ref": "#/definitions/ProcessId" + } + ], + "description": "Caller-chosen handle, reserved before any side effect: a start whose\nid is already in use (running, retained or remembered by a tombstone)\nis `alreadyExists` and runs nothing." + }, + "pty": { + "anyOf": [ + { + "$ref": "#/definitions/PtySize" + } + ], + "description": "Run under a PTY of this size. The PTY master is the input: `input`\nmust then be `none` or `bytes`, and `process/write` is always\naccepted while the process runs. Requires\n`capabilities.process.pty`." + } + }, + "required": [ + "processId", + "command" + ], + "type": "object" + }, + "StartProcessResponse": { + "additionalProperties": false, + "description": "`process/start` response, sent once the process runs.", + "properties": { + "pid": { + "description": "OS pid of the root process, which is also its process group id.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "StoppedBy": { + "description": "Why the daemon stopped a process or job that did not exit on its own.\nSet only when the daemon signalled a root that was still running.", + "oneOf": [ + { + "const": "timeout", + "description": "`timeoutMs` expired: `SIGTERM`, the grace, then `SIGKILL`.", + "type": "string" + }, + { + "const": "terminate", + "description": "`process/terminate` with `terminate`, or `job/cancel` without\n`force`: `SIGTERM`, the grace, then `SIGKILL`.", + "type": "string" + }, + { + "const": "kill", + "description": "`process/terminate` with `kill`, or `job/cancel` with `force`:\n`SIGKILL` at once.", + "type": "string" + } + ] + }, + "SymlinkPolicy": { + "description": "Symlinks met during a capture.", + "oneOf": [ + { + "const": "expandInside", + "description": "Capture a link whose target stays inside the selection as the entry\nit points to; a link leaving the selection or a cycle fails the\noperation (`symlinkRejected`). A dangling link inside the selection\nis left out and reported in `skipped` as `symlink`.", + "type": "string" + }, + { + "const": "skip", + "description": "Leave every link out and report it in `skipped`.", + "type": "string" + }, + { + "const": "reject", + "description": "Fail on the first link.", + "type": "string" + } + ] + }, + "TerminateProcessParams": { + "additionalProperties": false, + "description": "`process/terminate` params.", + "properties": { + "processId": { + "$ref": "#/definitions/ProcessId" + }, + "signal": { + "$ref": "#/definitions/ProcessSignal" + } + }, + "required": [ + "processId", + "signal" + ], + "type": "object" + }, + "TerminateProcessResponse": { + "additionalProperties": false, + "description": "`process/terminate` response, sent once the signal was delivered (any\nescalation continues in the background).", + "properties": { + "running": { + "description": "True when the root process was still running when the signal was\nsent.", + "type": "boolean" + } + }, + "type": "object" + }, + "TransferCeilings": { + "additionalProperties": false, + "description": "Ceilings of `transfer/*`: capacity, page and chunk sizes, receipt\nretention, then the per-operation quotas.", + "properties": { + "maxActiveOperations": { + "description": "Operations not yet complete, aborted or failed, across connections;\na begin beyond it is `busy` (`transferCapacity`).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxContentChunkBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDepth": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxInventoryPageEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxManifestBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "receiptRetentionMs": { + "description": "How long a finished operation's receipt is answered.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxActiveOperations", + "maxContentChunkBytes", + "maxInventoryPageEntries", + "receiptRetentionMs", + "maxEntries", + "maxDepth", + "maxFileBytes", + "maxTotalBytes", + "maxManifestBytes", + "maxDurationMs" + ], + "type": "object" + }, + "TransferLimits": { + "additionalProperties": false, + "description": "Caller quotas of one operation; each at most its advertised ceiling.", + "properties": { + "maxDepth": { + "description": "The selected root has depth 0.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "description": "From begin to commit.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "description": "Entries, the selected root included.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxManifestBytes": { + "description": "Encoded inventory size.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxEntries", + "maxDepth", + "maxFileBytes", + "maxTotalBytes", + "maxManifestBytes", + "maxDurationMs" + ], + "type": "object" + }, + "TransferOnExisting": { + "description": "What a materialize does with an existing destination.", + "oneOf": [ + { + "const": "error", + "description": "Refuse at begin (`alreadyExists`).", + "type": "string" + }, + { + "const": "replace", + "description": "Replace it atomically at commit.", + "type": "string" + } + ] + }, + "TransferOperationId": { + "description": "`operationId` of one transfer operation.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "TransferOperationParams": { + "additionalProperties": false, + "description": "Params of `transfer/advance`, `transfer/getStatus`, `transfer/commit`\nand `transfer/abort`: each names only the operation.", + "properties": { + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId" + ], + "type": "object" + }, + "TransferPhase": { + "description": "Where an operation stands; see the module docs for the order per\nselection kind.", + "oneOf": [ + { + "enum": [ + "complete", + "aborted" + ], + "type": "string" + }, + { + "const": "scanning", + "description": "The daemon walks a tree: the capture source, or the destination of a\nmaterialize. Advance until the phase changes.", + "type": "string" + }, + { + "const": "inventory", + "description": "Capture: the inventory is sealed and pageable. Materialize: waiting\nfor appended entries.", + "type": "string" + }, + { + "const": "content", + "description": "Waiting for the content of `transfer/listMissing` digests.", + "type": "string" + }, + { + "const": "staging", + "description": "The daemon assembles the staged tree. Advance until `ready`.", + "type": "string" + }, + { + "const": "ready", + "description": "Ready for `transfer/commit`.", + "type": "string" + }, + { + "const": "failed", + "description": "Stopped by an error; see `error`.", + "type": "string" + } + ] + }, + "TransferSelection": { + "description": "What one operation transfers.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read a file or tree out of the environment.", + "properties": { + "filter": { + "$ref": "#/definitions/TreeFilter" + }, + "kind": { + "const": "capture", + "type": "string" + }, + "source": { + "$ref": "#/definitions/EnvironmentPath" + }, + "specialFiles": { + "$ref": "#/definitions/SpecialFilePolicy" + }, + "symlinks": { + "$ref": "#/definitions/SymlinkPolicy" + } + }, + "required": [ + "kind", + "source" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Write a tree into the environment, replacing `destination` atomically\nat commit.", + "properties": { + "destination": { + "$ref": "#/definitions/EnvironmentPath" + }, + "kind": { + "const": "materialize", + "type": "string" + }, + "onExisting": { + "$ref": "#/definitions/TransferOnExisting" + } + }, + "required": [ + "kind", + "destination" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Write exactly one file with `fs/writeFile` semantics: staged,\natomic, an existing target keeps its permissions, an in-root symlink\nat `destination` is followed.", + "properties": { + "create": { + "$ref": "#/definitions/WriteCondition" + }, + "destination": { + "$ref": "#/definitions/EnvironmentPath" + }, + "executable": { + "type": "boolean" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ] + }, + "kind": { + "const": "writeFile", + "type": "string" + } + }, + "required": [ + "kind", + "destination" + ], + "type": "object" + } + ] + }, + "TransferSkip": { + "additionalProperties": false, + "description": "An entry a capture left out.", + "properties": { + "path": { + "anyOf": [ + { + "$ref": "#/definitions/TreePath" + } + ], + "description": "Absent when the entry's name is not UTF-8." + }, + "reason": { + "$ref": "#/definitions/TransferSkipReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "TransferSkipReason": { + "description": "Why a capture left an entry out.", + "oneOf": [ + { + "const": "symlink", + "description": "`symlinks: skip`, or a dangling link under `symlinks: expandInside`.", + "type": "string" + }, + { + "const": "specialFile", + "description": "`specialFiles: skip`.", + "type": "string" + }, + { + "const": "nonUtf8Name", + "description": "The name is not UTF-8; such entries are always skipped.", + "type": "string" + } + ] + }, + "TransferStatus": { + "additionalProperties": false, + "description": "The operation's status: the response of `transfer/begin`,\n`transfer/advance`, `transfer/getStatus`, `transfer/append`,\n`transfer/commit` and `transfer/abort`.", + "properties": { + "deadlineAtMs": { + "description": "When `maxDurationMs` runs out for an active operation.", + "format": "int64", + "type": "integer" + }, + "entryCount": { + "description": "Entries in the inventory so far.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/definitions/ProtocolError" + } + ], + "description": "Why the operation failed; present exactly when `phase` is `failed`." + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + }, + "phase": { + "$ref": "#/definitions/TransferPhase" + }, + "receiptExpiresAtMs": { + "description": "When the receipt of a finished (complete, aborted or failed)\noperation stops being answered; afterwards calls are `notFound`\n(`expired`).", + "format": "int64", + "type": "integer" + }, + "reusedBytes": { + "description": "Content bytes satisfied from files already at the destination.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "skipped": { + "description": "The first\n`MAX_LISTED_SKIPS` entries a\ncapture left out under its policies.", + "items": { + "$ref": "#/definitions/TransferSkip" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Total number of entries left out, listed or not; present whenever it\nis not zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "totalBytes": { + "description": "File bytes in the inventory so far.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "transferredBytes": { + "description": "Content bytes moved over the wire.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "operationId", + "phase", + "entryCount", + "totalBytes", + "transferredBytes", + "reusedBytes" + ], + "type": "object" + }, + "TreeContent": { + "description": "What a tree entry is: a scan entry or a transfer inventory entry.\nDigests identify raw file bytes, not paths or modes.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "directory", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "digest": { + "anyOf": [ + { + "$ref": "#/definitions/ContentDigest" + } + ], + "description": "Scans: present when `includeDigests` was set. Transfer\ninventories: always present; an appended file entry without it\nis `invalidParams`." + }, + "executable": { + "type": "boolean" + }, + "kind": { + "const": "file", + "type": "string" + }, + "sizeBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "sizeBytes" + ], + "type": "object" + } + ] + }, + "TreeFilter": { + "additionalProperties": false, + "description": "Which entries a walk visits. The default visits everything except\nhidden and ignored entries.", + "properties": { + "exclude": { + "description": "Drop entries matching any pattern; a matching directory is pruned.", + "items": { + "type": "string" + }, + "type": "array" + }, + "ignoreCase": { + "description": "Match patterns case-insensitively.", + "type": "boolean" + }, + "include": { + "description": "Select entries matching any pattern; empty selects every entry.", + "items": { + "type": "string" + }, + "type": "array" + }, + "includeHidden": { + "description": "Visit entries whose name starts with `.`.", + "type": "boolean" + }, + "includeIgnored": { + "description": "Visit entries that ignore files exclude.", + "type": "boolean" + }, + "maxDepth": { + "description": "Deepest entry depth visited: the root's direct children are depth\n1, so `1` selects direct children only. Unbounded when absent.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "TreePath": { + "description": "A relative, normalized path below a selected root: segments joined by\n`/`, without empty, `.` or `..` segments. The single segment `.` names\nthe root itself. At most 4096 bytes. Used for scan, inventory and\ntransfer entries.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(\\.|(?!\\.\\.?(/|$))[^/\\u0000]+(/(?!\\.\\.?(/|$))[^/\\u0000]+)*)$", + "type": "string" + }, + "WriteCondition": { + "description": "When `fs/writeFile` may write.", + "oneOf": [ + { + "const": "always", + "description": "Create or replace.", + "type": "string" + }, + { + "const": "ifAbsent", + "description": "Only create; an existing entry is `failedPrecondition`\n(`createCondition`).", + "type": "string" + }, + { + "const": "ifPresent", + "description": "Only replace; a missing file is `failedPrecondition`\n(`createCondition`).", + "type": "string" + } + ] + }, + "WriteFileParams": { + "additionalProperties": false, + "description": "`fs/writeFile` params. The write is staged in the target's directory and\npublished by an atomic rename: readers see the old or the new content,\nnever a mix. An existing target keeps its permissions. A symlink at\n`path` whose target stays inside the root is followed like any path: the\nfinal target is replaced and the link itself is kept (a dangling link\ncreates its target, subject to `create`); a link that leaves the root is\n`permissionDenied` (`outsideRoot`). A special file is\n`failedPrecondition` (`notRegularFile`), a directory\n`failedPrecondition` (`isDirectory`). The parent of the final target\nmust exist.", + "properties": { + "create": { + "$ref": "#/definitions/WriteCondition" + }, + "data": { + "allOf": [ + { + "$ref": "#/definitions/ByteChunk" + } + ], + "description": "At most `limits.filesystem.maxWriteFileBytes`; larger files go\nthrough `transfer/*` with a `writeFile` selection." + }, + "executable": { + "description": "Set the execute bits (the read bits' pattern). Without it a new file\ngets the default mode and an existing file keeps its mode.", + "type": "boolean" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ], + "description": "Write only when the current file's version equals this token;\notherwise `failedPrecondition` (`ifMatchFailed`)." + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path", + "data" + ], + "type": "object" + }, + "WriteFileResponse": { + "additionalProperties": false, + "description": "`fs/writeFile` response.", + "properties": { + "version": { + "allOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ], + "description": "Version of the written file." + } + }, + "required": [ + "version" + ], + "type": "object" + }, + "WriteProcessParams": { + "additionalProperties": false, + "description": "`process/write` params.", + "properties": { + "closeInput": { + "description": "Close the input after `data`: end of file on a pipe, `VEOF` on a\nPTY. Applied only when every byte of `data` was accepted.", + "type": "boolean" + }, + "data": { + "allOf": [ + { + "$ref": "#/definitions/ByteChunk" + } + ], + "description": "Bytes for the process's input pipe or PTY." + }, + "processId": { + "$ref": "#/definitions/ProcessId" + } + }, + "required": [ + "processId" + ], + "type": "object" + }, + "WriteProcessResponse": { + "additionalProperties": false, + "description": "`process/write` response.", + "properties": { + "acceptedBytes": { + "description": "Leading bytes of `data` queued to the process's writer. Fewer than\nsent (possibly 0) means the input queue is full: resend the rest\nlater.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "acceptedBytes" + ], + "type": "object" + }, + "WriteTransferContentParams": { + "additionalProperties": false, + "description": "`transfer/write` params: upload content for a missing digest. Chunks for\none digest arrive in order; an identical chunk may be retried.\nDifferent digests may be uploaded concurrently.", + "properties": { + "data": { + "allOf": [ + { + "$ref": "#/definitions/ByteChunk" + } + ], + "description": "At most `limits.filesystem.transfer.maxContentChunkBytes`." + }, + "digest": { + "$ref": "#/definitions/ContentDigest" + }, + "offset": { + "description": "Must equal the digest's next expected offset.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "digest", + "offset", + "data" + ], + "type": "object" + }, + "WriteTransferContentResponse": { + "additionalProperties": false, + "description": "`transfer/write` response.", + "properties": { + "nextOffset": { + "description": "The digest's next expected offset; equal to its size once complete\nand verified.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "nextOffset" + ], + "type": "object" + } + }, + "description": "Every wire type of the data plane, protocol version 3.", + "title": "Lightspeed environment data plane" +} diff --git a/crates/environment-protocol/contract/registration.openrpc.json b/crates/environment-protocol/contract/registration.openrpc.json new file mode 100644 index 000000000..e5299c57b --- /dev/null +++ b/crates/environment-protocol/contract/registration.openrpc.json @@ -0,0 +1,1420 @@ +{ + "components": { + "schemas": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "Admission": { + "description": "Whether this admission created the environment or rejoined it.", + "oneOf": [ + { + "const": "created", + "description": "First admission of this daemon identity: a new environment.", + "type": "string" + }, + { + "const": "reconnected", + "description": "A known identity reconnected to its environment.", + "type": "string" + } + ] + }, + "ArtifactSha256": { + "description": "`sha256` of a published archive: 64 lowercase hex digits.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "BuildFacts": { + "description": "What `lightspeed-envd --print-build` prints, as one JSON object.", + "properties": { + "gitSha": { + "type": "string" + }, + "name": { + "description": "Always `lightspeed-envd`.", + "type": "string" + }, + "protocolVersion": { + "description": "The binary's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "target": { + "description": "Rust target triple.", + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "name", + "version", + "gitSha", + "target", + "protocolVersion" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "adoptTarget", + "description": "The controller's `adoptTarget` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "ChallengeNonce": { + "description": "`nonce`: the gateway's 32 random bytes for one control connection.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "ChallengeParams": { + "description": "`registration/challenge` params.\n\n**Frozen.** This is the first frame of every control connection and the\nonly frame a daemon of any version must parse to learn it is stale: it\nonly ever gains fields, is decoded tolerantly (unknown members are\nignored), and a receiver reads `protocolVersion` with\n`challenge_protocol_version` before decoding anything else. The whole\nframe is\n`{\"jsonrpc\":\"2.0\",\"method\":\"registration/challenge\",\"params\":{\"protocolVersion\":N,\"nonce\":\"<64 hex>\"}}`.", + "properties": { + "nonce": { + "$ref": "#/components/schemas/ChallengeNonce" + }, + "protocolVersion": { + "description": "The gateway's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "nonce" + ], + "type": "object" + }, + "ClosedParams": { + "additionalProperties": false, + "description": "`registration/closed` params: the gateway ends the session for good and\ncloses the socket after this frame. The daemon exits and does not\nreconnect.", + "properties": { + "message": { + "anyOf": [ + { + "$ref": "#/components/schemas/DescriptiveText" + } + ] + }, + "reason": { + "$ref": "#/components/schemas/ClosedReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "ClosedReason": { + "description": "Why the gateway ended the session for good.", + "oneOf": [ + { + "const": "superseded", + "description": "Another process with this daemon identity connected; the newest\nconnection wins and this daemon exits with its own status naming the\nshared identity.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The identity's environment was closed; the identity is spent.", + "type": "string" + } + ] + }, + "ControlConnectionId": { + "description": "`controlConnectionId` the gateway assigns to one control connection.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DaemonId": { + "description": "`daemonId` the gateway assigns to an admitted daemon identity.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DaemonPublicKey": { + "description": "`daemonPublicKey`: a raw Ed25519 public key.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "DescriptiveText": { + "description": "A short human-readable text: 1 to 1024 bytes without control characters\nother than line feed and tab. Never parsed.", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DiscoveryArtifact": { + "description": "One published envd archive.", + "properties": { + "file": { + "description": "Archive file name.", + "type": "string" + }, + "sha256": { + "allOf": [ + { + "$ref": "#/components/schemas/ArtifactSha256" + } + ], + "description": "Digest of the archive's bytes." + }, + "url": { + "description": "Absolute download URL; absent (or null) when the archive is served\nnext to the document.", + "type": "string" + } + }, + "required": [ + "file", + "sha256" + ], + "type": "object" + }, + "DiscoveryDocument": { + "description": "The discovery document.", + "properties": { + "artifacts": { + "additionalProperties": { + "$ref": "#/components/schemas/DiscoveryArtifact" + }, + "description": "Archives by Rust target triple.", + "type": "object" + }, + "builtAtMs": { + "format": "int64", + "type": "integer" + }, + "channel": { + "description": "Release channel, such as `stable` or `snapshot`.", + "type": "string" + }, + "gitSha": { + "type": "string" + }, + "protocolVersion": { + "description": "The published envd's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "version": { + "description": "Release version of the published envd.", + "type": "string" + } + }, + "required": [ + "version", + "gitSha", + "protocolVersion", + "artifacts" + ], + "type": "object" + }, + "DisplayName": { + "description": "A display name: 1 to 128 bytes without control characters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "Ed25519Signature": { + "description": "`signature`: an Ed25519 signature over\n`signed_registration_message`.", + "pattern": "^[0-9a-f]{128}$", + "type": "string" + }, + "EnvironmentId": { + "description": "`environmentId` of a Lightspeed environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/components/schemas/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/components/schemas/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "GoodbyeParams": { + "additionalProperties": false, + "description": "`registration/goodbye` params: the daemon is stopping deliberately.", + "properties": { + "reason": { + "$ref": "#/components/schemas/GoodbyeReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "GoodbyeReason": { + "description": "Why the daemon is leaving.", + "oneOf": [ + { + "const": "shutdown", + "description": "`SIGTERM` or `SIGINT`.", + "type": "string" + } + ] + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "IncarnationId": { + "description": "`incarnationId` of one incarnation of an environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "Metadata": { + "additionalProperties": { + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "maxProperties": 32, + "propertyNames": { + "maxLength": 64, + "minLength": 1, + "not": { + "pattern": "^lightspeed\\." + }, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "OpenStreamParams": { + "additionalProperties": false, + "description": "`registration/openStream` params: the gateway asks the daemon to dial\none stream.", + "properties": { + "stream": { + "$ref": "#/components/schemas/StreamSpec" + }, + "token": { + "allOf": [ + { + "$ref": "#/components/schemas/SecretString" + } + ], + "description": "One-time bearer token for the stream dial; short-lived." + } + }, + "required": [ + "token", + "stream" + ], + "type": "object" + }, + "OpenStreamResponse": { + "additionalProperties": false, + "description": "`registration/openStream` response, sent once the stream socket is\nupgraded and served. A failed dial answers `unavailable`; the gateway\nfails the waiting route at once.", + "type": "object" + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner, or (`target/adopt`) the owner\nalready holds a different target.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + }, + { + "const": "notAdoptable", + "description": "`target/adopt`: the source target exists but the provider cannot take\nit over, for example an instance of a kind it does not manage.", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/components/schemas/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "RegisterParams": { + "additionalProperties": false, + "description": "`registration/register` params. `registrationKey`, `displayName` and\n`metadata` are read only at a daemon identity's first admission; a\ndaemon sends the key until it is admitted once and never afterwards.\n\nThe gateway decodes them with `RegisterParams::decode`: build facts,\na display name or metadata out of bounds refuse admission with\n`invalidRequest`; anything else that does not decode is\n`invalidParams`.", + "properties": { + "daemonPublicKey": { + "$ref": "#/components/schemas/DaemonPublicKey" + }, + "displayName": { + "anyOf": [ + { + "$ref": "#/components/schemas/DisplayName" + } + ] + }, + "implementation": { + "$ref": "#/components/schemas/ImplementationInfo" + }, + "metadata": { + "$ref": "#/components/schemas/Metadata" + }, + "protocolVersion": { + "description": "The daemon's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "registrationKey": { + "anyOf": [ + { + "$ref": "#/components/schemas/SecretString" + } + ] + }, + "signature": { + "allOf": [ + { + "$ref": "#/components/schemas/Ed25519Signature" + } + ], + "description": "Over `signed_registration_message` of the challenge nonce and the\ncanonical origin of the URL the daemon dialed." + } + }, + "required": [ + "protocolVersion", + "daemonPublicKey", + "signature", + "implementation" + ], + "type": "object" + }, + "RegisterResponse": { + "additionalProperties": false, + "description": "`registration/register` response: the daemon is admitted.", + "properties": { + "heartbeatIntervalMs": { + "description": "How often the gateway pings; see the liveness rule.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "receipt": { + "$ref": "#/components/schemas/RegistrationReceipt" + } + }, + "required": [ + "receipt", + "heartbeatIntervalMs" + ], + "type": "object" + }, + "RegistrationIdentityMode": { + "description": "Registration-key policy for the daemon's environment.", + "oneOf": [ + { + "const": "persistent", + "description": "The environment outlives disconnects.", + "type": "string" + }, + { + "const": "ephemeral", + "description": "The environment closes after it stays disconnected past its grace.", + "type": "string" + } + ] + }, + "RegistrationReceipt": { + "additionalProperties": false, + "description": "What the daemon learns about the environment it serves. Written to the\nreceipt file as is; contains no secret.", + "properties": { + "admission": { + "$ref": "#/components/schemas/Admission" + }, + "controlConnectionId": { + "allOf": [ + { + "$ref": "#/components/schemas/ControlConnectionId" + } + ], + "description": "The gateway's id for this control connection." + }, + "daemonId": { + "$ref": "#/components/schemas/DaemonId" + }, + "environmentId": { + "$ref": "#/components/schemas/EnvironmentId" + }, + "identityMode": { + "$ref": "#/components/schemas/RegistrationIdentityMode" + }, + "incarnationId": { + "$ref": "#/components/schemas/IncarnationId" + } + }, + "required": [ + "environmentId", + "incarnationId", + "daemonId", + "controlConnectionId", + "identityMode", + "admission" + ], + "type": "object" + }, + "SecretString": { + "description": "A secret value carried on the wire, such as a `secretEnv` value, a\nregistration key or a stream token. It serializes as a plain string;\n`Debug` is redacted so structured logging never prints it.", + "type": "string" + }, + "StreamSpec": { + "description": "What the dialed stream carries.", + "oneOf": [ + { + "additionalProperties": false, + "description": "The data plane, served by the daemon as on any data connection.", + "properties": { + "kind": { + "const": "dataProtocol", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + } + } + }, + "info": { + "description": "JSON-RPC 2.0 over WebSocket text frames of at most 16777216 bytes. Exact-match protocolVersion 3. Errors carry code, message and data.kind; see x-lightspeed-error-kinds.", + "title": "Lightspeed environment registration channel", + "version": "3" + }, + "methods": [ + { + "name": "registration/register", + "paramStructure": "by-name", + "params": [ + { + "name": "daemonPublicKey", + "required": true, + "schema": { + "$ref": "#/components/schemas/DaemonPublicKey" + } + }, + { + "name": "displayName", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/DisplayName" + } + ] + } + }, + { + "name": "implementation", + "required": true, + "schema": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + { + "name": "metadata", + "required": false, + "schema": { + "$ref": "#/components/schemas/Metadata" + } + }, + { + "name": "protocolVersion", + "required": true, + "schema": { + "description": "The daemon's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "registrationKey", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SecretString" + } + ] + } + }, + { + "name": "signature", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/Ed25519Signature" + } + ], + "description": "Over `signed_registration_message` of the challenge nonce and the\ncanonical origin of the URL the daemon dialed." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/RegisterResponse" + } + }, + "summary": "Prove the daemon identity and be admitted to its environment.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "RegisterParams", + "x-lightspeed-result-type": "RegisterResponse" + }, + { + "name": "registration/openStream", + "paramStructure": "by-name", + "params": [ + { + "name": "stream", + "required": true, + "schema": { + "$ref": "#/components/schemas/StreamSpec" + } + }, + { + "name": "token", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/SecretString" + } + ], + "description": "One-time bearer token for the stream dial; short-lived." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/OpenStreamResponse" + } + }, + "summary": "Ask the daemon to dial one reverse stream.", + "x-lightspeed-direction": "serverToClient", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "OpenStreamParams", + "x-lightspeed-result-type": "OpenStreamResponse" + }, + { + "name": "registration/challenge", + "paramStructure": "by-name", + "params": [ + { + "name": "nonce", + "required": true, + "schema": { + "$ref": "#/components/schemas/ChallengeNonce" + } + }, + { + "name": "protocolVersion", + "required": true, + "schema": { + "description": "The gateway's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + } + ], + "summary": "The frozen first frame: the gateway's version and nonce.", + "x-lightspeed-direction": "serverToClient", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "notification", + "x-lightspeed-params-type": "ChallengeParams" + }, + { + "name": "registration/goodbye", + "paramStructure": "by-name", + "params": [ + { + "name": "reason", + "required": true, + "schema": { + "$ref": "#/components/schemas/GoodbyeReason" + } + } + ], + "summary": "The daemon is stopping deliberately.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "notification", + "x-lightspeed-params-type": "GoodbyeParams" + }, + { + "name": "registration/closed", + "paramStructure": "by-name", + "params": [ + { + "name": "message", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/DescriptiveText" + } + ] + } + }, + { + "name": "reason", + "required": true, + "schema": { + "$ref": "#/components/schemas/ClosedReason" + } + } + ], + "summary": "The gateway ends the session for good.", + "x-lightspeed-direction": "serverToClient", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "notification", + "x-lightspeed-params-type": "ClosedParams" + } + ], + "openrpc": "1.3.2", + "x-lightspeed-bootstrap": { + "buildFacts": { + "$ref": "#/components/schemas/BuildFacts" + }, + "discoveryDocument": { + "$ref": "#/components/schemas/DiscoveryDocument" + } + }, + "x-lightspeed-error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "x-lightspeed-error-kinds": [ + { + "code": -32700, + "kind": "parseError", + "retryable": false + }, + { + "code": -32600, + "kind": "invalidRequest", + "retryable": false + }, + { + "code": -32601, + "kind": "methodNotFound", + "retryable": false + }, + { + "code": -32602, + "kind": "invalidParams", + "retryable": false + }, + { + "code": -32002, + "kind": "notInitialized", + "retryable": false + }, + { + "code": -32005, + "kind": "versionMismatch", + "retryable": false + }, + { + "code": -32004, + "kind": "notFound", + "retryable": false + }, + { + "code": -32009, + "kind": "alreadyExists", + "retryable": false + }, + { + "code": -32007, + "kind": "failedPrecondition", + "retryable": false + }, + { + "code": -32003, + "kind": "permissionDenied", + "retryable": false + }, + { + "code": -32006, + "kind": "capabilityUnavailable", + "retryable": false + }, + { + "code": -32008, + "kind": "busy", + "retryable": true + }, + { + "code": -32015, + "kind": "deadlineExceeded", + "retryable": false + }, + { + "code": -32016, + "kind": "expired", + "retryable": false + }, + { + "code": -32017, + "kind": "aborted", + "retryable": true + }, + { + "code": -32013, + "kind": "payloadTooLarge", + "retryable": false + }, + { + "code": -32018, + "kind": "processFailed", + "retryable": false + }, + { + "code": -32019, + "kind": "unavailable", + "retryable": true + }, + { + "code": -32603, + "kind": "internal", + "retryable": false + } + ], + "x-lightspeed-plane": "registration" +} diff --git a/crates/environment-protocol/contract/registration.schema.json b/crates/environment-protocol/contract/registration.schema.json new file mode 100644 index 000000000..2749b1691 --- /dev/null +++ b/crates/environment-protocol/contract/registration.schema.json @@ -0,0 +1,1117 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "Admission": { + "description": "Whether this admission created the environment or rejoined it.", + "oneOf": [ + { + "const": "created", + "description": "First admission of this daemon identity: a new environment.", + "type": "string" + }, + { + "const": "reconnected", + "description": "A known identity reconnected to its environment.", + "type": "string" + } + ] + }, + "ArtifactSha256": { + "description": "`sha256` of a published archive: 64 lowercase hex digits.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "BuildFacts": { + "description": "What `lightspeed-envd --print-build` prints, as one JSON object.", + "properties": { + "gitSha": { + "type": "string" + }, + "name": { + "description": "Always `lightspeed-envd`.", + "type": "string" + }, + "protocolVersion": { + "description": "The binary's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "target": { + "description": "Rust target triple.", + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "name", + "version", + "gitSha", + "target", + "protocolVersion" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "adoptTarget", + "description": "The controller's `adoptTarget` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "ChallengeNonce": { + "description": "`nonce`: the gateway's 32 random bytes for one control connection.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "ChallengeParams": { + "description": "`registration/challenge` params.\n\n**Frozen.** This is the first frame of every control connection and the\nonly frame a daemon of any version must parse to learn it is stale: it\nonly ever gains fields, is decoded tolerantly (unknown members are\nignored), and a receiver reads `protocolVersion` with\n`challenge_protocol_version` before decoding anything else. The whole\nframe is\n`{\"jsonrpc\":\"2.0\",\"method\":\"registration/challenge\",\"params\":{\"protocolVersion\":N,\"nonce\":\"<64 hex>\"}}`.", + "properties": { + "nonce": { + "$ref": "#/definitions/ChallengeNonce" + }, + "protocolVersion": { + "description": "The gateway's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "nonce" + ], + "type": "object" + }, + "ClosedParams": { + "additionalProperties": false, + "description": "`registration/closed` params: the gateway ends the session for good and\ncloses the socket after this frame. The daemon exits and does not\nreconnect.", + "properties": { + "message": { + "anyOf": [ + { + "$ref": "#/definitions/DescriptiveText" + } + ] + }, + "reason": { + "$ref": "#/definitions/ClosedReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "ClosedReason": { + "description": "Why the gateway ended the session for good.", + "oneOf": [ + { + "const": "superseded", + "description": "Another process with this daemon identity connected; the newest\nconnection wins and this daemon exits with its own status naming the\nshared identity.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The identity's environment was closed; the identity is spent.", + "type": "string" + } + ] + }, + "ControlConnectionId": { + "description": "`controlConnectionId` the gateway assigns to one control connection.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DaemonId": { + "description": "`daemonId` the gateway assigns to an admitted daemon identity.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DaemonPublicKey": { + "description": "`daemonPublicKey`: a raw Ed25519 public key.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "DescriptiveText": { + "description": "A short human-readable text: 1 to 1024 bytes without control characters\nother than line feed and tab. Never parsed.", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DiscoveryArtifact": { + "description": "One published envd archive.", + "properties": { + "file": { + "description": "Archive file name.", + "type": "string" + }, + "sha256": { + "allOf": [ + { + "$ref": "#/definitions/ArtifactSha256" + } + ], + "description": "Digest of the archive's bytes." + }, + "url": { + "description": "Absolute download URL; absent (or null) when the archive is served\nnext to the document.", + "type": "string" + } + }, + "required": [ + "file", + "sha256" + ], + "type": "object" + }, + "DiscoveryDocument": { + "description": "The discovery document.", + "properties": { + "artifacts": { + "additionalProperties": { + "$ref": "#/definitions/DiscoveryArtifact" + }, + "description": "Archives by Rust target triple.", + "type": "object" + }, + "builtAtMs": { + "format": "int64", + "type": "integer" + }, + "channel": { + "description": "Release channel, such as `stable` or `snapshot`.", + "type": "string" + }, + "gitSha": { + "type": "string" + }, + "protocolVersion": { + "description": "The published envd's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "version": { + "description": "Release version of the published envd.", + "type": "string" + } + }, + "required": [ + "version", + "gitSha", + "protocolVersion", + "artifacts" + ], + "type": "object" + }, + "DisplayName": { + "description": "A display name: 1 to 128 bytes without control characters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "Ed25519Signature": { + "description": "`signature`: an Ed25519 signature over\n`signed_registration_message`.", + "pattern": "^[0-9a-f]{128}$", + "type": "string" + }, + "EnvironmentId": { + "description": "`environmentId` of a Lightspeed environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/definitions/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/definitions/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "GoodbyeParams": { + "additionalProperties": false, + "description": "`registration/goodbye` params: the daemon is stopping deliberately.", + "properties": { + "reason": { + "$ref": "#/definitions/GoodbyeReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "GoodbyeReason": { + "description": "Why the daemon is leaving.", + "oneOf": [ + { + "const": "shutdown", + "description": "`SIGTERM` or `SIGINT`.", + "type": "string" + } + ] + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "IncarnationId": { + "description": "`incarnationId` of one incarnation of an environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "Metadata": { + "additionalProperties": { + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "maxProperties": 32, + "propertyNames": { + "maxLength": 64, + "minLength": 1, + "not": { + "pattern": "^lightspeed\\." + }, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "OpenStreamParams": { + "additionalProperties": false, + "description": "`registration/openStream` params: the gateway asks the daemon to dial\none stream.", + "properties": { + "stream": { + "$ref": "#/definitions/StreamSpec" + }, + "token": { + "allOf": [ + { + "$ref": "#/definitions/SecretString" + } + ], + "description": "One-time bearer token for the stream dial; short-lived." + } + }, + "required": [ + "token", + "stream" + ], + "type": "object" + }, + "OpenStreamResponse": { + "additionalProperties": false, + "description": "`registration/openStream` response, sent once the stream socket is\nupgraded and served. A failed dial answers `unavailable`; the gateway\nfails the waiting route at once.", + "type": "object" + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner, or (`target/adopt`) the owner\nalready holds a different target.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + }, + { + "const": "notAdoptable", + "description": "`target/adopt`: the source target exists but the provider cannot take\nit over, for example an instance of a kind it does not manage.", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "RegisterParams": { + "additionalProperties": false, + "description": "`registration/register` params. `registrationKey`, `displayName` and\n`metadata` are read only at a daemon identity's first admission; a\ndaemon sends the key until it is admitted once and never afterwards.\n\nThe gateway decodes them with `RegisterParams::decode`: build facts,\na display name or metadata out of bounds refuse admission with\n`invalidRequest`; anything else that does not decode is\n`invalidParams`.", + "properties": { + "daemonPublicKey": { + "$ref": "#/definitions/DaemonPublicKey" + }, + "displayName": { + "anyOf": [ + { + "$ref": "#/definitions/DisplayName" + } + ] + }, + "implementation": { + "$ref": "#/definitions/ImplementationInfo" + }, + "metadata": { + "$ref": "#/definitions/Metadata" + }, + "protocolVersion": { + "description": "The daemon's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "registrationKey": { + "anyOf": [ + { + "$ref": "#/definitions/SecretString" + } + ] + }, + "signature": { + "allOf": [ + { + "$ref": "#/definitions/Ed25519Signature" + } + ], + "description": "Over `signed_registration_message` of the challenge nonce and the\ncanonical origin of the URL the daemon dialed." + } + }, + "required": [ + "protocolVersion", + "daemonPublicKey", + "signature", + "implementation" + ], + "type": "object" + }, + "RegisterResponse": { + "additionalProperties": false, + "description": "`registration/register` response: the daemon is admitted.", + "properties": { + "heartbeatIntervalMs": { + "description": "How often the gateway pings; see the liveness rule.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "receipt": { + "$ref": "#/definitions/RegistrationReceipt" + } + }, + "required": [ + "receipt", + "heartbeatIntervalMs" + ], + "type": "object" + }, + "RegistrationIdentityMode": { + "description": "Registration-key policy for the daemon's environment.", + "oneOf": [ + { + "const": "persistent", + "description": "The environment outlives disconnects.", + "type": "string" + }, + { + "const": "ephemeral", + "description": "The environment closes after it stays disconnected past its grace.", + "type": "string" + } + ] + }, + "RegistrationReceipt": { + "additionalProperties": false, + "description": "What the daemon learns about the environment it serves. Written to the\nreceipt file as is; contains no secret.", + "properties": { + "admission": { + "$ref": "#/definitions/Admission" + }, + "controlConnectionId": { + "allOf": [ + { + "$ref": "#/definitions/ControlConnectionId" + } + ], + "description": "The gateway's id for this control connection." + }, + "daemonId": { + "$ref": "#/definitions/DaemonId" + }, + "environmentId": { + "$ref": "#/definitions/EnvironmentId" + }, + "identityMode": { + "$ref": "#/definitions/RegistrationIdentityMode" + }, + "incarnationId": { + "$ref": "#/definitions/IncarnationId" + } + }, + "required": [ + "environmentId", + "incarnationId", + "daemonId", + "controlConnectionId", + "identityMode", + "admission" + ], + "type": "object" + }, + "SecretString": { + "description": "A secret value carried on the wire, such as a `secretEnv` value, a\nregistration key or a stream token. It serializes as a plain string;\n`Debug` is redacted so structured logging never prints it.", + "type": "string" + }, + "StreamSpec": { + "description": "What the dialed stream carries.", + "oneOf": [ + { + "additionalProperties": false, + "description": "The data plane, served by the daemon as on any data connection.", + "properties": { + "kind": { + "const": "dataProtocol", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + } + }, + "description": "Every wire type of the registration plane, protocol version 3.", + "title": "Lightspeed environment registration channel", + "x-lightspeed-bootstrap": { + "buildFacts": { + "$ref": "#/definitions/BuildFacts" + }, + "discoveryDocument": { + "$ref": "#/definitions/DiscoveryDocument" + } + } +} diff --git a/crates/environment-protocol/fixtures/bootstrap/build-facts.json b/crates/environment-protocol/fixtures/bootstrap/build-facts.json new file mode 100644 index 000000000..e4dd66a6b --- /dev/null +++ b/crates/environment-protocol/fixtures/bootstrap/build-facts.json @@ -0,0 +1,7 @@ +{ + "name": "lightspeed-envd", + "version": "0.5.0", + "gitSha": "86f6adf3", + "target": "x86_64-unknown-linux-musl", + "protocolVersion": 3 +} diff --git a/crates/environment-protocol/fixtures/bootstrap/discovery-document.json b/crates/environment-protocol/fixtures/bootstrap/discovery-document.json new file mode 100644 index 000000000..b02b13851 --- /dev/null +++ b/crates/environment-protocol/fixtures/bootstrap/discovery-document.json @@ -0,0 +1,14 @@ +{ + "version": "0.5.0", + "gitSha": "86f6adf3", + "channel": "stable", + "protocolVersion": 3, + "builtAtMs": 1761400000000, + "artifacts": { + "x86_64-unknown-linux-musl": { + "file": "lightspeed-envd-x86_64-unknown-linux-musl.tar.gz", + "sha256": "abababababababababababababababababababababababababababababababab", + "url": "https://releases.example.com/lightspeed-envd-x86_64-unknown-linux-musl.tar.gz" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/ingress.ensure.json b/crates/environment-protocol/fixtures/controller/ingress.ensure.json new file mode 100644 index 000000000..a594e5383 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/ingress.ensure.json @@ -0,0 +1,49 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "targetId": "ls-3f2a" + } + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "displayName": "env_1", + "state": { + "phase": "active", + "power": "running", + "daemonReady": true + }, + "supportedPowerStates": [ + "running", + "paused", + "stopped" + ], + "metadata": { + "incusMember": "node-1" + }, + "ingress": { + "endpoint": "https://env-1.apps.example.com" + } + } + }, + "error": { + "code": -32006, + "message": "this provider serves no public ingress", + "data": { + "kind": "capabilityUnavailable", + "capability": "ingress" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/ingress.remove.json b/crates/environment-protocol/fixtures/controller/ingress.remove.json new file mode 100644 index 000000000..f5ab1d2d7 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/ingress.remove.json @@ -0,0 +1,45 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "targetId": "ls-3f2a" + } + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "displayName": "env_1", + "state": { + "phase": "active", + "power": "running", + "daemonReady": true + }, + "supportedPowerStates": [ + "running", + "paused", + "stopped" + ], + "metadata": { + "incusMember": "node-1" + } + } + }, + "error": { + "code": -32004, + "message": "target ls-3f2a no longer exists", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/initialize.json b/crates/environment-protocol/fixtures/controller/initialize.json new file mode 100644 index 000000000..a449dd0cf --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/initialize.json @@ -0,0 +1,33 @@ +{ + "params": { + "protocolVersion": 3, + "clientInfo": { + "name": "lightspeed-runtime", + "version": "0.5.0" + } + }, + "response": { + "protocolVersion": 3, + "plane": "controller", + "serverInfo": { + "name": "lightspeed-provider-incus", + "version": "0.5.0" + }, + "capabilities": { + "adoptTarget": true, + "ingress": true + }, + "routes": { + "data": "wss://incus.example.com:19090/routes" + } + }, + "error": { + "code": -32005, + "message": "protocol version 4 is not supported; this peer speaks 3", + "data": { + "kind": "versionMismatch", + "expected": 3, + "received": 4 + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.adopt.json b/crates/environment-protocol/fixtures/controller/target.adopt.json new file mode 100644 index 000000000..a8dc7100a --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.adopt.json @@ -0,0 +1,44 @@ +{ + "params": { + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "sourceTarget": "default/legacy-vm" + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "displayName": "env_1", + "state": { + "phase": "active", + "power": "running", + "daemonReady": true + }, + "supportedPowerStates": [ + "running", + "paused", + "stopped" + ], + "metadata": { + "incusMember": "node-1" + } + } + }, + "error": { + "code": -32006, + "message": "this provider does not adopt targets", + "data": { + "kind": "capabilityUnavailable", + "capability": "adoptTarget" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.close.json b/crates/environment-protocol/fixtures/controller/target.close.json new file mode 100644 index 000000000..06dd35918 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.close.json @@ -0,0 +1,33 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + } + } + }, + "response": { + "target": { + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "state": { + "phase": "closed" + } + } + }, + "error": { + "code": -32019, + "message": "the Incus API is unreachable", + "data": { + "kind": "unavailable", + "retryAfterMs": 5000 + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.create.json b/crates/environment-protocol/fixtures/controller/target.create.json new file mode 100644 index 000000000..48700553e --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.create.json @@ -0,0 +1,34 @@ +{ + "params": { + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "templateId": "ubuntu-24.04-small" + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "state": { + "phase": "provisioning", + "message": "pulling image" + } + } + }, + "error": { + "code": -32007, + "message": "template ubuntu-24.04-small is deprecated", + "data": { + "kind": "failedPrecondition", + "reason": "templateDeprecated" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.get.json b/crates/environment-protocol/fixtures/controller/target.get.json new file mode 100644 index 000000000..edf6ab717 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.get.json @@ -0,0 +1,48 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "targetId": "ls-3f2a" + } + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "displayName": "env_1", + "state": { + "phase": "active", + "power": "running", + "daemonReady": true + }, + "supportedPowerStates": [ + "running", + "paused", + "stopped" + ], + "metadata": { + "incusMember": "node-1" + }, + "ingress": { + "endpoint": "https://env-1.apps.example.com" + } + } + }, + "error": { + "code": -32004, + "message": "target ls-3f2a no longer exists", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.setPower.json b/crates/environment-protocol/fixtures/controller/target.setPower.json new file mode 100644 index 000000000..9f9da2c9b --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.setPower.json @@ -0,0 +1,46 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "targetId": "ls-3f2a" + }, + "power": "paused" + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "bindingId": "primary", + "environmentId": "env_1", + "incarnationId": "inc_1" + }, + "displayName": "env_1", + "state": { + "phase": "active", + "power": "paused" + }, + "supportedPowerStates": [ + "running", + "paused", + "stopped" + ], + "metadata": { + "incusMember": "node-1" + } + } + }, + "error": { + "code": -32007, + "message": "target ls-3f2a belongs to another incarnation", + "data": { + "kind": "failedPrecondition", + "reason": "ownerMismatch" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/template.list.json b/crates/environment-protocol/fixtures/controller/template.list.json new file mode 100644 index 000000000..e0126e578 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/template.list.json @@ -0,0 +1,34 @@ +{ + "params": { + "binding": { + "universeId": "u1", + "bindingId": "primary" + } + }, + "response": { + "templates": [ + { + "templateId": "ubuntu-24.04-small", + "displayName": "Ubuntu 24.04 (small)", + "description": "2 vCPU, 4 GiB", + "publicIngress": true, + "deprecated": true, + "resources": { + "vcpus": 2, + "memoryBytes": 4294967296, + "diskBytes": 21474836480 + }, + "metadata": { + "imageFingerprint": "a1b2c3" + } + } + ] + }, + "error": { + "code": -32019, + "message": "the Incus API is unreachable", + "data": { + "kind": "unavailable" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller_create_target_params.json b/crates/environment-protocol/fixtures/controller_create_target_params.json deleted file mode 100644 index 2d2ffa85c..000000000 --- a/crates/environment-protocol/fixtures/controller_create_target_params.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "binding": { - "bindingId": "primary", - "universeId": "universe-1" - }, - "environmentId": "environment-1", - "incarnationId": "incarnation-1", - "requestId": "request-1", - "templateId": "lightspeed-dev-v1" -} diff --git a/crates/environment-protocol/fixtures/controller_create_target_response.json b/crates/environment-protocol/fixtures/controller_create_target_response.json deleted file mode 100644 index 10b9ddf56..000000000 --- a/crates/environment-protocol/fixtures/controller_create_target_response.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "target": { - "capabilities": { - "filesystemRead": true, - "filesystemSearch": true, - "filesystemGlob": true, - "filesystemRangedRead": true, - "filesystemWrite": true, - "jobCancel": true, - "jobDependencies": true, - "jobList": true, - "jobRead": true, - "jobQueueKeys": true, - "jobStart": true, - "jobWaitHint": false, - "network": true, - "processOutputNotifications": true, - "processOutputPolling": true, - "processPty": false, - "processStart": true, - "processStdin": true, - "processTerminate": true - }, - "defaultCwd": "/workspace", - "displayName": "lightspeed dev", - "metadata": { - "provider": "smolvm" - }, - "scope": { - "sessionId": "sandbox-123", - "type": "session" - }, - "status": "ready", - "targetId": "sandbox-123" - } -} diff --git a/crates/environment-protocol/fixtures/controller_initialize_params.json b/crates/environment-protocol/fixtures/controller_initialize_params.json deleted file mode 100644 index 55abb5153..000000000 --- a/crates/environment-protocol/fixtures/controller_initialize_params.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "clientName": "lightspeed-test", - "protocolVersion": 2 -} diff --git a/crates/environment-protocol/fixtures/controller_list_targets_response.json b/crates/environment-protocol/fixtures/controller_list_targets_response.json deleted file mode 100644 index 5cb884925..000000000 --- a/crates/environment-protocol/fixtures/controller_list_targets_response.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "targets": [ - { - "capabilities": { - "filesystemRead": true, - "filesystemSearch": true, - "filesystemGlob": true, - "filesystemRangedRead": true, - "filesystemWrite": true, - "jobCancel": true, - "jobDependencies": true, - "jobList": true, - "jobRead": true, - "jobQueueKeys": true, - "jobStart": true, - "jobWaitHint": false, - "network": true, - "processOutputNotifications": true, - "processOutputPolling": true, - "processPty": false, - "processStart": true, - "processStdin": true, - "processTerminate": true - }, - "defaultCwd": "/workspace", - "displayName": "lightspeed dev", - "metadata": { - "provider": "smolvm" - }, - "scope": { - "sessionId": "sandbox-123", - "type": "session" - }, - "status": "ready", - "targetId": "sandbox-123" - } - ] -} diff --git a/crates/environment-protocol/fixtures/data/env.getIdle.json b/crates/environment-protocol/fixtures/data/env.getIdle.json new file mode 100644 index 000000000..9bd9ac05e --- /dev/null +++ b/crates/environment-protocol/fixtures/data/env.getIdle.json @@ -0,0 +1,15 @@ +{ + "params": {}, + "response": { + "idleForMs": 0, + "runningProcessCount": 1, + "leftoverProcessGroupCount": 1 + }, + "error": { + "code": -32002, + "message": "initialize must be the first request on a connection", + "data": { + "kind": "notInitialized" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.createDirectory.json b/crates/environment-protocol/fixtures/data/fs.createDirectory.json new file mode 100644 index 000000000..705890f70 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.createDirectory.json @@ -0,0 +1,15 @@ +{ + "params": { + "path": "/workspace/build/out", + "recursive": true + }, + "response": {}, + "error": { + "code": -32006, + "message": "the filesystem is read-only", + "data": { + "kind": "capabilityUnavailable", + "capability": "filesystemWrite" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.getMetadata.json b/crates/environment-protocol/fixtures/data/fs.getMetadata.json new file mode 100644 index 000000000..d8c16aba5 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.getMetadata.json @@ -0,0 +1,24 @@ +{ + "params": { + "path": "/workspace/link", + "followSymlinks": true + }, + "response": { + "metadata": { + "kind": "file", + "sizeBytes": 1024, + "executable": true, + "modifiedAtMs": 1761400000000, + "createdAtMs": 1761300000000, + "version": "v1:1761400000000000000:1024:42" + } + }, + "error": { + "code": -32003, + "message": "/workspace/link resolves outside the filesystem root", + "data": { + "kind": "permissionDenied", + "reason": "outsideRoot" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.globFiles.json b/crates/environment-protocol/fixtures/data/fs.globFiles.json new file mode 100644 index 000000000..f10ee1502 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.globFiles.json @@ -0,0 +1,33 @@ +{ + "params": { + "root": "/workspace", + "filter": { + "include": [ + "**/SKILL.md" + ], + "ignoreCase": true, + "includeHidden": true, + "includeIgnored": true + }, + "limits": { + "maxMatches": 1000, + "maxFiles": 5000, + "maxDurationMs": 30000 + } + }, + "response": { + "matches": [ + "/workspace/.agents/skills/review/SKILL.md" + ], + "visitedFileCount": 310, + "elapsedMs": 12 + }, + "error": { + "code": -32017, + "message": "request cancelled", + "data": { + "kind": "aborted", + "reason": "cancelled" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.readDirectory.json b/crates/environment-protocol/fixtures/data/fs.readDirectory.json new file mode 100644 index 000000000..430060a53 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.readDirectory.json @@ -0,0 +1,40 @@ +{ + "params": { + "path": "/workspace", + "limit": 2, + "cursor": "c:src" + }, + "response": { + "entries": [ + { + "name": "src", + "metadata": { + "kind": "directory", + "sizeBytes": 4096, + "executable": true, + "modifiedAtMs": 1761400000000, + "version": "v1:1761400000000000000:4096:7" + } + }, + { + "name": "link", + "metadata": { + "kind": "symlink", + "sizeBytes": 6, + "modifiedAtMs": 1761400000000, + "symlinkTarget": "target", + "version": "v1:1761400000000000000:6:9" + } + } + ], + "nextCursor": "c:link", + "skippedEntryCount": 1 + }, + "error": { + "code": -32004, + "message": "/workspace/missing does not exist", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.readFile.json b/crates/environment-protocol/fixtures/data/fs.readFile.json new file mode 100644 index 000000000..e4deeccc3 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.readFile.json @@ -0,0 +1,21 @@ +{ + "params": { + "path": "/workspace/src/lib.rs", + "offset": 1024, + "maxBytes": 65536 + }, + "response": { + "data": "aGVsbG8=", + "fileSize": 4096, + "truncated": true, + "version": "v1:1761400000000000000:4096:42" + }, + "error": { + "code": -32007, + "message": "/dev/zero is not a regular file", + "data": { + "kind": "failedPrecondition", + "reason": "notRegularFile" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.remove.json b/crates/environment-protocol/fixtures/data/fs.remove.json new file mode 100644 index 000000000..e2a9d4e7c --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.remove.json @@ -0,0 +1,16 @@ +{ + "params": { + "path": "/workspace/build", + "recursive": true, + "force": true + }, + "response": {}, + "error": { + "code": -32003, + "message": "/var/lib/lightspeed-envd is the daemon state directory", + "data": { + "kind": "permissionDenied", + "reason": "daemonState" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.rename.json b/crates/environment-protocol/fixtures/data/fs.rename.json new file mode 100644 index 000000000..cbca3341d --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.rename.json @@ -0,0 +1,16 @@ +{ + "params": { + "source": "/workspace/a.rs", + "destination": "/workspace/b.rs", + "overwrite": true, + "ifMatch": "v1:1761400000000000000:4096:42" + }, + "response": {}, + "error": { + "code": -32009, + "message": "/workspace/b.rs exists", + "data": { + "kind": "alreadyExists" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.scan.json b/crates/environment-protocol/fixtures/data/fs.scan.json new file mode 100644 index 000000000..b4ad2ca79 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.scan.json @@ -0,0 +1,89 @@ +{ + "params": { + "roots": [ + { + "path": "/workspace", + "maxDepth": 0 + }, + { + "path": "/workspace/.agents/skills", + "ifNoneMatch": "scan:v1:9f2c" + }, + { + "path": "/home/dev/.agents/skills" + } + ], + "filter": { + "include": [ + "SKILL.md", + "*/SKILL.md" + ], + "maxDepth": 2 + }, + "readContent": true, + "includeDigests": true, + "followSymlinks": true, + "limits": { + "maxEntries": 4096, + "maxFileBytes": 65536, + "maxTotalBytes": 4194304, + "maxDurationMs": 2000 + } + }, + "response": { + "roots": [ + { + "kind": "complete", + "path": "/workspace", + "fingerprint": "scan:v1:0001" + }, + { + "kind": "unchanged", + "path": "/workspace/.agents/skills", + "fingerprint": "scan:v1:9f2c" + }, + { + "kind": "complete", + "path": "/home/dev/.agents/skills", + "canonicalPath": "/home/dev/dotfiles/skills", + "fingerprint": "scan:v1:77aa", + "entries": [ + { + "path": "review", + "content": { + "kind": "directory" + } + }, + { + "path": "review/SKILL.md", + "content": { + "kind": "file", + "sizeBytes": 5, + "digest": "sha256:abababababababababababababababababababababababababababababababab" + }, + "data": "aGVsbG8=" + } + ], + "skipped": [ + { + "path": "huge/SKILL.md", + "reason": "fileTooLarge" + }, + { + "reason": "nonUtf8Name" + } + ], + "skippedEntryCount": 2 + } + ] + }, + "error": { + "code": -32602, + "message": "roots has 40 entries; the ceiling is 32", + "data": { + "kind": "invalidParams", + "field": "roots", + "ceiling": 32 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.searchText.json b/crates/environment-protocol/fixtures/data/fs.searchText.json new file mode 100644 index 000000000..59f51f6eb --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.searchText.json @@ -0,0 +1,47 @@ +{ + "params": { + "root": "/workspace", + "pattern": "fn main", + "patternIgnoreCase": true, + "filter": { + "include": [ + "**/*.rs" + ], + "exclude": [ + "target/**" + ], + "maxDepth": 8 + }, + "limits": { + "maxMatches": 1000, + "maxFiles": 5000, + "maxBytes": 67108864, + "maxDurationMs": 30000, + "maxLineBytes": 4096 + } + }, + "response": { + "matches": [ + { + "path": "/workspace/src/main.rs", + "lineNumber": 3, + "line": "fn main() {" + } + ], + "searchedFileCount": 120, + "searchedBytes": 480000, + "skippedFileCount": 2, + "skippedEntryCount": 1, + "elapsedMs": 41, + "stopped": "matchLimit" + }, + "error": { + "code": -32602, + "message": "limits.maxDurationMs is 90000; the ceiling is 60000", + "data": { + "kind": "invalidParams", + "field": "limits.maxDurationMs", + "ceiling": 60000 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.writeFile.json b/crates/environment-protocol/fixtures/data/fs.writeFile.json new file mode 100644 index 000000000..15ba5f557 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.writeFile.json @@ -0,0 +1,20 @@ +{ + "params": { + "path": "/workspace/src/lib.rs", + "data": "aGVsbG8=", + "create": "ifPresent", + "ifMatch": "v1:1761400000000000000:4096:42", + "executable": true + }, + "response": { + "version": "v1:1761400001000000000:5:42" + }, + "error": { + "code": -32007, + "message": "/workspace/src/lib.rs changed since it was read", + "data": { + "kind": "failedPrecondition", + "reason": "ifMatchFailed" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/initialize.json b/crates/environment-protocol/fixtures/data/initialize.json new file mode 100644 index 000000000..2c98620d0 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/initialize.json @@ -0,0 +1,97 @@ +{ + "params": { + "protocolVersion": 3, + "clientInfo": { + "name": "lightspeed-runtime", + "version": "0.5.0" + } + }, + "response": { + "protocolVersion": 3, + "plane": "data", + "daemonInstanceId": "envd_01jb8x7k2m", + "serverInfo": { + "name": "lightspeed-envd", + "version": "0.5.0", + "gitSha": "86f6adf3", + "target": "x86_64-unknown-linux-musl" + }, + "capabilities": { + "filesystem": { + "write": true + }, + "process": { + "pty": true + }, + "job": {} + }, + "limits": { + "maxConcurrentRequests": 32, + "maxWaitMs": 1800000, + "process": { + "maxReadBytes": 4194304, + "maxRetainedOutputBytes": 1048576, + "retentionAfterExitMs": 60000, + "unreadRetentionMs": 7200000, + "maxExitedRetainedBytes": 67108864, + "terminateGraceMs": 5000, + "maxInputQueueBytes": 1048576 + }, + "job": { + "maxReadBytes": 4194304, + "maxRequestJobs": 64, + "maxRetainedOutputBytes": 2097152, + "recordRetentionMs": 604800000, + "cancelGraceMs": 5000 + }, + "filesystem": { + "maxReadFileBytes": 8388608, + "maxWriteFileBytes": 8388608, + "maxDirectoryEntries": 1000, + "search": { + "maxMatches": 10000, + "maxFiles": 100000, + "maxBytes": 1073741824, + "maxDurationMs": 60000, + "maxLineBytes": 4096 + }, + "glob": { + "maxMatches": 100000, + "maxFiles": 1000000, + "maxDurationMs": 60000 + }, + "scan": { + "maxRoots": 32, + "maxPatterns": 32, + "maxEntries": 10000, + "maxFileBytes": 262144, + "maxTotalBytes": 4194304, + "maxDurationMs": 30000 + }, + "transfer": { + "maxActiveOperations": 8, + "maxContentChunkBytes": 4194304, + "maxInventoryPageEntries": 1000, + "receiptRetentionMs": 86400000, + "maxEntries": 100000, + "maxDepth": 64, + "maxFileBytes": 1099511627776, + "maxTotalBytes": 1099511627776, + "maxManifestBytes": 33554432, + "maxDurationMs": 86400000 + } + } + }, + "defaultCwd": "/workspace", + "homeDirectory": "/home/dev" + }, + "error": { + "code": -32005, + "message": "protocol version 2 is not supported; this peer speaks 3", + "data": { + "kind": "versionMismatch", + "expected": 3, + "received": 2 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/job.cancel.json b/crates/environment-protocol/fixtures/data/job.cancel.json new file mode 100644 index 000000000..d16db2602 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/job.cancel.json @@ -0,0 +1,43 @@ +{ + "params": { + "jobIds": [ + "build" + ], + "scope": "dependents", + "force": true + }, + "response": { + "jobs": [ + { + "kind": "job", + "summary": { + "jobId": "build", + "requestId": "jobreq-run1-turn2-call3", + "status": "running", + "createdAtMs": 1761400000000, + "startedAtMs": 1761400000200, + "cancelRequestedAtMs": 1761400005000 + } + }, + { + "kind": "error", + "jobId": "deploy", + "error": { + "code": -32004, + "message": "unknown job id: deploy", + "data": { + "kind": "notFound" + } + } + } + ] + }, + "error": { + "code": -32602, + "message": "jobIds must not be empty", + "data": { + "kind": "invalidParams", + "field": "jobIds" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/job.read.json b/crates/environment-protocol/fixtures/data/job.read.json new file mode 100644 index 000000000..811427576 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/job.read.json @@ -0,0 +1,115 @@ +{ + "params": { + "jobs": [ + { + "jobId": "build", + "tailBytes": 8192, + "maxBytes": 8192 + }, + { + "jobId": "setup", + "after": 512 + }, + { + "jobId": "gone" + } + ], + "maxBytes": 16384, + "waitMs": 0, + "returnOn": "exit" + }, + "response": { + "jobs": [ + { + "kind": "job", + "summary": { + "jobId": "build", + "requestId": "jobreq-run1-turn2-call3", + "name": "build", + "status": "failed", + "dependsOn": [ + "setup" + ], + "queueKey": "cargo", + "createdAtMs": 1761400000000, + "queuedAtMs": 1761400000100, + "startedAtMs": 1761400000200, + "finishedAtMs": 1761400009200, + "durationMs": 9000, + "exit": { + "kind": "code", + "code": 101 + }, + "leftovers": { + "disposition": "swept", + "processes": [ + { + "pid": 4242, + "command": "sleep 600" + } + ] + }, + "error": { + "code": -32018, + "message": "output capture failed", + "data": { + "kind": "processFailed" + } + } + }, + "output": { + "chunks": [ + { + "offset": 3000000, + "stream": "stderr", + "data": "ZXJyb3I=" + } + ], + "next": 3000005, + "end": 3000005 + } + }, + { + "kind": "job", + "summary": { + "jobId": "setup", + "requestId": "jobreq-run1-turn2-call3", + "name": "setup", + "status": "succeeded", + "createdAtMs": 1761400000000, + "startedAtMs": 1761400000050, + "finishedAtMs": 1761400003050, + "durationMs": 3000, + "exit": { + "kind": "code", + "code": 0 + } + }, + "output": { + "next": 512, + "end": 512 + } + }, + { + "kind": "error", + "jobId": "gone", + "error": { + "code": -32004, + "message": "unknown job id: gone", + "data": { + "kind": "notFound" + } + } + } + ] + }, + "error": { + "code": -32602, + "message": "waitMs is 3600000; the ceiling is 1800000", + "data": { + "kind": "invalidParams", + "field": "waitMs", + "ceiling": 1800000 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/job.start.json b/crates/environment-protocol/fixtures/data/job.start.json new file mode 100644 index 000000000..af185d018 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/job.start.json @@ -0,0 +1,75 @@ +{ + "params": { + "requestId": "jobreq-run1-turn2-call3", + "jobs": [ + { + "jobId": "setup", + "name": "setup", + "command": { + "argv": [ + "npm", + "ci" + ] + } + }, + { + "jobId": "build", + "command": { + "argv": [ + "cargo", + "build" + ], + "cwd": "/workspace", + "input": { + "kind": "bytes", + "data": "aGVsbG8=" + }, + "timeoutMs": 1800000 + }, + "dependsOn": [ + { + "kind": "name", + "name": "setup" + }, + { + "kind": "jobId", + "jobId": "lint" + } + ], + "dependencyPolicy": "allTerminal", + "queueKey": "cargo" + } + ] + }, + "response": { + "jobs": [ + { + "jobId": "setup", + "requestId": "jobreq-run1-turn2-call3", + "name": "setup", + "status": "running", + "createdAtMs": 1761400000000, + "startedAtMs": 1761400000050 + }, + { + "jobId": "build", + "requestId": "jobreq-run1-turn2-call3", + "status": "queued", + "dependsOn": [ + "setup", + "lint" + ], + "queueKey": "cargo", + "createdAtMs": 1761400000000 + } + ] + }, + "error": { + "code": -32007, + "message": "job id build names an earlier run; choose a new job id", + "data": { + "kind": "failedPrecondition", + "reason": "jobIdInUse" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/process.read.json b/crates/environment-protocol/fixtures/data/process.read.json new file mode 100644 index 000000000..74529eb76 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/process.read.json @@ -0,0 +1,63 @@ +{ + "params": { + "processId": "proc-7", + "after": 16384, + "ack": 16384, + "maxBytes": 65536, + "waitMs": 20000, + "returnOn": "anyOutput" + }, + "response": { + "output": { + "chunks": [ + { + "offset": 16384, + "stream": "stdout", + "data": "aGVsbG8=" + }, + { + "offset": 1048581, + "stream": "stderr", + "data": "ZmFpbA==" + } + ], + "next": 1048585, + "end": 1048585, + "gaps": [ + { + "offset": 16389, + "omittedBytes": 1032192, + "reason": "overflow" + } + ] + }, + "state": "exited", + "exit": { + "kind": "signal", + "signal": "SIGSEGV", + "coreDumped": true + }, + "stoppedBy": "terminate", + "pid": 4242, + "leftovers": { + "disposition": "kept", + "processes": [ + { + "pid": 4250, + "command": "postgres -D /data" + }, + { + "pid": 4251 + } + ] + } + }, + "error": { + "code": -32004, + "message": "unknown process id: proc-7", + "data": { + "kind": "notFound", + "reason": "expired" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/process.start.json b/crates/environment-protocol/fixtures/data/process.start.json new file mode 100644 index 000000000..0cf4495bc --- /dev/null +++ b/crates/environment-protocol/fixtures/data/process.start.json @@ -0,0 +1,38 @@ +{ + "params": { + "processId": "proc-7", + "command": { + "argv": [ + "sh", + "-lc", + "cat" + ], + "cwd": "/workspace", + "env": { + "RUST_LOG": "debug" + }, + "secretEnv": { + "API_TOKEN": "s3cr3t" + }, + "input": { + "kind": "bytes", + "data": "aGVsbG8=" + }, + "timeoutMs": 600000 + }, + "pty": { + "rows": 40, + "cols": 120 + } + }, + "response": { + "pid": 4242 + }, + "error": { + "code": -32009, + "message": "process id proc-7 is in use", + "data": { + "kind": "alreadyExists" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/process.terminate.json b/crates/environment-protocol/fixtures/data/process.terminate.json new file mode 100644 index 000000000..e0596a728 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/process.terminate.json @@ -0,0 +1,17 @@ +{ + "params": { + "processId": "proc-7", + "signal": "terminate" + }, + "response": { + "running": true + }, + "error": { + "code": -32004, + "message": "unknown process id: proc-7", + "data": { + "kind": "notFound", + "reason": "daemonRestarted" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/process.write.json b/crates/environment-protocol/fixtures/data/process.write.json new file mode 100644 index 000000000..efff05c73 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/process.write.json @@ -0,0 +1,18 @@ +{ + "params": { + "processId": "proc-7", + "data": "aGVsbG8K", + "closeInput": true + }, + "response": { + "acceptedBytes": 6 + }, + "error": { + "code": -32007, + "message": "process proc-7 has no open input", + "data": { + "kind": "failedPrecondition", + "reason": "inputClosed" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/rpc.cancelRequest.json b/crates/environment-protocol/fixtures/data/rpc.cancelRequest.json new file mode 100644 index 000000000..b3e9ca10a --- /dev/null +++ b/crates/environment-protocol/fixtures/data/rpc.cancelRequest.json @@ -0,0 +1,5 @@ +{ + "params": { + "id": 42 + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.abort.json b/crates/environment-protocol/fixtures/data/transfer.abort.json new file mode 100644 index 000000000..8229f7231 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.abort.json @@ -0,0 +1,22 @@ +{ + "params": { + "operationId": "op-2" + }, + "response": { + "operationId": "op-2", + "phase": "aborted", + "entryCount": 3, + "totalBytes": 2048, + "transferredBytes": 1024, + "reusedBytes": 512, + "deadlineAtMs": 1761400000000, + "receiptExpiresAtMs": 1761486400000 + }, + "error": { + "code": -32004, + "message": "unknown operation op-2", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.advance.json b/crates/environment-protocol/fixtures/data/transfer.advance.json new file mode 100644 index 000000000..c6f52bfba --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.advance.json @@ -0,0 +1,38 @@ +{ + "params": { + "operationId": "op-1" + }, + "response": { + "operationId": "op-1", + "phase": "failed", + "entryCount": 3, + "totalBytes": 2048, + "transferredBytes": 1024, + "reusedBytes": 512, + "deadlineAtMs": 1761400000000, + "skipped": [ + { + "path": "node_modules/.bin/tsc", + "reason": "symlink" + } + ], + "skippedEntryCount": 140, + "error": { + "code": -32017, + "message": "src/lib.rs changed while it was read", + "data": { + "kind": "aborted", + "reason": "sourceChanged" + } + }, + "receiptExpiresAtMs": 1761486400000 + }, + "error": { + "code": -32007, + "message": "advance is not valid in the complete phase", + "data": { + "kind": "failedPrecondition", + "reason": "transferPhase" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.append.json b/crates/environment-protocol/fixtures/data/transfer.append.json new file mode 100644 index 000000000..4270216ae --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.append.json @@ -0,0 +1,40 @@ +{ + "params": { + "operationId": "op-2", + "index": 0, + "entries": [ + { + "path": ".", + "content": { + "kind": "directory" + } + }, + { + "path": "README.md", + "content": { + "kind": "file", + "sizeBytes": 12, + "digest": "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd" + } + } + ], + "last": true + }, + "response": { + "operationId": "op-2", + "phase": "scanning", + "entryCount": 2, + "totalBytes": 12, + "transferredBytes": 0, + "reusedBytes": 0, + "deadlineAtMs": 1761400000000 + }, + "error": { + "code": -32007, + "message": "operation op-2 was begun with different input", + "data": { + "kind": "failedPrecondition", + "reason": "operationIdInUse" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.begin.json b/crates/environment-protocol/fixtures/data/transfer.begin.json new file mode 100644 index 000000000..7b548e881 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.begin.json @@ -0,0 +1,43 @@ +{ + "params": { + "operationId": "op-1", + "selection": { + "kind": "capture", + "source": "/workspace", + "filter": { + "exclude": [ + "node_modules/**", + ".git/**" + ] + }, + "symlinks": "skip", + "specialFiles": "skip" + }, + "limits": { + "maxEntries": 100000, + "maxDepth": 64, + "maxFileBytes": 1073741824, + "maxTotalBytes": 1073741824, + "maxManifestBytes": 33554432, + "maxDurationMs": 86400000 + } + }, + "response": { + "operationId": "op-1", + "phase": "scanning", + "entryCount": 0, + "totalBytes": 0, + "transferredBytes": 0, + "reusedBytes": 0, + "deadlineAtMs": 1761400000000 + }, + "error": { + "code": -32008, + "message": "8 transfers are active", + "data": { + "kind": "busy", + "reason": "transferCapacity", + "retryAfterMs": 1000 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.commit.json b/crates/environment-protocol/fixtures/data/transfer.commit.json new file mode 100644 index 000000000..b78e69dd6 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.commit.json @@ -0,0 +1,22 @@ +{ + "params": { + "operationId": "op-2" + }, + "response": { + "operationId": "op-2", + "phase": "complete", + "entryCount": 3, + "totalBytes": 2048, + "transferredBytes": 1024, + "reusedBytes": 512, + "deadlineAtMs": 1761400000000, + "receiptExpiresAtMs": 1761486400000 + }, + "error": { + "code": -32015, + "message": "operation op-2 ran out of its 86400000 ms", + "data": { + "kind": "deadlineExceeded" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.getStatus.json b/crates/environment-protocol/fixtures/data/transfer.getStatus.json new file mode 100644 index 000000000..8ce8c3a14 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.getStatus.json @@ -0,0 +1,23 @@ +{ + "params": { + "operationId": "op-1" + }, + "response": { + "operationId": "op-1", + "phase": "complete", + "entryCount": 3, + "totalBytes": 2048, + "transferredBytes": 1024, + "reusedBytes": 512, + "deadlineAtMs": 1761400000000, + "receiptExpiresAtMs": 1761486400000 + }, + "error": { + "code": -32004, + "message": "transfer receipt op-1 expired", + "data": { + "kind": "notFound", + "reason": "expired" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.listMissing.json b/crates/environment-protocol/fixtures/data/transfer.listMissing.json new file mode 100644 index 000000000..2bc25ac9f --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.listMissing.json @@ -0,0 +1,20 @@ +{ + "params": { + "operationId": "op-2", + "limit": 64 + }, + "response": { + "digests": [ + "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd" + ], + "remainingCount": 3 + }, + "error": { + "code": -32007, + "message": "listMissing is not valid in the inventory phase", + "data": { + "kind": "failedPrecondition", + "reason": "transferPhase" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.read.json b/crates/environment-protocol/fixtures/data/transfer.read.json new file mode 100644 index 000000000..70d89110e --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.read.json @@ -0,0 +1,18 @@ +{ + "params": { + "operationId": "op-1", + "digest": "sha256:abababababababababababababababababababababababababababababababab", + "offset": 0, + "maxBytes": 4194304 + }, + "response": { + "data": "aGVsbG8=" + }, + "error": { + "code": -32004, + "message": "digest is not part of the inventory", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.readInventory.json b/crates/environment-protocol/fixtures/data/transfer.readInventory.json new file mode 100644 index 000000000..5a6a26baf --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.readInventory.json @@ -0,0 +1,35 @@ +{ + "params": { + "operationId": "op-1", + "index": 0, + "limit": 128 + }, + "response": { + "entries": [ + { + "path": ".", + "content": { + "kind": "directory" + } + }, + { + "path": "bin/tool", + "content": { + "kind": "file", + "sizeBytes": 2048, + "executable": true, + "digest": "sha256:abababababababababababababababababababababababababababababababab" + } + } + ], + "nextIndex": 2 + }, + "error": { + "code": -32007, + "message": "the inventory is not sealed yet", + "data": { + "kind": "failedPrecondition", + "reason": "transferPhase" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.write.json b/crates/environment-protocol/fixtures/data/transfer.write.json new file mode 100644 index 000000000..080567086 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.write.json @@ -0,0 +1,19 @@ +{ + "params": { + "operationId": "op-2", + "digest": "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + "offset": 0, + "data": "aGVsbG8gd29ybGQK" + }, + "response": { + "nextOffset": 12 + }, + "error": { + "code": -32007, + "message": "uploaded bytes do not hash to the digest", + "data": { + "kind": "failedPrecondition", + "reason": "digestMismatch" + } + } +} diff --git a/crates/environment-protocol/fixtures/data_initialize_params.json b/crates/environment-protocol/fixtures/data_initialize_params.json deleted file mode 100644 index 0e07e72db..000000000 --- a/crates/environment-protocol/fixtures/data_initialize_params.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "clientName": "lightspeed-test", - "protocolVersion": 2, - "resumeConnectionId": "conn-prev", - "scope": { - "sessionId": "sandbox-123", - "type": "session" - } -} diff --git a/crates/environment-protocol/fixtures/fs_read_file_response.json b/crates/environment-protocol/fixtures/fs_read_file_response.json deleted file mode 100644 index 5340bffb7..000000000 --- a/crates/environment-protocol/fixtures/fs_read_file_response.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "data": "aGVsbG8K" -} diff --git a/crates/environment-protocol/fixtures/job_read_response.json b/crates/environment-protocol/fixtures/job_read_response.json deleted file mode 100644 index b8656ee16..000000000 --- a/crates/environment-protocol/fixtures/job_read_response.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "jobs": [ - { - "artifacts": [ - { - "kind": "file", - "metadata": { - "role": "report" - }, - "path": "/workspace/result.txt" - } - ], - "outputChunks": [ - { - "chunk": "b2sK", - "seq": 1, - "stream": "stdout" - } - ], - "outputNextSeq": 2, - "summary": { - "createdAtMs": 1, - "dependencies": [ - "checkout" - ], - "exitCode": 0, - "finishedAtMs": 4, - "jobId": "tests", - "name": "tests", - "namespace": "session_1", - "orphanedDescendants": true, - "queuedAtMs": 2, - "queueKey": "repo", - "startedAtMs": 3, - "status": "succeeded" - } - } - ] -} diff --git a/crates/environment-protocol/fixtures/job_start_params.json b/crates/environment-protocol/fixtures/job_start_params.json deleted file mode 100644 index 64fd721b4..000000000 --- a/crates/environment-protocol/fixtures/job_start_params.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "jobs": [ - { - "argv": [ - "git", - "status" - ], - "cwd": "/workspace", - "dependencyPolicy": "allSucceeded", - "dependsOn": [ - { - "name": "setup" - } - ], - "env": { - "RUST_LOG": "debug" - }, - "name": "checkout", - "jobId": "checkout", - "queueKey": "repo", - "stdin": "aGVsbG8K", - "timeoutMs": 60000 - } - ], - "namespace": "session_1", - "requestId": "request-1" -} diff --git a/crates/environment-protocol/fixtures/process_read_response.json b/crates/environment-protocol/fixtures/process_read_response.json deleted file mode 100644 index 2141c3ff8..000000000 --- a/crates/environment-protocol/fixtures/process_read_response.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "chunks": [ - { - "chunk": "b2sK", - "seq": 1, - "stream": "stdout" - }, - { - "chunk": "d2Fybgo=", - "seq": 2, - "stream": "stderr" - } - ], - "closed": true, - "exitCode": 0, - "exited": true, - "leftoverProcesses": [ - { - "command": "python -m http.server 8080", - "pid": 91 - } - ], - "nextSeq": 3, - "pid": 90 -} diff --git a/crates/environment-protocol/fixtures/process_start_params.json b/crates/environment-protocol/fixtures/process_start_params.json deleted file mode 100644 index 9c6e5a190..000000000 --- a/crates/environment-protocol/fixtures/process_start_params.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "argv": [ - "sh", - "-lc", - "cat" - ], - "cwd": "/workspace", - "env": { - "RUST_LOG": "debug" - }, - "processId": "proc-1", - "stdin": "aW5wdXQK", - "timeoutMs": 60000, - "tty": false -} diff --git a/crates/environment-protocol/fixtures/registration/registration.challenge.json b/crates/environment-protocol/fixtures/registration/registration.challenge.json new file mode 100644 index 000000000..055bcd29d --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.challenge.json @@ -0,0 +1,6 @@ +{ + "params": { + "protocolVersion": 3, + "nonce": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f" + } +} diff --git a/crates/environment-protocol/fixtures/registration/registration.closed.json b/crates/environment-protocol/fixtures/registration/registration.closed.json new file mode 100644 index 000000000..975d0bdf1 --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.closed.json @@ -0,0 +1,6 @@ +{ + "params": { + "reason": "superseded", + "message": "another process with this daemon identity connected" + } +} diff --git a/crates/environment-protocol/fixtures/registration/registration.goodbye.json b/crates/environment-protocol/fixtures/registration/registration.goodbye.json new file mode 100644 index 000000000..3675fe277 --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.goodbye.json @@ -0,0 +1,5 @@ +{ + "params": { + "reason": "shutdown" + } +} diff --git a/crates/environment-protocol/fixtures/registration/registration.openStream.json b/crates/environment-protocol/fixtures/registration/registration.openStream.json new file mode 100644 index 000000000..31ac34c21 --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.openStream.json @@ -0,0 +1,16 @@ +{ + "params": { + "token": "tok_5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + "stream": { + "kind": "dataProtocol" + } + }, + "response": {}, + "error": { + "code": -32019, + "message": "dialing wss://gw.example.com/environment-gateway/data failed: connection refused", + "data": { + "kind": "unavailable" + } + } +} diff --git a/crates/environment-protocol/fixtures/registration/registration.register.json b/crates/environment-protocol/fixtures/registration/registration.register.json new file mode 100644 index 000000000..aeaaa0f3b --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.register.json @@ -0,0 +1,37 @@ +{ + "params": { + "protocolVersion": 3, + "registrationKey": "lsrk_0123456789abcdef", + "daemonPublicKey": "3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b", + "signature": "9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c", + "displayName": "ci-runner-7", + "metadata": { + "harbor.trialId": "trial-1" + }, + "implementation": { + "name": "lightspeed-envd", + "version": "0.5.0", + "gitSha": "86f6adf3", + "target": "x86_64-unknown-linux-musl" + } + }, + "response": { + "receipt": { + "environmentId": "env_1", + "incarnationId": "inc_1", + "daemonId": "daemon_3b3b", + "controlConnectionId": "connection_7", + "identityMode": "ephemeral", + "admission": "created" + }, + "heartbeatIntervalMs": 30000 + }, + "error": { + "code": -32003, + "message": "the registration signature does not verify", + "data": { + "kind": "permissionDenied", + "reason": "invalidSignature" + } + } +} diff --git a/crates/environment-protocol/src/bin/export-contract.rs b/crates/environment-protocol/src/bin/export-contract.rs new file mode 100644 index 000000000..b3f077732 --- /dev/null +++ b/crates/environment-protocol/src/bin/export-contract.rs @@ -0,0 +1,25 @@ +//! Writes the committed contract under `crates/environment-protocol/contract/`. +//! +//! Usage: `cargo run -p environment-protocol --features schema --bin export-contract [output-dir]` + +use std::{env, fs, path::PathBuf}; + +fn main() { + let out_dir = env::args() + .nth(1) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("contract")); + fs::create_dir_all(&out_dir).expect("create output directory"); + for plane in environment_protocol::contract::export() { + for (name, value) in [ + (plane.schema_file_name(), &plane.schema_bundle), + (plane.openrpc_file_name(), &plane.openrpc), + ] { + let mut text = serde_json::to_string_pretty(value).expect("serialize contract"); + text.push('\n'); + let path = out_dir.join(name); + fs::write(&path, text).expect("write contract"); + println!("wrote {}", path.display()); + } + } +} diff --git a/crates/environment-protocol/src/bootstrap.rs b/crates/environment-protocol/src/bootstrap.rs new file mode 100644 index 000000000..fe995ffda --- /dev/null +++ b/crates/environment-protocol/src/bootstrap.rs @@ -0,0 +1,70 @@ +//! The frozen bootstrap surface besides the `registration/challenge` frame: +//! the envd discovery document the gateway serves at +//! [`DISCOVERY_PATH`](crate::routes::DISCOVERY_PATH), and the build facts +//! `lightspeed-envd --print-build` prints. +//! +//! Daemons of every version read these to upgrade themselves, so they only +//! ever gain fields and are decoded tolerantly: unknown members are +//! ignored. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::bytes::hex_value; + +hex_value!( + /// `sha256` of a published archive: 64 lowercase hex digits. + ArtifactSha256, + 32, + "^[0-9a-f]{64}$" +); + +/// The discovery document. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct DiscoveryDocument { + /// Release version of the published envd. + pub version: String, + pub git_sha: String, + /// Release channel, such as `stable` or `snapshot`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub channel: Option, + /// The published envd's `ENVD_PROTOCOL_VERSION`. + pub protocol_version: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub built_at_ms: Option, + /// Archives by Rust target triple. + pub artifacts: BTreeMap, +} + +/// One published envd archive. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct DiscoveryArtifact { + /// Archive file name. + pub file: String, + /// Digest of the archive's bytes. + pub sha256: ArtifactSha256, + /// Absolute download URL; absent (or null) when the archive is served + /// next to the document. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, +} + +/// What `lightspeed-envd --print-build` prints, as one JSON object. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct BuildFacts { + /// Always `lightspeed-envd`. + pub name: String, + pub version: String, + pub git_sha: String, + /// Rust target triple. + pub target: String, + /// The binary's `ENVD_PROTOCOL_VERSION`. + pub protocol_version: u32, +} diff --git a/crates/environment-protocol/src/bytes.rs b/crates/environment-protocol/src/bytes.rs new file mode 100644 index 000000000..35b8b84cf --- /dev/null +++ b/crates/environment-protocol/src/bytes.rs @@ -0,0 +1,303 @@ +//! Byte encodings. +//! +//! Payload bytes are padded standard base64 ([`ByteChunk`]). Fixed-size +//! cryptographic values are canonical lowercase hex, decoded byte by byte +//! so malformed input fails cleanly and never panics. + +use std::fmt; + +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +/// JSON Schema pattern of padded standard base64. +pub const BASE64_PATTERN: &str = "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"; + +/// Payload bytes, always named `data` on the wire, as padded standard +/// base64. +#[derive(Clone, Default, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct ByteChunk( + #[cfg_attr( + feature = "schema", + schemars( + with = "String", + pattern(BASE64_PATTERN), + extend("contentEncoding" = "base64") + ) + )] + Vec, +); + +impl ByteChunk { + pub fn new(bytes: impl Into>) -> Self { + Self(bytes.into()) + } + + pub fn into_inner(self) -> Vec { + self.0 + } + + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for ByteChunk { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "ByteChunk({} bytes)", self.0.len()) + } +} + +impl From> for ByteChunk { + fn from(value: Vec) -> Self { + Self(value) + } +} + +impl From<&[u8]> for ByteChunk { + fn from(value: &[u8]) -> Self { + Self(value.to_vec()) + } +} + +impl Serialize for ByteChunk { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&BASE64_STANDARD.encode(&self.0)) + } +} + +impl<'de> Deserialize<'de> for ByteChunk { + fn deserialize>(deserializer: D) -> Result { + let encoded = String::deserialize(deserializer)?; + BASE64_STANDARD + .decode(encoded) + .map(Self) + .map_err(D::Error::custom) + } +} + +/// A secret value carried on the wire, such as a `secretEnv` value, a +/// registration key or a stream token. It serializes as a plain string; +/// `Debug` is redacted so structured logging never prints it. +#[derive(Clone, PartialEq, Eq)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "schema", schemars(with = "String"))] +pub struct SecretString(String); + +impl SecretString { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn expose(&self) -> &str { + &self.0 + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for SecretString { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SecretString()") + } +} + +impl From for SecretString { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for SecretString { + fn from(value: &str) -> Self { + Self(value.to_owned()) + } +} + +impl Serialize for SecretString { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for SecretString { + fn deserialize>(deserializer: D) -> Result { + String::deserialize(deserializer).map(Self) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum HexError { + #[error("hex value has {actual} digits; expected {expected}")] + Length { expected: usize, actual: usize }, + + #[error("hex value has an odd number of digits")] + OddLength, + + #[error("byte {index} is not a lowercase hex digit")] + Digit { index: usize }, +} + +/// Lowercase hex of `bytes`. +pub fn encode_hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut text = String::with_capacity(bytes.len() * 2); + for byte in bytes { + text.push(DIGITS[usize::from(byte >> 4)] as char); + text.push(DIGITS[usize::from(byte & 0x0f)] as char); + } + text +} + +/// Decode canonical lowercase hex of any even length. +pub fn decode_hex(value: &str) -> Result, HexError> { + let digits = value.as_bytes(); + if !digits.len().is_multiple_of(2) { + return Err(HexError::OddLength); + } + digits + .as_chunks::<2>() + .0 + .iter() + .enumerate() + .map(|(pair, [high, low])| { + let high = hex_digit(*high).ok_or(HexError::Digit { index: pair * 2 })?; + let low = hex_digit(*low).ok_or(HexError::Digit { + index: pair * 2 + 1, + })?; + Ok((high << 4) | low) + }) + .collect() +} + +/// Decode canonical lowercase hex of exactly `N` bytes. +pub fn decode_hex_array(value: &str) -> Result<[u8; N], HexError> { + if value.len() != N * 2 { + return Err(HexError::Length { + expected: N * 2, + actual: value.len(), + }); + } + let bytes = decode_hex(value)?; + let mut array = [0u8; N]; + array.copy_from_slice(&bytes); + Ok(array) +} + +fn hex_digit(digit: u8) -> Option { + match digit { + b'0'..=b'9' => Some(digit - b'0'), + b'a'..=b'f' => Some(digit - b'a' + 10), + _ => None, + } +} + +/// A fixed-size value that travels as canonical lowercase hex. +macro_rules! hex_value { + ($(#[$doc:meta])* $name:ident, $bytes:literal, $pattern:literal) => { + $(#[$doc])* + #[derive(Clone, Copy, PartialEq, Eq, Hash)] + #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] + pub struct $name( + #[cfg_attr( + feature = "schema", + schemars( + with = "String", + pattern($pattern) + ) + )] + [u8; $bytes], + ); + + impl $name { + pub const BYTES: usize = $bytes; + + pub fn from_bytes(bytes: [u8; $bytes]) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8; $bytes] { + &self.0 + } + + pub fn parse(value: &str) -> Result { + $crate::bytes::decode_hex_array::<$bytes>(value).map(Self) + } + + pub fn to_hex(&self) -> String { + $crate::bytes::encode_hex(&self.0) + } + } + + impl std::fmt::Debug for $name { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "{}({})", stringify!($name), self.to_hex()) + } + } + + impl std::fmt::Display for $name { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.to_hex()) + } + } + + impl std::str::FromStr for $name { + type Err = $crate::bytes::HexError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } + } + + impl serde::Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_hex()) + } + } + + impl<'de> serde::Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + Self::parse(&value).map_err(::custom) + } + } + }; +} + +pub(crate) use hex_value; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn byte_chunks_are_padded_standard_base64() { + let value = serde_json::to_value(ByteChunk::from(b"hello".as_slice())).unwrap(); + assert_eq!(value, serde_json::json!("aGVsbG8=")); + assert!(serde_json::from_value::(serde_json::json!("aGVsbG8")).is_err()); + } + + #[test] + fn hex_decoding_is_canonical_and_never_panics() { + assert_eq!(decode_hex("0001abff").unwrap(), [0, 1, 0xab, 0xff]); + assert_eq!(encode_hex(&[0, 1, 0xab, 0xff]), "0001abff"); + for bad in ["abc", "zz", "AB", "+1", "a\u{e9}b", "aaaaaaaaaaaaaaa\u{e9}"] { + assert!(decode_hex(bad).is_err(), "{bad:?}"); + } + assert!(matches!( + decode_hex_array::<2>("00"), + Err(HexError::Length { .. }) + )); + } +} diff --git a/crates/environment-protocol/src/contract.rs b/crates/environment-protocol/src/contract.rs new file mode 100644 index 000000000..191b1ea6a --- /dev/null +++ b/crates/environment-protocol/src/contract.rs @@ -0,0 +1,345 @@ +//! Machine-readable contract export, one set of documents per plane: +//! +//! - `.schema.json`: a draft-07 JSON Schema bundle of every wire type +//! the plane uses, under `definitions`; +//! - `.openrpc.json`: an OpenRPC document of the plane's methods with +//! by-name params, the manifest facts as `x-lightspeed-*` members, and the +//! error kinds. +//! +//! The committed copies under `crates/environment-protocol/contract/` are +//! checked by the `contract` integration test; regenerate them with +//! `cargo run -p environment-protocol --features schema --bin export-contract`. + +use std::collections::BTreeMap; + +use schemars::generate::SchemaSettings; +use serde_json::{Value, json}; + +use crate::{ + CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION, + bootstrap::{BuildFacts, DiscoveryDocument}, + error::{ProtocolError, ProtocolErrorKind}, + handshake::Plane, + limits::MAX_MESSAGE_BYTES, + manifest::{MessageKind, plane_methods}, +}; + +/// The exported documents of one plane. +pub struct PlaneContract { + pub plane: Plane, + pub schema_bundle: Value, + pub openrpc: Value, +} + +impl PlaneContract { + pub fn schema_file_name(&self) -> String { + format!("{}.schema.json", self.plane) + } + + pub fn openrpc_file_name(&self) -> String { + format!("{}.openrpc.json", self.plane) + } +} + +pub fn export() -> Vec { + [Plane::Data, Plane::Controller, Plane::Registration] + .into_iter() + .map(export_plane) + .collect() +} + +fn export_plane(plane: Plane) -> PlaneContract { + let mut generator = SchemaSettings::draft07().into_generator(); + let error_schema = to_value(generator.subschema_for::()); + let bootstrap = (plane == Plane::Registration).then(|| { + json!({ + "discoveryDocument": to_value(generator.subschema_for::()), + "buildFacts": to_value(generator.subschema_for::()), + }) + }); + let mut entries = Vec::new(); + for spec in plane_methods(plane) { + let schemas = (spec.schemas)(&mut generator); + entries.push(( + spec, + to_value(schemas.params), + schemas.response.map(to_value), + )); + } + let definitions: BTreeMap = generator + .take_definitions(true) + .into_iter() + .map(|(name, mut schema)| { + strip_null(&mut schema); + clean_descriptions(&mut schema); + (name, schema) + }) + .collect(); + + let methods: Vec = entries + .into_iter() + .map(|(spec, params, response)| { + let mut method = json!({ + "name": spec.name, + "summary": spec.summary, + "paramStructure": "by-name", + "params": by_name_params(¶ms, &definitions), + "x-lightspeed-kind": spec.kind.as_str(), + "x-lightspeed-direction": spec.direction.as_str(), + "x-lightspeed-family": spec.family.as_str(), + "x-lightspeed-params-type": spec.params_type, + }); + if let Some(activity) = spec.activity { + method["x-lightspeed-activity"] = json!(activity.as_str()); + } + if let Some(capability) = spec.capability { + method["x-lightspeed-capability"] = json!(capability); + } + if spec.abortable { + method["x-lightspeed-abortable"] = json!(true); + } + if let (MessageKind::Request, Some(response)) = (spec.kind, response) { + method["result"] = json!({"name": "result", "schema": response}); + method["x-lightspeed-result-type"] = json!(spec.response_type); + } + method + }) + .collect(); + + let (title, version) = match plane { + Plane::Data => ("Lightspeed environment data plane", ENVD_PROTOCOL_VERSION), + Plane::Controller => ( + "Lightspeed environment controller plane", + CONTROLLER_PROTOCOL_VERSION, + ), + Plane::Registration => ( + "Lightspeed environment registration channel", + ENVD_PROTOCOL_VERSION, + ), + }; + let error_kinds: Vec = ProtocolErrorKind::ALL + .into_iter() + .map(|kind| { + json!({ + "kind": kind.as_str(), + "code": kind.code(), + "retryable": kind.is_retryable(), + }) + }) + .collect(); + + let mut schema_bundle = json!({ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": title, + "description": format!("Every wire type of the {plane} plane, protocol version {version}."), + "definitions": definitions, + }); + if let Some(bootstrap) = &bootstrap { + schema_bundle["x-lightspeed-bootstrap"] = bootstrap.clone(); + } + + let mut openrpc = json!({ + "openrpc": "1.3.2", + "info": { + "title": title, + "version": version.to_string(), + "description": format!( + "JSON-RPC 2.0 over WebSocket text frames of at most {MAX_MESSAGE_BYTES} bytes. \ + Exact-match protocolVersion {version}. Errors carry code, message and \ + data.kind; see x-lightspeed-error-kinds." + ), + }, + "methods": methods, + "components": { + "schemas": schema_bundle["definitions"].clone(), + }, + "x-lightspeed-plane": plane.as_str(), + "x-lightspeed-error": error_schema, + "x-lightspeed-error-kinds": error_kinds, + }); + if let Some(bootstrap) = bootstrap { + openrpc["x-lightspeed-bootstrap"] = bootstrap; + } + rewrite_refs(&mut openrpc, "#/definitions/", "#/components/schemas/"); + + PlaneContract { + plane, + schema_bundle, + openrpc, + } +} + +/// One OpenRPC content descriptor per property of the params object. +fn by_name_params(schema: &Value, definitions: &BTreeMap) -> Value { + let resolved = schema + .get("$ref") + .and_then(Value::as_str) + .and_then(|reference| reference.strip_prefix("#/definitions/")) + .and_then(|name| definitions.get(name)) + .unwrap_or(schema); + let required: Vec<&str> = resolved + .get("required") + .and_then(Value::as_array) + .map(|names| names.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + let Some(properties) = resolved.get("properties").and_then(Value::as_object) else { + return json!([]); + }; + Value::Array( + properties + .iter() + .map(|(name, schema)| { + json!({ + "name": name, + "required": required.contains(&name.as_str()), + "schema": schema, + }) + }) + .collect(), + ) +} + +fn to_value(schema: schemars::Schema) -> Value { + let mut value = serde_json::to_value(schema).expect("schema serializes"); + strip_null(&mut value); + clean_descriptions(&mut value); + value +} + +/// Descriptions come from rustdoc comments; implementers read them without +/// rustdoc, so intra-doc links become their plain text. +fn clean_descriptions(value: &mut Value) { + match value { + Value::Object(map) => { + for (key, entry) in map.iter_mut() { + match entry { + Value::String(text) if key == "description" => *text = strip_doc_links(text), + _ => clean_descriptions(entry), + } + } + } + Value::Array(entries) => { + for entry in entries { + clean_descriptions(entry); + } + } + _ => {} + } +} + +/// `[`X`](path)` and `[`X`]` become `` `X` ``, `[text](path)` becomes +/// `text`; brackets inside code spans are left alone. +fn strip_doc_links(text: &str) -> String { + let mut output = String::with_capacity(text.len()); + let mut rest = text; + while let Some(index) = rest.find(['`', '[']) { + output.push_str(&rest[..index]); + rest = &rest[index..]; + if let Some(after_tick) = rest.strip_prefix('`') { + let end = after_tick.find('`').map_or(rest.len(), |end| end + 2); + output.push_str(&rest[..end]); + rest = &rest[end..]; + continue; + } + let Some(close) = rest.find(']') else { + output.push_str(rest); + return output; + }; + let inner = &rest[1..close]; + let after = &rest[close + 1..]; + if let Some(target) = after.strip_prefix('(') + && let Some(target_end) = target.find(')') + { + output.push_str(inner); + rest = &target[target_end + 1..]; + } else if inner.len() >= 2 && inner.starts_with('`') && inner.ends_with('`') { + output.push_str(inner); + rest = after; + } else { + output.push('['); + rest = &rest[1..]; + } + } + output.push_str(rest); + output +} + +/// Absent options are skipped on the wire, never sent as `null`, so the +/// `null` alternatives the schema generator adds for `Option` fields are +/// removed. +fn strip_null(value: &mut Value) { + match value { + Value::Object(map) => { + if let Some(Value::Array(types)) = map.get_mut("type") { + types.retain(|kind| kind != "null"); + if types.len() == 1 { + let only = types.remove(0); + map.insert("type".to_owned(), only); + } + } + for key in ["anyOf", "oneOf"] { + if let Some(Value::Array(variants)) = map.get_mut(key) { + variants.retain(|variant| variant.get("type") != Some(&json!("null"))); + } + } + for entry in map.values_mut() { + strip_null(entry); + } + } + Value::Array(entries) => { + for entry in entries { + strip_null(entry); + } + } + _ => {} + } +} + +fn rewrite_refs(value: &mut Value, from: &str, to: &str) { + match value { + Value::Object(map) => { + for (key, entry) in map.iter_mut() { + if key == "$ref" + && let Value::String(reference) = entry + && let Some(rest) = reference.strip_prefix(from) + { + *reference = format!("{to}{rest}"); + } else { + rewrite_refs(entry, from, to); + } + } + } + Value::Array(entries) => { + for entry in entries { + rewrite_refs(entry, from, to); + } + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn doc_links_become_plain_text() { + assert_eq!( + strip_doc_links("see [`MAX_PATH_BYTES`] and [`X`](crate::x::X) or [the docs](a::b)"), + "see `MAX_PATH_BYTES` and `X` or the docs" + ); + assert_eq!( + strip_doc_links("`[...]` is a class; [1, 2] stays"), + "`[...]` is a class; [1, 2] stays" + ); + } + + #[test] + fn exported_descriptions_carry_no_doc_links() { + for plane in export() { + let text = plane.schema_bundle.to_string(); + assert!(!text.contains("](crate::"), "{}", plane.plane); + assert!(!text.contains("[`"), "{}", plane.plane); + } + } +} diff --git a/crates/environment-protocol/src/control/handshake.rs b/crates/environment-protocol/src/control/handshake.rs index 4895f7073..727a97c86 100644 --- a/crates/environment-protocol/src/control/handshake.rs +++ b/crates/environment-protocol/src/control/handshake.rs @@ -1,46 +1,68 @@ -//! Controller-plane connection handshake. +//! Controller-plane `initialize`. +//! +//! The same handshake rules as the data plane apply: `initialize` first +//! (`notInitialized` otherwise), once (`invalidRequest` on a repeat), with +//! [`CONTROLLER_PROTOCOL_VERSION`](crate::CONTROLLER_PROTOCOL_VERSION) +//! (`versionMismatch` otherwise). The core methods (`template/list` and +//! every `target/*` method except `target/adopt`) are mandatory at this +//! version; the capabilities name the optional ones. use serde::{Deserialize, Serialize}; -use crate::shared::{CURRENT_PROTOCOL_VERSION, ImplementationInfo}; +use crate::{ + handshake::{ImplementationInfo, Plane}, + routes::DataRouteBase, + serde_util::is_false, +}; +/// `initialize` params on the controller plane. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ControllerInitializeParams { - #[serde(default = "default_protocol_version")] + /// The client's `CONTROLLER_PROTOCOL_VERSION`. pub protocol_version: u32, - pub client_name: String, + pub client_info: ImplementationInfo, } +/// `initialize` response on the controller plane. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ControllerInitializeResponse { pub protocol_version: u32, + /// Always `controller`. + pub plane: Plane, + pub server_info: ImplementationInfo, pub capabilities: ControllerCapabilities, - pub implementation: ImplementationInfo, + pub routes: ControllerRoutes, } +/// Optional controller methods. Calling an absent one is +/// `capabilityUnavailable`. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ControllerCapabilities { - #[serde(default)] - pub list_templates: bool, - #[serde(default)] - pub list_targets: bool, - #[serde(default)] - pub create_target: bool, - #[serde(default)] + /// `target/adopt` is served. + #[serde(default, skip_serializing_if = "is_false")] pub adopt_target: bool, - #[serde(default)] - pub get_target: bool, - #[serde(default)] - pub close_target: bool, - #[serde(default)] - pub set_target_power: bool, - #[serde(default)] + /// `ingress/ensure` and `ingress/remove` are served. + #[serde(default, skip_serializing_if = "is_false")] pub ingress: bool, } -fn default_protocol_version() -> u32 { - CURRENT_PROTOCOL_VERSION +/// Where Lightspeed reaches a target's data plane through this provider. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ControllerRoutes { + /// Base URL of the data routes. A target's data route is this base + /// followed by + /// `/{universeId}/{bindingId}/{environmentId}/{incarnationId}/{targetId}`, + /// each segment percent-encoded + /// ([`provider_data_route`](crate::routes::provider_data_route)), and is + /// dialed with the same bearer credential as the controller. A + /// controller reached over `wss` must advertise a `wss` base. + pub data: DataRouteBase, } diff --git a/crates/environment-protocol/src/control/ingress.rs b/crates/environment-protocol/src/control/ingress.rs index 230f2f93c..519681232 100644 --- a/crates/environment-protocol/src/control/ingress.rs +++ b/crates/environment-protocol/src/control/ingress.rs @@ -1,40 +1,29 @@ -//! Provider-owned public ingress lifecycle payloads. +//! `ingress/*`: public ingress of one target. Requires +//! `capabilities.ingress`. `target/close` removes ingress by contract, so +//! `ingress/remove` exists only to disable ingress on a target that stays. +//! Both answer [`TargetResponse`](crate::control::target::TargetResponse) +//! with the realized ingress in the observation; the runtime converges its +//! recorded intent against that observation. use serde::{Deserialize, Serialize}; -use crate::{control::targets::ProviderBindingContext, shared::ProviderTargetId}; +use crate::control::target::TargetRef; +/// `ingress/ensure` params: route the template's public ingress to the +/// target. Idempotent. A target whose template has no `publicIngress` is +/// `failedPrecondition` (`ingressUnsupported`). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct EnsureIngressParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, + pub target: TargetRef, } +/// `ingress/remove` params: stop routing public ingress to the target. +/// Idempotent. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RemoveIngressParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ProviderIngressStatus { - Ready, - Disabled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct IngressResponse { - pub status: ProviderIngressStatus, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub public_endpoint: Option, + pub target: TargetRef, } diff --git a/crates/environment-protocol/src/control/methods.rs b/crates/environment-protocol/src/control/methods.rs deleted file mode 100644 index b36371e7e..000000000 --- a/crates/environment-protocol/src/control/methods.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! JSON-RPC method names for the provider controller plane. - -pub const INITIALIZE_METHOD: &str = "controller/initialize"; - -pub const LIST_TEMPLATES_METHOD: &str = "controller/listTemplates"; -pub const LIST_TARGETS_METHOD: &str = "controller/listTargets"; -pub const CREATE_TARGET_METHOD: &str = "controller/createTarget"; -pub const ADOPT_TARGET_METHOD: &str = "controller/adoptTarget"; -pub const GET_TARGET_METHOD: &str = "controller/getTarget"; -pub const CLOSE_TARGET_METHOD: &str = "controller/closeTarget"; -pub const SET_TARGET_POWER_METHOD: &str = "controller/setTargetPower"; -pub const ENSURE_INGRESS_METHOD: &str = "controller/ensureIngress"; -pub const REMOVE_INGRESS_METHOD: &str = "controller/removeIngress"; diff --git a/crates/environment-protocol/src/control/mod.rs b/crates/environment-protocol/src/control/mod.rs index 1b621996d..d13749385 100644 --- a/crates/environment-protocol/src/control/mod.rs +++ b/crates/environment-protocol/src/control/mod.rs @@ -1,6 +1,22 @@ -//! Controller-plane protocol for provider target lifecycle and connection discovery. +//! The controller plane: a provider controller creating, observing, +//! powering and closing targets for Lightspeed environments. The runtime +//! is the client; the provider is the server. +//! +//! The runtime connects with the provider credential as an +//! `Authorization: Bearer` header, on the controller URL and on every data +//! route the controller advertises; the provider checks it in constant time +//! before the WebSocket upgrade. +//! +//! **Bounded calls.** Every target method answers `{target: observation}` +//! promptly. Slow work (an image pull, a VM boot, a graceful shutdown) +//! continues in the background and is followed with `target/get`. +//! +//! **Ownership.** A target belongs to exactly one [`TargetOwner`](target::TargetOwner). Every +//! target method carries the owner; a provider resolves the target from +//! the owner when no `targetId` is given, and refuses a target that belongs +//! to another owner with `failedPrecondition` (`ownerMismatch`). pub mod handshake; pub mod ingress; -pub mod methods; -pub mod targets; +pub mod target; +pub mod template; diff --git a/crates/environment-protocol/src/control/target.rs b/crates/environment-protocol/src/control/target.rs new file mode 100644 index 000000000..f3b7e49fe --- /dev/null +++ b/crates/environment-protocol/src/control/target.rs @@ -0,0 +1,230 @@ +//! `target/*`: create, adopt, observe, power and close provider targets. + +use serde::{Deserialize, Serialize}; + +use crate::{ + ids::{ + BindingId, EnvironmentId, IncarnationId, ProviderNativeRef, ProviderTargetId, TemplateId, + UniverseId, + }, + metadata::{DescriptiveText, DisplayName, Metadata}, + routes::PublicUrl, + serde_util::is_false, +}; + +/// The Lightspeed incarnation a target belongs to. A provider owns at most +/// one target per owner. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetOwner { + pub universe_id: UniverseId, + pub binding_id: BindingId, + pub environment_id: EnvironmentId, + pub incarnation_id: IncarnationId, +} + +/// The provider binding a call is scoped to, for calls that concern no +/// particular target. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BindingRef { + pub universe_id: UniverseId, + pub binding_id: BindingId, +} + +/// A reference to an existing target: its owner and, once known, its id. +/// Without `targetId` the provider resolves the owner's target itself. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetRef { + pub owner: TargetOwner, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_id: Option, +} + +/// What the provider observes about a target. Every target method answers +/// with one. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetObservation { + /// The provider's id for the target. Absent in two cases only: while + /// `phase` is `provisioning` and the provider has not assigned an id + /// yet (an asynchronous create answers before it has one; the runtime + /// records the id from the first observation that carries it), and in a + /// `closed` observation of an owner the provider holds no target for. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_id: Option, + /// Echoed from the request. + pub owner: TargetOwner, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display_name: Option, + pub state: TargetState, + /// Power states `target/setPower` can move this target to. Empty when + /// the provider offers no power control beyond create and close. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub supported_power_states: Vec, + /// Public ingress as realized; absent when disabled. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ingress: Option, + /// Provider-specific display facts. + #[serde(default, skip_serializing_if = "Metadata::is_empty")] + pub metadata: Metadata, +} + +/// Realized public ingress of a target. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct IngressObservation { + /// Public URL that reaches the target's application. + pub endpoint: PublicUrl, +} + +/// A target's state on two axes: lifecycle `phase`, and `power` while +/// active. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetState { + pub phase: TargetPhase, + /// Present exactly when `phase` is `active`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub power: Option, + /// envd answers on the target's data route (running targets only). + #[serde(default, skip_serializing_if = "is_false")] + pub daemon_ready: bool, + /// Why the target is `failed` or `unknown`, or what a transition waits + /// for. Providers build it with + /// [`DescriptiveText::lossy`] from whatever their backend reports. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +/// Lifecycle phase of a target. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum TargetPhase { + /// Being created; not yet usable. + Provisioning, + /// Exists; see `power`. + Active, + /// Being destroyed. + Closing, + /// Gone, and nothing of it remains. + Closed, + /// Broken beyond the provider's own recovery; see `message`. + Failed, + /// The provider cannot tell; see `message`. + Unknown, +} + +/// Steady power state of an active target, in provider-neutral terms. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum PowerState { + Running, + /// Execution halted, memory retained; resuming is fast. + Paused, + /// Execution state persisted, compute released; resuming restores it. + Suspended, + /// Powered off; disks retained. + Stopped, +} + +impl PowerState { + pub const ALL: [PowerState; 4] = [ + PowerState::Running, + PowerState::Paused, + PowerState::Suspended, + PowerState::Stopped, + ]; + + pub fn as_str(self) -> &'static str { + match self { + Self::Running => "running", + Self::Paused => "paused", + Self::Suspended => "suspended", + Self::Stopped => "stopped", + } + } +} + +impl std::fmt::Display for PowerState { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// `target/create` params. Idempotent per owner: when the owner already +/// has a target, the provider returns its observation and applies no +/// creation policy (template deprecation, binding or image checks). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CreateTargetParams { + pub owner: TargetOwner, + pub template_id: TemplateId, +} + +/// `target/adopt` params: take ownership of an existing provider target. +/// Idempotent per owner. Requires `capabilities.adoptTarget`. A source that +/// belongs to another owner, or an owner that already holds a different +/// target, is `failedPrecondition` (`ownerMismatch`); a source the provider +/// cannot take over is `failedPrecondition` (`notAdoptable`). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AdoptTargetParams { + pub owner: TargetOwner, + /// Provider-native reference to the existing target; the provider + /// defines its format. + pub source_target: ProviderNativeRef, +} + +/// `target/get` params. A pure read: it never creates or changes anything. +/// A target that no longer exists is `notFound`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GetTargetParams { + pub target: TargetRef, +} + +/// `target/setPower` params: converge the target toward `power`. A target +/// already in, or moving toward, that state is observed without further +/// action. Resuming may replace the underlying host process. A target whose +/// `phase` is not `active` is `failedPrecondition` (`targetNotActive`); a +/// state outside `supportedPowerStates` is `invalidParams`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SetTargetPowerParams { + pub target: TargetRef, + /// One of the target's `supportedPowerStates`. + pub power: PowerState, +} + +/// `target/close` params: destroy the owner's target and every resource it +/// holds, public ingress included. Graceful within the provider's own bound, +/// then forced. Answers `closing` until the target is gone, then `closed`; +/// `closed` also when the provider holds no target for the owner. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CloseTargetParams { + pub target: TargetRef, +} + +/// Response of every target method and both ingress methods. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetResponse { + pub target: TargetObservation, +} diff --git a/crates/environment-protocol/src/control/targets.rs b/crates/environment-protocol/src/control/targets.rs deleted file mode 100644 index 742c20434..000000000 --- a/crates/environment-protocol/src/control/targets.rs +++ /dev/null @@ -1,250 +0,0 @@ -//! Controller-plane target lifecycle payloads. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; - -use crate::shared::{EnvironmentCapabilities, EnvironmentPath, EnvironmentScope, ProviderTargetId}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProviderBindingContext { - pub universe_id: String, - pub binding_id: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentTemplate { - pub template_id: String, - pub display_name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(default)] - pub public_ingress: bool, - #[serde(default)] - pub deprecated: bool, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListTemplatesParams { - pub binding: ProviderBindingContext, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListTemplatesResponse { - pub templates: Vec, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListTargetsParams { - pub binding: ProviderBindingContext, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListTargetsResponse { - pub targets: Vec, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreateTargetParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub template_id: String, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreateTargetResponse { - pub target: ProviderTargetSummary, -} - -/// Explicit ownership transfer of an existing provider target into a -/// Lightspeed-managed binding namespace. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AdoptTargetParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - /// Provider-native reference to the existing target. Incus uses - /// `/` and defaults the project to `default`. - pub source_target: String, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AdoptTargetResponse { - pub target: ProviderTargetSummary, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetTargetParams { - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetTargetResponse { - pub target: ProviderTargetSummary, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CloseTargetParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, - #[serde(default)] - pub force: bool, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CloseTargetResponse { - pub target_id: ProviderTargetId, - pub status: ProviderTargetStatus, -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SandboxTargetSpec { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub template: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub image: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cwd: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub env: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub labels: BTreeMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_options: Option, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProviderTargetSummary { - pub target_id: ProviderTargetId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub display_name: Option, - pub status: ProviderTargetStatus, - pub scope: EnvironmentScope, - pub capabilities: EnvironmentCapabilities, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_cwd: Option, - /// Power states this target can be moved to with `controller/setTargetPower`. - /// Empty when the provider offers no power control beyond create/close. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub power_states: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ProviderTargetStatus { - Creating, - Starting, - Ready, - /// VMM alive with execution frozen; RAM stays resident, resume is instant. - Paused, - /// Execution state saved to disk and the VMM released; resume restores it. - Suspended, - Stopped, - Closing, - Closed, - Failed, - Unknown, -} - -impl ProviderTargetStatus { - /// The power state a steady-state observation corresponds to, or `None` - /// while the target is transitioning or gone. - pub fn power_state(self) -> Option { - match self { - Self::Ready => Some(PowerState::Running), - Self::Paused => Some(PowerState::Paused), - Self::Suspended => Some(PowerState::Suspended), - Self::Stopped => Some(PowerState::Stopped), - Self::Creating - | Self::Starting - | Self::Closing - | Self::Closed - | Self::Failed - | Self::Unknown => None, - } - } -} - -/// Requested steady power state of a provider target. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum PowerState { - Running, - Paused, - Suspended, - Stopped, -} - -impl PowerState { - pub const ALL: [PowerState; 4] = [ - PowerState::Running, - PowerState::Paused, - PowerState::Suspended, - PowerState::Stopped, - ]; - - pub fn as_str(self) -> &'static str { - match self { - Self::Running => "running", - Self::Paused => "paused", - Self::Suspended => "suspended", - Self::Stopped => "stopped", - } - } -} - -impl std::fmt::Display for PowerState { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.as_str()) - } -} - -/// Converge one target to a steady power state. Idempotent by inventory: a -/// target already in `power` (or transitioning toward it) is reported as -/// observed without further action; resuming may replace the underlying -/// host process. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SetTargetPowerParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, - pub power: PowerState, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SetTargetPowerResponse { - pub target: ProviderTargetSummary, -} diff --git a/crates/environment-protocol/src/control/template.rs b/crates/environment-protocol/src/control/template.rs new file mode 100644 index 000000000..03a4504a9 --- /dev/null +++ b/crates/environment-protocol/src/control/template.rs @@ -0,0 +1,63 @@ +//! `template/list`: the templates a binding may create targets from. + +use serde::{Deserialize, Serialize}; + +use crate::{ + control::target::BindingRef, + ids::TemplateId, + metadata::{DescriptiveText, DisplayName, Metadata}, + serde_util::is_false, +}; + +/// `template/list` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListTemplatesParams { + pub binding: BindingRef, +} + +/// `template/list` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListTemplatesResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub templates: Vec, +} + +/// A template targets are created from. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct EnvironmentTemplate { + pub template_id: TemplateId, + pub display_name: DisplayName, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Targets of this template can serve public ingress. + #[serde(default, skip_serializing_if = "is_false")] + pub public_ingress: bool, + /// Creates no new targets; existing targets keep working. + #[serde(default, skip_serializing_if = "is_false")] + pub deprecated: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resources: Option, + /// Provider-specific display facts, such as an image fingerprint. + #[serde(default, skip_serializing_if = "Metadata::is_empty")] + pub metadata: Metadata, +} + +/// Compute a template's targets get, in provider-neutral units. Each +/// field is absent when the provider does not fix it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TemplateResources { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vcpus: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub disk_bytes: Option, +} diff --git a/crates/environment-protocol/src/data/events.rs b/crates/environment-protocol/src/data/events.rs deleted file mode 100644 index 0f7fefe96..000000000 --- a/crates/environment-protocol/src/data/events.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! Data-plane notifications. - -use serde::{Deserialize, Serialize}; - -use crate::{ - data::process::{ProcessOutputChunk, ProcessOutputStream}, - shared::{ByteChunk, ProcessId}, -}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessOutputNotification { - pub process_id: ProcessId, - pub seq: u64, - pub stream: ProcessOutputStream, - pub chunk: ByteChunk, -} - -impl ProcessOutputNotification { - pub fn from_chunk(process_id: ProcessId, chunk: ProcessOutputChunk) -> Self { - Self { - process_id, - seq: chunk.seq, - stream: chunk.stream, - chunk: chunk.chunk, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessExitedNotification { - pub process_id: ProcessId, - pub seq: u64, - pub exit_code: i32, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessClosedNotification { - pub process_id: ProcessId, - pub seq: u64, -} diff --git a/crates/environment-protocol/src/data/exec.rs b/crates/environment-protocol/src/data/exec.rs new file mode 100644 index 000000000..bde6edfe8 --- /dev/null +++ b/crates/environment-protocol/src/data/exec.rs @@ -0,0 +1,373 @@ +//! The execution primitive processes and jobs share: the command, its +//! input, its output stream and its outcome. +//! +//! **Output positions.** A handle's output is one byte stream: stdout, +//! stderr and the PTY interleaved in arrival order. Positions are byte +//! offsets into it. `end` is the number of bytes produced so far; a page +//! cut at `maxBytes` loses nothing because the next read starts at `next`; +//! a tail read starts at `end - tailBytes`. Bytes the daemon no longer +//! retains are reported as typed [`OutputGap`]s, never skipped silently. +//! +//! **Settled exit.** `state` becomes `exited` only once the root process +//! has exited and its output has settled (every reader reached end of file +//! or was discarded), so `end` is final from then on. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::{ + bytes::{ByteChunk, SecretString}, + paths::EnvironmentPath, + serde_util::{is_false, is_zero_u32}, +}; + +/// What to run. Shared by `process/start` and `job/start`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CommandSpec { + /// Program and arguments; `argv[0]` is resolved through `PATH`. Not + /// empty. + pub argv: Vec, + /// Working directory; the daemon's `defaultCwd` when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cwd: Option, + /// Variables added to the daemon's scrubbed environment. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, + /// Like `env`, but every value is redacted from the command's output. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub secret_env: BTreeMap, + /// Standard input; none by default. + #[serde(default, skip_serializing_if = "ExecInput::is_none")] + pub input: ExecInput, + /// Stop deadline from start. On expiry the daemon terminates the process + /// group (`SIGTERM`, the advertised grace, then `SIGKILL`) and records + /// `stoppedBy: timeout`. Absent means no deadline. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + +/// Standard input of a command. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ExecInput { + /// `/dev/null`: the command reads end of file. Model-facing tools use + /// this. + None {}, + /// These bytes, then end of file. Written by a writer task after the + /// output readers, the exit watcher and the deadline are in place. + Bytes { data: ByteChunk }, + /// A persistent pipe fed by `process/write` until closed. Processes + /// only; `job/start` refuses it with `invalidParams`. + Pipe {}, +} + +impl Default for ExecInput { + fn default() -> Self { + Self::None {} + } +} + +impl ExecInput { + pub fn is_none(&self) -> bool { + matches!(self, Self::None {}) + } +} + +/// Terminal size of a PTY. Each dimension is 1 to 4096. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PtySize { + pub rows: u32, + pub cols: u32, +} + +impl PtySize { + pub const MAX_DIMENSION: u32 = 4096; +} + +/// Which descriptor produced a chunk. Jobs never produce `pty`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum OutputStream { + Stdout, + Stderr, + /// The PTY master, which merges stdout and stderr. + Pty, +} + +/// Contiguous output bytes from one stream. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OutputChunk { + /// Byte offset of the chunk's first byte in the combined output. + pub offset: u64, + pub stream: OutputStream, + pub data: ByteChunk, +} + +/// Output bytes in `[offset, offset + omittedBytes)` that a page cannot +/// return. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OutputGap { + pub offset: u64, + pub omitted_bytes: u64, + pub reason: GapReason, +} + +/// Why a range of output cannot be returned. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum GapReason { + /// Never delivered: dropped from the middle because retained output + /// exceeded the handle's cap. The head and the newest bytes are kept. + Overflow, + /// Already delivered through the daemon cursor (or acknowledged) and + /// released since. + Expired, +} + +/// One page of a handle's output. Returned by `process/read` and in every +/// `job/read` entry. A page holds at most +/// [`MAX_PAGE_CHUNKS`](crate::limits::MAX_PAGE_CHUNKS) chunks and gaps +/// together (for `job/read`, across all entries); one that reaches it ends +/// early, and `next` says where to continue. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OutputPage { + /// In offset order, at most `maxBytes` of `data` in total. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub chunks: Vec, + /// Offset to read from next: past the last returned byte or gap. + pub next: u64, + /// Bytes produced so far; final once the handle has exited. + pub end: u64, + /// Unreturnable ranges between the requested start and `next`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub gaps: Vec, +} + +/// When a waiting read returns. Every mode also returns when `waitMs` +/// elapses, when the handle reaches a settled exit, or at once when there +/// is nothing left to wait for. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ReturnOn { + /// Collect for the whole window; return early only when unread output + /// reaches `maxBytes`. + Window, + /// Return as soon as any output past the start position exists. + AnyOutput, + /// Return only at the settled exit (for `job/read`: once every entry is + /// terminal), even if more than `maxBytes` of output accumulates. + Exit, +} + +/// Lifecycle state of a process. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ExecState { + Running, + /// The root exited and its output settled. + Exited, +} + +/// How the root process ended. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ExitStatus { + /// Normal exit with this status code. + Code { code: i32 }, + /// Killed by this signal. + Signal { + signal: SignalName, + #[serde(default, skip_serializing_if = "is_false")] + core_dumped: bool, + }, +} + +/// Why the daemon stopped a process or job that did not exit on its own. +/// Set only when the daemon signalled a root that was still running. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum StoppedBy { + /// `timeoutMs` expired: `SIGTERM`, the grace, then `SIGKILL`. + Timeout, + /// `process/terminate` with `terminate`, or `job/cancel` without + /// `force`: `SIGTERM`, the grace, then `SIGKILL`. + Terminate, + /// `process/terminate` with `kill`, or `job/cancel` with `force`: + /// `SIGKILL` at once. + Kill, +} + +/// A POSIX signal name such as `SIGSEGV` or `SIGRTMIN+3`: `SIG` followed by +/// 1 to 16 uppercase letters, digits or `+`. Names, not numbers, because +/// numbers differ between platforms. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct SignalName( + #[cfg_attr(feature = "schema", schemars(pattern(SIGNAL_NAME_PATTERN)))] String, +); + +/// JSON Schema pattern of [`SignalName`]. +pub const SIGNAL_NAME_PATTERN: &str = "^SIG[A-Z0-9+]{1,16}$"; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +#[error("signal name must be SIG followed by 1..=16 uppercase letters, digits or '+': {0:?}")] +pub struct SignalNameError(pub String); + +impl SignalName { + pub fn new(name: impl Into) -> Result { + let name = name.into(); + let valid = name.strip_prefix("SIG").is_some_and(|rest| { + (1..=16).contains(&rest.len()) + && rest + .bytes() + .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'+') + }); + if valid { + Ok(Self(name)) + } else { + Err(SignalNameError(name)) + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for SignalName { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(formatter) + } +} + +impl Serialize for SignalName { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for SignalName { + fn deserialize>(deserializer: D) -> Result { + let name = String::deserialize(deserializer)?; + Self::new(name).map_err(serde::de::Error::custom) + } +} + +/// Processes of a command's group still alive after its root exited. +/// Present only when there is at least one. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct LeftoverReport { + pub disposition: LeftoverDisposition, + /// At most [`MAX_LEFTOVER_PROCESSES`](crate::limits::MAX_LEFTOVER_PROCESSES), + /// lowest pid first; never empty. + pub processes: Vec, + /// Leftovers beyond the listed ones. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub omitted_process_count: u32, +} + +/// What the daemon did with leftovers. The policies differ on purpose. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum LeftoverDisposition { + /// Left running (processes): they keep the environment's lifetime and + /// are reported until they exit. + Kept, + /// Terminated when the job finished (jobs); the submitter should fix + /// its script's teardown. + Swept, +} + +/// One leftover process. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct LeftoverProcess { + pub pid: u32, + /// Command line when the platform exposes it, otherwise the executable + /// name, cut to + /// [`MAX_LEFTOVER_COMMAND_BYTES`](crate::limits::MAX_LEFTOVER_COMMAND_BYTES) + /// on a character boundary; absent when neither could be read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub command: Option, +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn exit_status_names_signals() { + let status = ExitStatus::Signal { + signal: SignalName::new("SIGSEGV").unwrap(), + core_dumped: true, + }; + assert_eq!( + serde_json::to_value(&status).unwrap(), + json!({"kind": "signal", "signal": "SIGSEGV", "coreDumped": true}) + ); + assert!(SignalName::new("SIGRTMIN+3").is_ok()); + for bad in ["SEGV", "SIG", "sigsegv", "SIGSEGV!"] { + assert!(SignalName::new(bad).is_err(), "{bad}"); + } + } + + #[test] + fn input_defaults_to_none_and_is_omitted() { + let spec: CommandSpec = serde_json::from_value(json!({"argv": ["true"]})).unwrap(); + assert!(spec.input.is_none()); + assert_eq!( + serde_json::to_value(&spec).unwrap(), + json!({"argv": ["true"]}) + ); + let piped: CommandSpec = + serde_json::from_value(json!({"argv": ["cat"], "input": {"kind": "pipe"}})).unwrap(); + assert_eq!(piped.input, ExecInput::Pipe {}); + assert!(serde_json::from_value::(json!({"argv": [], "stdin": "x"})).is_err()); + } + + #[test] + fn fieldless_input_kinds_reject_unknown_members() { + for kind in ["none", "pipe"] { + assert!(serde_json::from_value::(json!({"kind": kind})).is_ok()); + assert!( + serde_json::from_value::(json!({"kind": kind, "data": "aGk="})).is_err(), + "{kind}" + ); + } + } +} diff --git a/crates/environment-protocol/src/data/fs.rs b/crates/environment-protocol/src/data/fs.rs index 8ff51c0f9..075ec6075 100644 --- a/crates/environment-protocol/src/data/fs.rs +++ b/crates/environment-protocol/src/data/fs.rs @@ -1,232 +1,529 @@ -//! Filesystem method payloads. +//! `fs/*`: path-based filesystem methods. +//! +//! Every path is an absolute [`EnvironmentPath`] resolved under the +//! confinement rule stated on that type. Methods that change the +//! filesystem need `capabilities.filesystem.write`; without it they answer +//! `capabilityUnavailable` (`filesystemWrite`). +//! +//! **Pattern grammar** ([`TreeFilter`]), shared by `fs/searchText`, +//! `fs/globFiles`, `fs/scan` and transfer captures. Patterns match paths +//! relative to the walk's root, gitignore-style: `*` and `?` match within +//! one segment, `**` spans segments, `[...]` is a character class and +//! `{a,b}` an alternation. A pattern without `/` matches the entry's name +//! at any depth; a leading `/` anchors it to the root. Walks always prune +//! VCS metadata directories (`.git`, `.hg`, `.svn`, `.jj`) and the daemon +//! state directory; by default they also respect ignore files (`.gitignore`, +//! `.ignore`, git excludes) and skip hidden entries. An entry a walk cannot +//! represent or read (a name that is not UTF-8, a directory that cannot be +//! listed) is skipped and counted in the result's `skippedEntryCount`; +//! it never ends the walk. +//! +//! **Two entry shapes, on purpose.** [`FileMetadata`] describes one path as +//! it is now (`fs/getMetadata`, `fs/readDirectory`): kind including +//! symlinks, times and a `version` for preconditions. [`TreeContent`] +//! describes what a tree holds (`fs/scan` entries, transfer inventories): +//! directories and files with size, mode and digest, the facts two trees +//! are compared by. Trees never hold symlink entries: scans skip or follow +//! links and captures expand or skip them. use serde::{Deserialize, Serialize}; -use crate::shared::{ByteChunk, EnvironmentPath}; +use crate::{ + bytes::ByteChunk, + ids::{DirectoryCursor, validate_opaque_id}, + paths::{ContentDigest, EntryName, EnvironmentPath}, + serde_util::{is_false, is_zero_u32}, +}; -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Kind of a filesystem entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct ReadFileParams { - pub path: EnvironmentPath, - /// Byte offset to start reading from. Environment endpoints advertising - /// `filesystem_ranged_read` honor it; others ignore it and return the - /// full file. +pub enum FileKind { + File, + Directory, + Symlink, + /// Device, FIFO, socket or another special file. + Other, +} + +/// One file-entry shape for `fs/getMetadata` and `fs/readDirectory`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct FileMetadata { + pub kind: FileKind, + /// Size in bytes; for a symlink, the length of its target text. + pub size_bytes: u64, + /// A regular file with any execute bit set; always false for + /// directories, symlinks and special files. + #[serde(default, skip_serializing_if = "is_false")] + pub executable: bool, + pub modified_at_ms: i64, + /// Absent when the filesystem records no birth time. #[serde(default, skip_serializing_if = "Option::is_none")] - pub offset: Option, - /// Maximum bytes to return. Environment endpoints advertising `filesystem_ranged_read` - /// truncate at the source and set `truncated`. + pub created_at_ms: Option, + /// Target text of a symlink, when it is UTF-8. #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_bytes: Option, + pub symlink_target: Option, + /// Changes whenever the entry's content or identity changes; compare it + /// with `ifMatch`. + pub version: FileVersion, +} + +/// Opaque version token of a file's content and identity. Equal tokens for +/// one path mean the content is unchanged. 1 to 255 bytes without control +/// characters. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct FileVersion( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 255), pattern(crate::ids::NO_CONTROL_PATTERN)) + )] + String, +); + +impl FileVersion { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + validate_opaque_id("FileVersion", &value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Serialize for FileVersion { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for FileVersion { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(serde::de::Error::custom) + } } +/// What a tree entry is: a scan entry or a transfer inventory entry. +/// Digests identify raw file bytes, not paths or modes. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum TreeContent { + Directory {}, + File { + size_bytes: u64, + #[serde(default, skip_serializing_if = "is_false")] + executable: bool, + /// Scans: present when `includeDigests` was set. Transfer + /// inventories: always present; an appended file entry without it + /// is `invalidParams`. + #[serde(default, skip_serializing_if = "Option::is_none")] + digest: Option, + }, +} +/// `fs/readFile` params. Reads a regular file only; anything else is +/// `failedPrecondition` (`notRegularFile`) and is never opened for a +/// blocking read. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadFileParams { + pub path: EnvironmentPath, + /// First byte to return; 0 by default. + #[serde(default, skip_serializing_if = "crate::serde_util::is_zero_u64")] + pub offset: u64, + /// Most bytes to return, at most `limits.filesystem.maxReadFileBytes`. + pub max_bytes: u64, +} + +/// `fs/readFile` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReadFileResponse { pub data: ByteChunk, - /// Total size of the file in the environment, independent of the returned range. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub file_size: Option, - /// True when the returned data is a strict prefix range of the file. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] + /// Size of the whole file when it was read. + pub file_size: u64, + /// Bytes exist after `offset` plus the length of `data`. + #[serde(default, skip_serializing_if = "is_false")] pub truncated: bool, + pub version: FileVersion, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// When `fs/writeFile` may write. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] +pub enum WriteCondition { + /// Create or replace. + #[default] + Always, + /// Only create; an existing entry is `failedPrecondition` + /// (`createCondition`). + IfAbsent, + /// Only replace; a missing file is `failedPrecondition` + /// (`createCondition`). + IfPresent, +} + +impl WriteCondition { + pub fn is_default(&self) -> bool { + *self == Self::Always + } +} + +/// `fs/writeFile` params. The write is staged in the target's directory and +/// published by an atomic rename: readers see the old or the new content, +/// never a mix. An existing target keeps its permissions. A symlink at +/// `path` whose target stays inside the root is followed like any path: the +/// final target is replaced and the link itself is kept (a dangling link +/// creates its target, subject to `create`); a link that leaves the root is +/// `permissionDenied` (`outsideRoot`). A special file is +/// `failedPrecondition` (`notRegularFile`), a directory +/// `failedPrecondition` (`isDirectory`). The parent of the final target +/// must exist. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct WriteFileParams { pub path: EnvironmentPath, + /// At most `limits.filesystem.maxWriteFileBytes`; larger files go + /// through `transfer/*` with a `writeFile` selection. pub data: ByteChunk, + #[serde(default, skip_serializing_if = "WriteCondition::is_default")] + pub create: WriteCondition, + /// Write only when the current file's version equals this token; + /// otherwise `failedPrecondition` (`ifMatchFailed`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub if_match: Option, + /// Set the execute bits (the read bits' pattern). Without it a new file + /// gets the default mode and an existing file keeps its mode. + #[serde(default, skip_serializing_if = "is_false")] + pub executable: bool, } -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct WriteFileResponse {} +/// `fs/writeFile` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WriteFileResponse { + /// Version of the written file. + pub version: FileVersion, +} +/// `fs/createDirectory` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct CreateDirectoryParams { pub path: EnvironmentPath, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub recursive: Option, + /// Create missing parents, and succeed when the directory exists. + #[serde(default, skip_serializing_if = "is_false")] + pub recursive: bool, } +/// `fs/createDirectory` response. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct CreateDirectoryResponse {} +/// `fs/getMetadata` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GetMetadataParams { pub path: EnvironmentPath, + /// Describe the symlink's target instead of the link (subject to + /// confinement). + #[serde(default, skip_serializing_if = "is_false")] + pub follow_symlinks: bool, } +/// `fs/getMetadata` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GetMetadataResponse { - pub is_directory: bool, - pub is_file: bool, - pub is_symlink: bool, - pub created_at_ms: i64, - pub modified_at_ms: i64, + pub metadata: FileMetadata, } +/// `fs/readDirectory` params. Pages in name order. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReadDirectoryParams { + /// A directory, or a symlink to one inside the root. pub path: EnvironmentPath, + /// Most entries per page, 1 to `limits.filesystem.maxDirectoryEntries`. + pub limit: u32, + /// `nextCursor` of the previous page; a cursor this daemon did not mint + /// is `invalidParams`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cursor: Option, } +/// `fs/readDirectory` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReadDirectoryEntry { - pub file_name: String, - pub is_directory: bool, - pub is_file: bool, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadDirectoryResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub entries: Vec, + /// Opaque cursor of the next page; absent on the last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_cursor: Option, + /// Entries in this page's range skipped because their names are not + /// UTF-8. Entries that vanish while the page is built are left out and + /// not counted. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_entry_count: u32, } +/// One `fs/readDirectory` entry. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReadDirectoryResponse { - pub entries: Vec, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DirectoryEntry { + pub name: EntryName, + /// The entry itself, not a symlink's target. Entries that vanish while + /// the page is built are left out. + pub metadata: FileMetadata, } +/// `fs/remove` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RemoveParams { + /// A symlink is removed, never followed. pub path: EnvironmentPath, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub recursive: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub force: Option, + /// Remove a directory and its contents. + #[serde(default, skip_serializing_if = "is_false")] + pub recursive: bool, + /// Succeed when the path does not exist. + #[serde(default, skip_serializing_if = "is_false")] + pub force: bool, } +/// `fs/remove` response. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RemoveResponse {} +/// `fs/rename` params. Atomic within one filesystem; across filesystems it +/// is `failedPrecondition` (`crossDevice`). The entry keeps its mode. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CopyParams { - pub source_path: EnvironmentPath, - pub destination_path: EnvironmentPath, - #[serde(default)] - pub recursive: bool, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RenameParams { + pub source: EnvironmentPath, + pub destination: EnvironmentPath, + /// Replace an existing destination file; otherwise an existing + /// destination is `alreadyExists`. A directory destination is never + /// replaced. + #[serde(default, skip_serializing_if = "is_false")] + pub overwrite: bool, + /// Rename only when the source's version equals this token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub if_match: Option, } +/// `fs/rename` response. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CopyResponse {} +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RenameResponse {} + +/// Which entries a walk visits. The default visits everything except +/// hidden and ignored entries. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TreeFilter { + /// Select entries matching any pattern; empty selects every entry. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub include: Vec, + /// Drop entries matching any pattern; a matching directory is pruned. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude: Vec, + /// Match patterns case-insensitively. + #[serde(default, skip_serializing_if = "is_false")] + pub ignore_case: bool, + /// Visit entries whose name starts with `.`. + #[serde(default, skip_serializing_if = "is_false")] + pub include_hidden: bool, + /// Visit entries that ignore files exclude. + #[serde(default, skip_serializing_if = "is_false")] + pub include_ignored: bool, + /// Deepest entry depth visited: the root's direct children are depth + /// 1, so `1` selects direct children only. Unbounded when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_depth: Option, +} + +impl TreeFilter { + pub fn is_default(&self) -> bool { + *self == Self::default() + } +} -/// Bounded recursive text search executed natively by the environment endpoint. -/// -/// The environment endpoint performs traversal and matching locally and returns only bounded -/// matches plus scan statistics, so a broad search does not become per-file -/// transfer over the data plane. Every limit is mandatory: an endpoint must stop -/// scanning when any bound is reached and report why. +/// `fs/searchText` params: a bounded recursive regex search run by the +/// daemon. A file larger than the remaining byte budget is skipped and +/// counted; it never ends the walk. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchTextParams { pub root: EnvironmentPath, - /// Regular expression (Rust `regex` dialect, the same dialect the - /// caller-side fallback uses). + /// Regular expression in the Rust `regex` crate's syntax, matched per + /// line. pub pattern: String, - /// Optional glob filter applied to the root-relative path or file name. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub include: Option, - #[serde(default)] - pub case_sensitive: bool, - /// Maximum directory depth below the root, matching the generic - /// traversal's semantics; absent means unbounded depth. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_depth: Option, + /// Match `pattern` case-insensitively. `filter.ignoreCase` governs the + /// path patterns instead. + #[serde(default, skip_serializing_if = "is_false")] + pub pattern_ignore_case: bool, + #[serde(default, skip_serializing_if = "TreeFilter::is_default")] + pub filter: TreeFilter, pub limits: SearchTextLimits, } +/// Caller quotas of one search; each at most its advertised ceiling. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchTextLimits { - pub max_matches: u64, - /// Maximum number of files searched (not merely visited by traversal). - pub max_files: u64, - /// Maximum cumulative bytes searched across files. + pub max_matches: u32, + /// Files searched, not merely visited. + pub max_files: u32, + /// Bytes actually read across files. pub max_bytes: u64, pub max_duration_ms: u64, + /// Longer matching lines are cut and marked `lineTruncated`. + pub max_line_bytes: u32, } +/// `fs/searchText` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchTextResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub matches: Vec, + pub searched_file_count: u32, + pub searched_bytes: u64, + /// Files skipped because they exceeded the remaining byte budget, are + /// binary or could not be read. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_file_count: u32, + /// Entries the walk could not represent or list: names that are not + /// UTF-8 and directories that could not be read. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_entry_count: u32, + pub elapsed_ms: u64, + /// Absent when the walk finished within its limits. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stopped: Option, +} + +/// One matching line. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchTextMatch { pub path: EnvironmentPath, - pub line_number: u64, + /// 1-based. + pub line_number: u32, + /// The matching line without its terminator, cut at `maxLineBytes` on a + /// character boundary. pub line: String, + #[serde(default, skip_serializing_if = "is_false")] + pub line_truncated: bool, } -/// Why a search stopped before exhausting the tree. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Why a search stopped before finishing its walk. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] pub enum SearchTextStop { + /// `maxMatches` matches were found. MatchLimit, + /// `maxFiles` files were searched. FileLimit, + /// `maxBytes` bytes were read. ByteLimit, + /// `maxDurationMs` ran out. TimeLimit, + /// The next match would push the encoded matches past + /// [`MAX_RESULT_BYTES`](crate::limits::MAX_RESULT_BYTES). + ResponseLimit, } +/// `fs/globFiles` params: bounded recursive enumeration of non-directory +/// entries matching `filter.include`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SearchTextResponse { - pub matches: Vec, - pub files_searched: u64, - pub bytes_searched: u64, - pub elapsed_ms: u64, - /// Absent when the search exhausted the tree within its limits. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stopped: Option, -} - -/// Bounded recursive file enumeration executed natively by the environment endpoint. -/// -/// Pattern semantics mirror the caller-side generic glob tool: a pattern -/// starting with `/` matches the caller-space absolute path; otherwise the -/// root-relative path matches, and a pattern without `/` also matches bare -/// file names. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GlobFilesParams { pub root: EnvironmentPath, - pub pattern: String, - /// Maximum directory depth below the root, matching the generic - /// traversal's semantics; absent means unbounded depth. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_depth: Option, + /// `include` must not be empty. + pub filter: TreeFilter, pub limits: GlobFilesLimits, } +/// Caller quotas of one enumeration; each at most its advertised ceiling. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GlobFilesLimits { - pub max_matches: u64, - /// Maximum directory entries (files and directories) visited by the - /// traversal. - pub max_entries: u64, + pub max_matches: u32, + /// Non-directory entries visited. + pub max_files: u32, pub max_duration_ms: u64, } -/// Why an enumeration stopped before exhausting the tree. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum GlobFilesStop { - MatchLimit, - EntryLimit, - TimeLimit, -} - +/// `fs/globFiles` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GlobFilesResponse { + /// In walk order (sorted by path). + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub matches: Vec, - pub entries_visited: u64, + pub visited_file_count: u32, + /// Entries the walk could not represent or list: names that are not + /// UTF-8 and directories that could not be read. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_entry_count: u32, pub elapsed_ms: u64, - /// Absent when the enumeration exhausted the tree within its limits. #[serde(default, skip_serializing_if = "Option::is_none")] pub stopped: Option, } + +/// Why an enumeration stopped before finishing its walk. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum GlobFilesStop { + /// `maxMatches` paths were found. + MatchLimit, + /// `maxFiles` non-directory entries were visited. + FileLimit, + /// `maxDurationMs` ran out. + TimeLimit, + /// The next match would push the encoded matches past + /// [`MAX_RESULT_BYTES`](crate::limits::MAX_RESULT_BYTES). + ResponseLimit, +} diff --git a/crates/environment-protocol/src/data/handshake.rs b/crates/environment-protocol/src/data/handshake.rs index 2a59d46e2..5def4fa5c 100644 --- a/crates/environment-protocol/src/data/handshake.rs +++ b/crates/environment-protocol/src/data/handshake.rs @@ -1,41 +1,230 @@ -//! Data-plane connection handshake. +//! Data-plane `initialize`. +//! +//! `initialize` is the first request on every data connection; any other +//! method before it is `notInitialized`, and a second `initialize` is +//! `invalidRequest`. A `protocolVersion` other than +//! [`ENVD_PROTOCOL_VERSION`](crate::ENVD_PROTOCOL_VERSION) is +//! `versionMismatch`. The connection stays open after all three. use serde::{Deserialize, Serialize}; -use crate::shared::{ - CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentConnectionId, EnvironmentScope, - ImplementationInfo, +use crate::{ + handshake::{ImplementationInfo, Plane}, + ids::DaemonInstanceId, + paths::EnvironmentPath, + serde_util::is_false, }; +/// `initialize` params on the data plane. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct InitializeParams { - #[serde(default = "default_protocol_version")] + /// The client's `ENVD_PROTOCOL_VERSION`. pub protocol_version: u32, - pub client_name: String, - pub scope: EnvironmentScope, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub resume_connection_id: Option, + pub client_info: ImplementationInfo, } +/// `initialize` response on the data plane. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct InitializeResponse { pub protocol_version: u32, - pub connection_id: EnvironmentConnectionId, - pub capabilities: EnvironmentCapabilities, + /// Always `data`. + pub plane: Plane, + /// Random per daemon start; a different value than on an earlier + /// connection means the daemon restarted and the earlier instance's + /// process handles are gone. + pub daemon_instance_id: DaemonInstanceId, + pub server_info: ImplementationInfo, + pub capabilities: DataCapabilities, + pub limits: DataLimits, + /// The daemon's working directory: the default `cwd` of commands. + pub default_cwd: EnvironmentPath, + /// Home directory of the user commands run as, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub home_directory: Option, +} + +/// Capability families. A present family means its base methods exist; a +/// family's fields are facts that vary per instance. A request to an +/// absent family, or one needing an absent fact, is +/// `capabilityUnavailable`. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DataCapabilities { + /// `fs/*` (search, glob and scan included) and `transfer/*`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filesystem: Option, + /// `process/*`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_cwd: Option, - /// Home directory of the endpoint execution user. + pub process: Option, + /// `job/*`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub home_directory: Option, - pub implementation: ImplementationInfo, + pub job: Option, } +/// The `filesystem` family: `fs/*` and `transfer/*`. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InitializedParams {} +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct FilesystemCapabilities { + /// Methods that change the filesystem are served; false on a daemon + /// started read-only. + #[serde(default, skip_serializing_if = "is_false")] + pub write: bool, +} + +/// The `process` family: `process/*`. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ProcessCapabilities { + /// `process/start` accepts `pty`. + #[serde(default, skip_serializing_if = "is_false")] + pub pty: bool, +} + +/// The `job` family: `job/*`. It has no instance facts. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobCapabilities {} + +/// Operational ceilings this daemon enforces. A request above a ceiling is +/// `invalidParams` naming the field and the ceiling; nothing is clamped. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DataLimits { + /// Requests in flight per connection before `busy`; at least + /// [`MIN_CONCURRENT_REQUESTS`](crate::limits::MIN_CONCURRENT_REQUESTS). + pub max_concurrent_requests: u32, + /// Largest `waitMs` of `process/read` and `job/read`. + pub max_wait_ms: u64, + pub process: ProcessLimits, + pub job: JobLimits, + pub filesystem: FilesystemLimits, +} + +/// Ceilings of `process/*`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ProcessLimits { + /// Largest `maxBytes` of one `process/read`. + pub max_read_bytes: u64, + /// Output a process retains; beyond it the oldest delivered bytes are + /// released (`expired` gaps), then the undelivered middle (`overflow` + /// gaps). + pub max_retained_output_bytes: u64, + /// How long an exited process stays readable after a read first + /// returned its settled exit with nothing left unread. + pub retention_after_exit_ms: u64, + /// How long an exited process whose output was never fully read stays + /// readable. + pub unread_retention_ms: u64, + /// Output retained across all exited processes; beyond it the oldest + /// are evicted early. + pub max_exited_retained_bytes: u64, + /// Grace between `SIGTERM` and `SIGKILL` for `terminate` and timeouts. + pub terminate_grace_ms: u64, + /// Bytes queued for a process's input before `process/write` accepts + /// fewer than it was sent. + pub max_input_queue_bytes: u64, +} + +/// Ceilings of `job/*`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobLimits { + /// Largest `maxBytes` of one `job/read`, across its entries. + pub max_read_bytes: u64, + /// Most jobs one `job/start`, `job/read` or `job/cancel` names, and most + /// entries a `job/cancel` response lists. + pub max_request_jobs: u32, + /// Output a job retains: the head and the tail, each half of this; the + /// middle is an `overflow` gap. + pub max_retained_output_bytes: u64, + /// How long a terminal job's record and output are kept. + pub record_retention_ms: u64, + /// Grace between `SIGTERM` and `SIGKILL` for a non-forced cancel and + /// timeouts. + pub cancel_grace_ms: u64, +} + +/// Ceilings of `fs/*` and `transfer/*`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct FilesystemLimits { + /// Largest `maxBytes` of `fs/readFile`. + pub max_read_file_bytes: u64, + /// Largest `data` of `fs/writeFile`. + pub max_write_file_bytes: u64, + /// Largest `limit` of `fs/readDirectory`. + pub max_directory_entries: u32, + pub search: SearchCeilings, + pub glob: GlobCeilings, + pub scan: ScanCeilings, + pub transfer: TransferCeilings, +} + +/// Ceilings of `fs/searchText` quotas, field for field. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchCeilings { + pub max_matches: u32, + pub max_files: u32, + pub max_bytes: u64, + pub max_duration_ms: u64, + pub max_line_bytes: u32, +} + +/// Ceilings of `fs/globFiles` quotas, field for field. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GlobCeilings { + pub max_matches: u32, + pub max_files: u32, + pub max_duration_ms: u64, +} + +/// Ceilings of `fs/scan`: root and pattern counts, then its quotas. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanCeilings { + pub max_roots: u32, + pub max_patterns: u32, + pub max_entries: u32, + pub max_file_bytes: u64, + pub max_total_bytes: u64, + pub max_duration_ms: u64, +} -fn default_protocol_version() -> u32 { - CURRENT_PROTOCOL_VERSION +/// Ceilings of `transfer/*`: capacity, page and chunk sizes, receipt +/// retention, then the per-operation quotas. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TransferCeilings { + /// Operations not yet complete, aborted or failed, across connections; + /// a begin beyond it is `busy` (`transferCapacity`). + pub max_active_operations: u32, + pub max_content_chunk_bytes: u64, + pub max_inventory_page_entries: u32, + /// How long a finished operation's receipt is answered. + pub receipt_retention_ms: u64, + pub max_entries: u32, + pub max_depth: u32, + pub max_file_bytes: u64, + pub max_total_bytes: u64, + pub max_manifest_bytes: u64, + pub max_duration_ms: u64, } diff --git a/crates/environment-protocol/src/data/idle.rs b/crates/environment-protocol/src/data/idle.rs index 4ebd0bbfa..d95831bbe 100644 --- a/crates/environment-protocol/src/data/idle.rs +++ b/crates/environment-protocol/src/data/idle.rs @@ -1,41 +1,38 @@ -//! Daemon activity report used by idle power policy. +//! `env/getIdle`: the daemon's activity report for idle power policy. //! -//! The daemon reports a monotonic idle *duration*, never a wall-clock -//! timestamp: after a freeze, stateful stop, or snapshot restore the guest -//! clock is stale until it resyncs, while a monotonic duration stays correct. +//! The daemon reports a monotonic idle duration, never a wall-clock +//! timestamp: after a freeze, stateful stop or snapshot restore the guest +//! clock is stale until it resyncs, while a monotonic duration stays +//! correct. Policy decides on `idleForMs` alone; the counts are +//! diagnostics. use serde::{Deserialize, Serialize}; +use crate::serde_util::is_zero_u32; + +/// `env/getIdle` params. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct IdleParams {} +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GetIdleParams {} +/// `env/getIdle` response. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct IdleResponse { - /// Milliseconds since the last data-plane request or since the last - /// process/job finished, whichever is later. Zero while anything runs. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GetIdleResponse { + /// Milliseconds since the last `use` request, or since running work + /// last ended, whichever is later. Zero while any process or job runs. pub idle_for_ms: u64, - /// Processes started through `process/start` that have not exited. - pub running_processes: u32, - /// Jobs that are running or cancelling. - pub running_jobs: u32, - /// Process groups of finished commands that still have members alive: - /// services and other leftovers a command started and did not stop. - /// They are not running work, so they never block idle power policy; - /// a policy may prefer freezing over stopping while the count is - /// non-zero. - #[serde(default, skip_serializing_if = "is_zero")] - pub leftover_process_groups: u32, -} - -fn is_zero(value: &u32) -> bool { - *value == 0 -} - -impl IdleResponse { - /// True when nothing is executing inside the environment. - pub fn is_quiescent(&self) -> bool { - self.running_processes == 0 && self.running_jobs == 0 - } + /// Processes started with `process/start` whose root has not exited. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub running_process_count: u32, + /// Jobs that are `running`. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub running_job_count: u32, + /// Process groups of exited processes that still have members alive: + /// services a command started and left running. They are not running + /// work and do not hold `idleForMs` at zero. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub leftover_process_group_count: u32, } diff --git a/crates/environment-protocol/src/data/inventory.rs b/crates/environment-protocol/src/data/inventory.rs deleted file mode 100644 index 42d9a58f0..000000000 --- a/crates/environment-protocol/src/data/inventory.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! Portable filesystem inventories. Digests identify raw file bytes, not paths or modes. -use crate::shared::{ByteChunk, EnvironmentPath}; -use serde::{Deserialize, Serialize}; - -pub const MAX_INVENTORY_PAGE: usize = 128; -pub const MAX_CONTENT_CHUNK: usize = 256 * 1024; -pub const MAX_INVENTORY_PATH_BYTES: usize = 4096; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "kind", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum InventoryContent { - Directory, - File { - size_bytes: u64, - executable: bool, - digest: String, - }, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InventoryEntry { - /// Empty for the selected root; otherwise a strict relative slash-separated path. - pub path: String, - pub content: InventoryContent, -} - -/// Total operation quotas, separate from wire chunk and page limits. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InventoryLimits { - pub max_entries: u32, - pub max_depth: u32, - pub max_file_bytes: u64, - pub max_total_bytes: u64, - pub max_manifest_bytes: u64, - pub max_duration_ms: u64, -} -impl Default for InventoryLimits { - fn default() -> Self { - Self { - max_entries: 100_000, - max_depth: 64, - max_file_bytes: 1024_u64.pow(4), - max_total_bytes: 1024_u64.pow(4), - max_manifest_bytes: 32 * 1024 * 1024, - max_duration_ms: 24 * 60 * 60 * 1000, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ScanDigestAlgorithm { - Sha256, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "kind", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum ScanContent { - Directory, - File { - size_bytes: u64, - executable: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - digest: Option, - }, -} - -/// Small, bounded observation for catalogs. Large inventories use transfer sessions. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScanParams { - pub roots: Vec, - /// Filename-only patterns without `/` or `**` select direct children only. - /// Other patterns permit recursive traversal within the operation quotas. - #[serde(default)] - pub include_patterns: Vec, - #[serde(default)] - pub read_content: bool, - /// Follow aliases only inside the endpoint filesystem access scope. - #[serde(default)] - pub follow_symlinks: bool, - #[serde(default)] - pub digest_algorithm: Option, - #[serde(default)] - pub limits: InventoryLimits, - pub if_none_match: Option, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScanEntry { - /// Resolved absolute identity, while root/path retain the requested spelling. - pub canonical_path: EnvironmentPath, - pub root: EnvironmentPath, - pub path: String, - pub content: ScanContent, - pub data: Option, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScanDiagnostic { - pub root: EnvironmentPath, - pub error: crate::error::EnvironmentProtocolError, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScanResponse { - pub fingerprint: Option, - pub unchanged: bool, - pub complete: bool, - pub entries: Vec, - pub diagnostics: Vec, -} diff --git a/crates/environment-protocol/src/data/job.rs b/crates/environment-protocol/src/data/job.rs new file mode 100644 index 000000000..30f5e8a04 --- /dev/null +++ b/crates/environment-protocol/src/data/job.rs @@ -0,0 +1,302 @@ +//! `job/*`: durable batch commands with dependencies and queues. +//! +//! A job is identified by its `jobId` on the daemon together with the +//! `requestId` of the `job/start` that created it. Its record and output +//! survive daemon restarts; a job that was queued or running when the +//! daemon stopped is `interrupted`. When a job's root process finishes, the +//! daemon sweeps the job's process group and reports what it swept. + +use serde::{Deserialize, Serialize}; + +use crate::{ + data::exec::{CommandSpec, ExitStatus, LeftoverReport, OutputPage, ReturnOn, StoppedBy}, + error::ProtocolError, + ids::{JobId, JobName, JobRequestId, QueueKey}, + serde_util::{is_false, is_zero_u32}, +}; + +/// `job/start` params. All-or-nothing: either every job is accepted (or +/// replayed) or none is. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct StartJobsParams { + /// Idempotency key of this start. Retrying with the same request id and + /// identical jobs replays the existing jobs; reusing a job id with + /// another request id or spec is `failedPrecondition` (`jobIdInUse`). + pub request_id: JobRequestId, + /// 1 to `limits.job.maxRequestJobs` jobs. + pub jobs: Vec, +} + +/// One job of a `job/start`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobSpec { + pub job_id: JobId, + /// Local name other jobs of the same `job/start` can depend on; unique + /// within the request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// `input` must be `none` or `bytes`. + pub command: CommandSpec, + /// Jobs that must finish first. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub depends_on: Vec, + /// What "finish" means for `dependsOn`; `allSucceeded` by default. + #[serde(default, skip_serializing_if = "JobDependencyPolicy::is_default")] + pub dependency_policy: JobDependencyPolicy, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, +} + +/// A dependency reference. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum JobDependency { + /// An existing job on this daemon, or one in the same request. + JobId { job_id: JobId }, + /// The `name` of a job in the same `job/start`. + Name { name: JobName }, +} + +/// What a job waits for from its `dependsOn` jobs. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum JobDependencyPolicy { + /// Run once every dependency `succeeded`; any other terminal status + /// makes this job `dependencyFailed`. + #[default] + AllSucceeded, + /// Run once every dependency is terminal, whatever its status. + AllTerminal, +} + +impl JobDependencyPolicy { + pub fn is_default(&self) -> bool { + *self == Self::AllSucceeded + } +} + +/// `job/start` response: one summary per requested job, in request order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct StartJobsResponse { + /// Never empty. + pub jobs: Vec, +} + +/// `job/read` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadJobsParams { + /// 1 to `limits.job.maxRequestJobs` entries, each with its own position. + pub jobs: Vec, + /// Most `data` bytes across all entries, at most + /// `limits.job.maxReadBytes`, filled in request order: an entry reached + /// after the budget is spent returns its summary and an empty page at + /// its start position. 0 reads only summaries. + pub max_bytes: u64, + /// How long to wait, at most `limits.maxWaitMs`. 0 polls. + pub wait_ms: u64, + /// `anyOutput`: return when any entry has output past its position or + /// changed status; `exit`: when every entry is terminal; `window`: wait + /// the whole window. + pub return_on: ReturnOn, +} + +/// One entry of a `job/read`. Job reads never consume: without `after` and +/// `tailBytes` the page starts at offset 0. One job may be named by several +/// entries, such as a head and a tail page. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobReadRequest { + pub job_id: JobId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub after: Option, + /// Start at `end - tailBytes`; excludes `after`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tail_bytes: Option, + /// Most `data` bytes for this entry; the request's `maxBytes` still + /// bounds the total. Absent: whatever of the request's budget remains. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_bytes: Option, +} + +/// `job/read` response: one entry per requested job, in request order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadJobsResponse { + /// Never empty. + pub jobs: Vec, +} + +/// A `job/read` entry: the job's summary and output page, or why it could +/// not be read (`notFound` for an unknown or garbage-collected job). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum JobReadEntry { + Job { + summary: Box, + output: OutputPage, + }, + Error { + job_id: JobId, + error: ProtocolError, + }, +} + +/// `job/cancel` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CancelJobsParams { + /// 1 to `limits.job.maxRequestJobs` ids. + pub job_ids: Vec, + /// `job` by default. + #[serde(default, skip_serializing_if = "JobCancelScope::is_default")] + pub scope: JobCancelScope, + /// Kill at once instead of `SIGTERM`, `limits.job.cancelGraceMs`, then + /// `SIGKILL`. + #[serde(default, skip_serializing_if = "is_false")] + pub force: bool, +} + +/// Which jobs a `job/cancel` affects. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum JobCancelScope { + /// Only the named jobs. + #[default] + Job, + /// The named jobs and every job that transitively depends on them. + Dependents, +} + +impl JobCancelScope { + pub fn is_default(&self) -> bool { + *self == Self::Job + } +} + +/// `job/cancel` response: one entry per affected job, named jobs first in +/// request order, then dependents, at most `limits.job.maxRequestJobs` +/// entries in total. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CancelJobsResponse { + /// Never empty. + pub jobs: Vec, + /// Dependents cancelled as well but not listed, beyond the entry limit. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub unlisted_job_count: u32, +} + +/// A job summary, or why the named job could not be found. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum JobSummaryEntry { + Job { summary: Box }, + Error { job_id: JobId, error: ProtocolError }, +} + +/// What a job is and how it ended. Instants are the guest's wall clock. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobSummary { + pub job_id: JobId, + pub request_id: JobRequestId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub status: JobStatus, + /// Resolved ids of the jobs this one depends on. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub depends_on: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, + pub created_at_ms: i64, + /// When the job became runnable and waited for its queue. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queued_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at_ms: Option, + /// When a `job/cancel` reached the job while it ran. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cancel_requested_at_ms: Option, + /// Monotonic run time from start to finish, immune to guest clock jumps. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// How the root process ended, once it ran and exited. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit: Option, + /// Set when the daemon stopped the running job: `timeout` for + /// `timedOut`, `terminate` or `kill` for a cancel without or with + /// `force`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stopped_by: Option, + /// What the daemon swept from the job's group at completion + /// (`disposition: swept`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub leftovers: Option, + /// Why a `failed` or `interrupted` job produced no exit, or an + /// infrastructure failure while it ran. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Job lifecycle. `queued` and `running` are live; the rest are terminal. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum JobStatus { + /// Accepted; waiting for dependencies or its queue. + Queued, + Running, + /// Exited with code 0. + Succeeded, + /// Exited otherwise, or could not start. + Failed, + /// Its `timeoutMs` expired and the daemon stopped it. + TimedOut, + /// Stopped or never started because of `job/cancel`. + Cancelled, + /// A dependency did not finish as the dependency policy requires. + DependencyFailed, + /// The daemon stopped while the job was queued or running. + Interrupted, +} + +impl JobStatus { + pub fn is_terminal(self) -> bool { + !matches!(self, Self::Queued | Self::Running) + } +} diff --git a/crates/environment-protocol/src/data/jobs.rs b/crates/environment-protocol/src/data/jobs.rs deleted file mode 100644 index 7db917d75..000000000 --- a/crates/environment-protocol/src/data/jobs.rs +++ /dev/null @@ -1,240 +0,0 @@ -//! Durable job method payloads. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; - -use crate::shared::{ByteChunk, EnvironmentPath, JobId, SecretString}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StartJobsParams { - pub namespace: String, - pub request_id: String, - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct JobStartSpec { - pub job_id: JobId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - pub argv: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cwd: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub env: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub secret_env: BTreeMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stdin: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout_ms: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub depends_on: Vec, - #[serde(default)] - pub dependency_policy: JobDependencyPolicy, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub queue_key: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct JobDependency { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub job_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, -} - -impl JobDependency { - pub fn job_id(job_id: impl Into) -> Self { - Self { - job_id: Some(job_id.into()), - name: None, - } - } - - pub fn name(name: impl Into) -> Self { - Self { - job_id: None, - name: Some(name.into()), - } - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum JobDependencyPolicy { - #[default] - AllSucceeded, - AllTerminal, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StartJobsResponse { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListJobsParams { - pub namespace: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub limit: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListJobsResponse { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReadJobsParams { - pub namespace: String, - pub jobs: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub after_seq: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_bytes: Option, - #[serde(default)] - pub include_artifacts: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub wait_ms: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReadJobsResponse { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct JobReadResult { - pub summary: JobSummary, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub output_chunks: Vec, - pub output_next_seq: u64, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub artifacts: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CancelJobsParams { - pub namespace: String, - pub jobs: Vec, - #[serde(default)] - pub scope: JobCancelScope, - #[serde(default)] - pub force: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CancelJobsResponse { - pub jobs: Vec, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum JobCancelScope { - #[default] - Job, - Dependents, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub struct JobSummary { - pub namespace: String, - pub job_id: JobId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - pub status: JobStatus, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub dependencies: Vec, - pub created_at_ms: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub queued_at_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub started_at_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub finished_at_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub exit_code: Option, - /// True when the job's root process exited while descendants it spawned - /// were still running. The environment daemon terminates them best-effort at job - /// completion; a submitting agent should fix its script's teardown. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub orphaned_descendants: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub failure: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub queue_key: Option, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub enum JobStatus { - Accepted, - Queued, - Running, - Succeeded, - Failed, - CancelRequested, - Cancelled, - TimedOut, - DependencyFailed, - Interrupted, - Lost, -} - -impl JobStatus { - pub fn is_terminal(self) -> bool { - matches!( - self, - Self::Succeeded - | Self::Failed - | Self::Cancelled - | Self::TimedOut - | Self::DependencyFailed - | Self::Interrupted - | Self::Lost - ) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct JobOutputChunk { - pub seq: u64, - pub stream: JobOutputStream, - pub chunk: ByteChunk, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub enum JobOutputStream { - Stdout, - Stderr, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub struct JobArtifact { - pub path: EnvironmentPath, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub kind: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, -} diff --git a/crates/environment-protocol/src/data/methods.rs b/crates/environment-protocol/src/data/methods.rs deleted file mode 100644 index 3f9647c80..000000000 --- a/crates/environment-protocol/src/data/methods.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! JSON-RPC method names for the environment data plane. - -pub const INITIALIZE_METHOD: &str = "initialize"; -pub const INITIALIZED_METHOD: &str = "initialized"; - -pub const FS_READ_FILE_METHOD: &str = "fs/readFile"; -pub const FS_WRITE_FILE_METHOD: &str = "fs/writeFile"; -pub const FS_CREATE_DIRECTORY_METHOD: &str = "fs/createDirectory"; -pub const FS_GET_METADATA_METHOD: &str = "fs/getMetadata"; -pub const FS_READ_DIRECTORY_METHOD: &str = "fs/readDirectory"; -pub const FS_REMOVE_METHOD: &str = "fs/remove"; -pub const FS_COPY_METHOD: &str = "fs/copy"; -pub const FS_SEARCH_TEXT_METHOD: &str = "fs/searchText"; -pub const FS_GLOB_FILES_METHOD: &str = "fs/globFiles"; - -pub const PROCESS_START_METHOD: &str = "process/start"; -pub const PROCESS_READ_METHOD: &str = "process/read"; -pub const PROCESS_WRITE_METHOD: &str = "process/write"; -pub const PROCESS_TERMINATE_METHOD: &str = "process/terminate"; -pub const PROCESS_RESIZE_METHOD: &str = "process/resize"; -pub const PROCESS_OUTPUT_METHOD: &str = "process/output"; -pub const PROCESS_EXITED_METHOD: &str = "process/exited"; -pub const PROCESS_CLOSED_METHOD: &str = "process/closed"; - -pub const JOB_START_METHOD: &str = "job/start"; -pub const JOB_LIST_METHOD: &str = "job/list"; -pub const JOB_READ_METHOD: &str = "job/read"; -pub const JOB_CANCEL_METHOD: &str = "job/cancel"; - -pub const ENV_IDLE_METHOD: &str = "env/idle"; - -pub const FS_MATERIALIZE_METHOD: &str = "fs/materialize"; -pub const FS_CAPTURE_METHOD: &str = "fs/capture"; - -pub const FS_SCAN_METHOD: &str = "fs/scan"; -pub const FS_TRANSFER_METHOD: &str = "fs/transfer"; diff --git a/crates/environment-protocol/src/data/mod.rs b/crates/environment-protocol/src/data/mod.rs index f4dcb3340..5e143ce19 100644 --- a/crates/environment-protocol/src/data/mod.rs +++ b/crates/environment-protocol/src/data/mod.rs @@ -1,13 +1,11 @@ -//! Data-plane protocol for one selected environment execution scope. +//! The data plane: one envd serving one environment's filesystem, +//! processes and jobs. The runtime is the client; envd is the server. -pub mod events; +pub mod exec; pub mod fs; pub mod handshake; pub mod idle; -pub mod jobs; -pub mod methods; +pub mod job; pub mod process; +pub mod scan; pub mod transfer; - -pub mod inventory; -pub mod transfer_session; diff --git a/crates/environment-protocol/src/data/process.rs b/crates/environment-protocol/src/data/process.rs index 57b69c643..fac452aea 100644 --- a/crates/environment-protocol/src/data/process.rs +++ b/crates/environment-protocol/src/data/process.rs @@ -1,203 +1,167 @@ -//! Process method payloads. +//! `process/*`: interactive processes read through byte-offset pages. //! -//! A process is started as the leader of its own process group and read -//! through a daemon-owned cursor: `process/read` without `afterSeq` returns -//! the output produced since the previous cursor read and advances the -//! cursor, so a handle used from a later connection never re-reads output. -//! The root's natural exit never sweeps the group; whatever it left running -//! is reported and keeps the environment's lifetime. - -use std::collections::BTreeMap; +//! A process runs as the leader of its own process group. Its root's +//! natural exit never sweeps the group: whatever it left running is +//! reported in `leftovers` and keeps the environment's lifetime. Each +//! process has a daemon-owned cursor; a read without `after` starts there, +//! and the cursor advances to `next` only after the response frame was +//! written to the connection. use serde::{Deserialize, Serialize}; -use crate::shared::{ByteChunk, EnvironmentPath, ProcessId, SecretString}; - +use crate::{ + bytes::ByteChunk, + data::exec::{ + CommandSpec, ExecState, ExitStatus, LeftoverReport, OutputPage, PtySize, ReturnOn, + StoppedBy, + }, + error::ProtocolError, + ids::ProcessId, + serde_util::is_false, +}; + +/// `process/start` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct StartProcessParams { + /// Caller-chosen handle, reserved before any side effect: a start whose + /// id is already in use (running, retained or remembered by a tombstone) + /// is `alreadyExists` and runs nothing. pub process_id: ProcessId, - pub argv: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cwd: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub env: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub secret_env: BTreeMap, - /// One-shot standard input, written and closed at start. Plain-pipe - /// processes otherwise read end of file from `/dev/null`; interactive - /// input requires `tty`, whose master is the process's input. + pub command: CommandSpec, + /// Run under a PTY of this size. The PTY master is the input: `input` + /// must then be `none` or `bytes`, and `process/write` is always + /// accepted while the process runs. Requires + /// `capabilities.process.pty`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub stdin: Option, - /// Kill deadline. Absent means the call never kills a running process. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout_ms: Option, - #[serde(default)] - pub tty: bool, + pub pty: Option, } +/// `process/start` response, sent once the process runs. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct StartProcessResponse { - pub process_id: ProcessId, + /// OS pid of the root process, which is also its process group id. + pub pid: u32, } +/// `process/read` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReadProcessParams { pub process_id: ProcessId, - /// Explicit re-read from a sequence number over the retained output. It - /// does not move the daemon's cursor; omit it to read after the last - /// delivered chunk and advance the cursor. + /// Start offset. Absent: the daemon cursor, which this read advances to + /// `next` once its response is written. Present: a non-consuming read + /// that leaves the cursor alone. #[serde(default, skip_serializing_if = "Option::is_none")] - pub after_seq: Option, + pub after: Option, + /// Start at `end - tailBytes` instead (clamped at 0). Non-consuming; + /// excludes `after`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_bytes: Option, - /// Collect output for this long, returning early only when the process - /// exits or `max_bytes` is reached. Absent blocks until exit. + pub tail_bytes: Option, + /// Acknowledge every byte before this offset first: the cursor moves to + /// it (never backwards) whether or not this read's response arrives. A + /// caller that tracks positions reads with `after` and acknowledges + /// with `ack` for exactly-once delivery. #[serde(default, skip_serializing_if = "Option::is_none")] - pub wait_ms: Option, + pub ack: Option, + /// Most `data` bytes to return, at most `limits.process.maxReadBytes`. + /// 0 reads only the state. + pub max_bytes: u64, + /// How long to wait, at most `limits.maxWaitMs`. 0 polls. + pub wait_ms: u64, + pub return_on: ReturnOn, } +/// `process/read` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessOutputChunk { - pub seq: u64, - pub stream: ProcessOutputStream, - pub chunk: ByteChunk, -} - -/// A process still running in the command's group after the root exited. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct LeftoverProcess { - pub pid: u32, - /// Command line when the platform exposes it, otherwise the executable - /// name; may be empty when neither could be read. - #[serde(default, skip_serializing_if = "String::is_empty")] - pub command: String, -} - -/// Why a process that did not exit on its own stopped. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ProcessTermination { - /// The start deadline (`timeoutMs`) expired and the group was killed. - TimedOut, - /// `process/terminate` with `kill` ended the group. - Killed, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReadProcessResponse { - pub chunks: Vec, - pub next_seq: u64, - pub exited: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub exit_code: Option, - pub closed: bool, + pub output: OutputPage, + pub state: ExecState, + /// How the root ended; present once `state` is `exited`, unless `error` + /// explains why it could not be observed. #[serde(default, skip_serializing_if = "Option::is_none")] - pub failure: Option, + pub exit: Option, /// Set when the daemon stopped the process; absent on a natural exit. #[serde(default, skip_serializing_if = "Option::is_none")] - pub termination: Option, - /// OS pid of the root process, which is also its process group id. + pub stopped_by: Option, + /// Infrastructure failure after start (output capture or wait failed). #[serde(default, skip_serializing_if = "Option::is_none")] - pub pid: Option, - /// Bytes dropped from the middle of the retained output since the last - /// cursor read because the retained buffer overflowed. The gap sits at - /// the sequence discontinuity between two returned chunks. - #[serde(default, skip_serializing_if = "is_zero")] - pub omitted_bytes: u64, - /// Processes still running in the command's group after the root - /// exited, sampled when this read was answered. Empty until the exit is - /// observed and on platforms without process enumeration. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub leftover_processes: Vec, -} - -fn is_zero(value: &u64) -> bool { - *value == 0 + pub error: Option, + /// OS pid of the root process. + pub pid: u32, + /// Group members alive after the root exited (`disposition: kept`), + /// sampled when this response was built. Absent while running, when + /// none remain, and on platforms without process enumeration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub leftovers: Option, } +/// `process/write` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct WriteProcessParams { pub process_id: ProcessId, - /// Bytes for the process's input. Requires an open input (a PTY); an - /// absent or empty chunk with `closeStdin: false` is a wait and is - /// accepted whether or not the process has exited. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub chunk: Option, - #[serde(default)] - pub close_stdin: bool, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum WriteProcessStatus { - Accepted, - /// The entry is gone: the process finished and its retention elapsed. - UnknownProcess, - /// Input was given but the process has no open input, or has exited. - StdinClosed, - Starting, + /// Bytes for the process's input pipe or PTY. + #[serde(default, skip_serializing_if = "ByteChunk::is_empty")] + pub data: ByteChunk, + /// Close the input after `data`: end of file on a pipe, `VEOF` on a + /// PTY. Applied only when every byte of `data` was accepted. + #[serde(default, skip_serializing_if = "is_false")] + pub close_input: bool, } +/// `process/write` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct WriteProcessResponse { - pub status: WriteProcessStatus, + /// Leading bytes of `data` queued to the process's writer. Fewer than + /// sent (possibly 0) means the input queue is full: resend the rest + /// later. + pub accepted_bytes: u64, } /// Signal delivered by `process/terminate`. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] pub enum ProcessSignal { - /// `SIGINT` to the process group; changes no daemon state, the next - /// read observes whatever the process did. + /// `SIGINT` to the foreground process group (the terminal's under a + /// PTY, otherwise the process group). Records nothing; the next read + /// shows what the process did. Interrupt, - /// Kill the whole group; the process is recorded as killed. + /// `SIGTERM` to the group, then `SIGKILL` after + /// `limits.process.terminateGraceMs` if members remain. Records + /// `stoppedBy: terminate`. + Terminate, + /// `SIGKILL` to the group. Records `stoppedBy: kill`. Kill, } +/// `process/terminate` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct TerminateProcessParams { pub process_id: ProcessId, pub signal: ProcessSignal, } +/// `process/terminate` response, sent once the signal was delivered (any +/// escalation continues in the background). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct TerminateProcessResponse { - /// True when the process was still running when the signal was sent. + /// True when the root process was still running when the signal was + /// sent. + #[serde(default, skip_serializing_if = "is_false")] pub running: bool, } - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TerminalSize { - pub rows: u16, - pub cols: u16, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ResizeProcessParams { - pub process_id: ProcessId, - pub size: TerminalSize, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ResizeProcessResponse {} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ProcessOutputStream { - Stdout, - Stderr, - Pty, -} diff --git a/crates/environment-protocol/src/data/scan.rs b/crates/environment-protocol/src/data/scan.rs new file mode 100644 index 000000000..b2aad6894 --- /dev/null +++ b/crates/environment-protocol/src/data/scan.rs @@ -0,0 +1,231 @@ +//! `fs/scan`: a small, bounded observation of several roots, for catalogs +//! such as skill and prompt discovery. An observation, not use: it never +//! resets the idle clock. Large trees go through `transfer/*`. +//! +//! Each root is reported on its own, so one bad entry or one failed root +//! never hides the others. + +use serde::{Deserialize, Serialize}; + +use crate::{ + bytes::ByteChunk, + data::fs::{TreeContent, TreeFilter}, + error::ProtocolError, + ids::validate_opaque_id, + paths::{EnvironmentPath, TreePath}, + serde_util::{is_false, is_zero_u32}, +}; + +/// `fs/scan` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanParams { + /// 1 to `limits.filesystem.scan.maxRoots` roots. + pub roots: Vec, + /// Applied below every root; at most `limits.filesystem.scan.maxPatterns` + /// patterns in total. + #[serde(default, skip_serializing_if = "TreeFilter::is_default")] + pub filter: TreeFilter, + /// Return the bytes of every selected file in `data`. + #[serde(default, skip_serializing_if = "is_false")] + pub read_content: bool, + /// Return each selected file's `digest`. + #[serde(default, skip_serializing_if = "is_false")] + pub include_digests: bool, + /// Follow symlinks whose targets stay inside the filesystem root; others + /// are skipped (`symlinkOutsideRoot`). Without it symlinks are skipped + /// (`symlink`). + #[serde(default, skip_serializing_if = "is_false")] + pub follow_symlinks: bool, + /// Caller quotas; each at most its ceiling. + pub limits: ScanLimits, +} + +/// One root of a scan. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanRoot { + pub path: EnvironmentPath, + /// The root's previous `fingerprint`: when the selection is unchanged + /// the result is `unchanged` and carries no entries. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub if_none_match: Option, + /// Overrides `filter.maxDepth` for this root. `0` examines only the + /// root itself: a `complete` result without entries proves it is an + /// existing directory, which is how discovery checks a working + /// directory in the same call. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_depth: Option, +} + +/// Caller quotas of one scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanLimits { + /// Selected entries per root; more fail that root (`payloadTooLarge` + /// with `limitEntries`). + pub max_entries: u32, + /// Larger selected files are skipped (`fileTooLarge`). + pub max_file_bytes: u64, + /// Content bytes returned across all roots, in root order: a root whose + /// content would exceed the rest fails (`payloadTooLarge` with + /// `limitBytes`) and later roots are still scanned. Independently, a + /// root whose encoded result (entries, skips and base64 content) would + /// push the scan's results past + /// [`MAX_RESULT_BYTES`](crate::limits::MAX_RESULT_BYTES) fails the same + /// way. + pub max_total_bytes: u64, + /// Walk time per root; exhausting it fails that root + /// (`deadlineExceeded`). + pub max_duration_ms: u64, +} + +/// `fs/scan` response: one result per requested root, in request order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanResponse { + /// Never empty. + pub roots: Vec, +} + +/// The outcome of one root. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ScanRootResult { + /// Every selected entry, plus what was skipped and why. + Complete { + path: EnvironmentPath, + /// Resolved absolute path of the root when it differs from `path` + /// (the root is a symlink). + #[serde(default, skip_serializing_if = "Option::is_none")] + canonical_path: Option, + fingerprint: ScanFingerprint, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + entries: Vec, + /// The first + /// [`MAX_LISTED_SKIPS`](crate::limits::MAX_LISTED_SKIPS) skipped + /// entries, in walk order. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + skipped: Vec, + /// Total number of skipped entries, listed or not; present whenever + /// it is not zero. + #[serde(default, skip_serializing_if = "is_zero_u32")] + skipped_entry_count: u32, + }, + /// The selection matches `ifNoneMatch`. + Unchanged { + path: EnvironmentPath, + fingerprint: ScanFingerprint, + }, + /// The root does not exist. + Missing { path: EnvironmentPath }, + /// The root could not be scanned; no entries are returned for it. + Failed { + path: EnvironmentPath, + error: ProtocolError, + }, +} + +/// One selected entry below a root. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanEntry { + /// Relative to the root as requested. + pub path: TreePath, + pub content: TreeContent, + /// The file's bytes when `readContent` was set. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +/// An entry a scan left out. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanSkip { + /// Absent when the entry's name is not UTF-8. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + pub reason: ScanSkipReason, +} + +/// Why a scan left an entry out. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ScanSkipReason { + /// Larger than `maxFileBytes`. + FileTooLarge, + /// A symlink, and `followSymlinks` is not set. + Symlink, + /// A symlink whose target leaves the filesystem root, or a link cycle. + SymlinkOutsideRoot, + /// A device, FIFO, socket or another special file. + SpecialFile, + /// The name is not UTF-8. + NonUtf8Name, + /// The entry vanished or could not be read during the walk. + Unreadable, +} + +/// Opaque fingerprint of one root's selection, prefixed `scan:v1:`, at most +/// 255 bytes without control characters. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct ScanFingerprint( + #[cfg_attr( + feature = "schema", + schemars(length(min = 9, max = 255), pattern(SCAN_FINGERPRINT_PATTERN)) + )] + String, +); + +/// JSON Schema pattern of [`ScanFingerprint`]. +pub const SCAN_FINGERPRINT_PATTERN: &str = "^scan:v1:[^\\u0000-\\u001F\\u007F-\\u009F]+$"; + +impl ScanFingerprint { + pub const PREFIX: &'static str = "scan:v1:"; + + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if !value.starts_with(Self::PREFIX) + || value.len() == Self::PREFIX.len() + || validate_opaque_id("ScanFingerprint", &value).is_err() + { + return Err(ScanFingerprintError(value)); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("scan fingerprint must start with scan:v1: and be at most 255 bytes: {0:?}")] +pub struct ScanFingerprintError(pub String); + +impl Serialize for ScanFingerprint { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for ScanFingerprint { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(serde::de::Error::custom) + } +} diff --git a/crates/environment-protocol/src/data/transfer.rs b/crates/environment-protocol/src/data/transfer.rs index cad1f5feb..0b8c130fc 100644 --- a/crates/environment-protocol/src/data/transfer.rs +++ b/crates/environment-protocol/src/data/transfer.rs @@ -1,111 +1,397 @@ -//! Bounded inline filesystem copies. No VFS identities or storage access are required. -use crate::shared::{ByteChunk, EnvironmentPath}; +//! `transfer/*`: one logical tree or file transfer as many bounded calls. +//! +//! No storage or VFS identities cross this boundary: entries are tree paths +//! and content digests. An operation is created by `transfer/begin` and +//! keyed by its caller-chosen `operationId`; retrying any call with the same +//! input is safe. Phases: +//! +//! - **capture** (environment to caller): `scanning` (advance until the +//! inventory is sealed) → `inventory` (page entries, read content) → +//! `complete` after `transfer/commit`. +//! - **materialize** and **writeFile** (caller to environment): +//! `inventory` (append the incoming inventory) → `scanning` (the daemon +//! reads destination metadata and hashes only files whose size matches an +//! incoming entry) → `content` (upload `transfer/listMissing` digests) → +//! `staging` (advance) → `ready` → `complete` after `transfer/commit`, +//! which publishes the staged tree atomically. +//! +//! A failure of the operation's own work (walking, hashing, staging, +//! publishing, a source changing, a digest mismatch, its duration bound) +//! moves it to `failed` with its `error`, and every later call except +//! `transfer/getStatus` and `transfer/abort` answers that error; nothing is +//! committed. A call refused before it acts (invalid params, a phase or +//! offset precondition, an unknown digest, `busy`) leaves the operation +//! unchanged, so a retried or misdirected call cannot fail it. A completed, aborted or failed operation's +//! receipt is answered for `limits.filesystem.transfer.receiptRetentionMs` +//! after it finished, across daemon restarts. A completed capture's +//! inventory stays pageable with `transfer/readInventory` for as long as its +//! receipt, so a retried capture answers the tree it first captured; its +//! content is no longer readable. + use serde::{Deserialize, Serialize}; -/// Hard ceilings for one operation, before base64/JSON framing. -pub const MAX_TRANSFER_BYTES: u64 = 8 * 1024 * 1024; -pub const MAX_TRANSFER_ENTRIES: u32 = 1024; -pub const MAX_TRANSFER_DEPTH: u32 = 32; -pub const MAX_TRANSFER_DURATION_MS: u64 = 30_000; -pub const MAX_TRANSFER_PATH_BYTES: usize = 4096; +use crate::{ + bytes::ByteChunk, + data::fs::{FileVersion, TreeContent, TreeFilter, WriteCondition}, + error::ProtocolError, + ids::TransferOperationId, + paths::{ContentDigest, EnvironmentPath, TreePath}, + serde_util::{is_false, is_zero_u32}, +}; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// What one operation transfers. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum TransferSelection { + /// Read a file or tree out of the environment. + Capture { + source: EnvironmentPath, + #[serde(default, skip_serializing_if = "TreeFilter::is_default")] + filter: TreeFilter, + #[serde(default, skip_serializing_if = "SymlinkPolicy::is_default")] + symlinks: SymlinkPolicy, + #[serde(default, skip_serializing_if = "SpecialFilePolicy::is_default")] + special_files: SpecialFilePolicy, + }, + /// Write a tree into the environment, replacing `destination` atomically + /// at commit. + Materialize { + destination: EnvironmentPath, + #[serde(default, skip_serializing_if = "TransferOnExisting::is_default")] + on_existing: TransferOnExisting, + }, + /// Write exactly one file with `fs/writeFile` semantics: staged, + /// atomic, an existing target keeps its permissions, an in-root symlink + /// at `destination` is followed. + WriteFile { + destination: EnvironmentPath, + #[serde(default, skip_serializing_if = "WriteCondition::is_default")] + create: WriteCondition, + #[serde(default, skip_serializing_if = "Option::is_none")] + if_match: Option, + #[serde(default, skip_serializing_if = "is_false")] + executable: bool, + }, +} + +/// Symlinks met during a capture. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum SymlinkPolicy { + /// Capture a link whose target stays inside the selection as the entry + /// it points to; a link leaving the selection or a cycle fails the + /// operation (`symlinkRejected`). A dangling link inside the selection + /// is left out and reported in `skipped` as `symlink`. + #[default] + ExpandInside, + /// Leave every link out and report it in `skipped`. + Skip, + /// Fail on the first link. + Reject, +} + +impl SymlinkPolicy { + pub fn is_default(&self) -> bool { + *self == Self::ExpandInside + } +} + +/// Devices, FIFOs and sockets met during a capture. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] +pub enum SpecialFilePolicy { + /// Fail the operation (`notRegularFile`). + #[default] + Reject, + /// Leave them out and report them in `skipped`. + Skip, +} + +impl SpecialFilePolicy { + pub fn is_default(&self) -> bool { + *self == Self::Reject + } +} + +/// What a materialize does with an existing destination. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum TransferOnExisting { + /// Refuse at begin (`alreadyExists`). + Error, + /// Replace it atomically at commit. + #[default] + Replace, +} + +impl TransferOnExisting { + pub fn is_default(&self) -> bool { + *self == Self::Replace + } +} + +/// Caller quotas of one operation; each at most its advertised ceiling. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct TransferLimits { - /// Counts the selected root, including an empty directory. + /// Entries, the selected root included. pub max_entries: u32, - /// Selected root has depth zero. + /// The selected root has depth 0. pub max_depth: u32, pub max_file_bytes: u64, pub max_total_bytes: u64, - /// Cooperative deadline checked between filesystem operations and byte chunks. - /// Does not interrupt an in-flight kernel syscall. + /// Encoded inventory size. + pub max_manifest_bytes: u64, + /// From begin to commit. pub max_duration_ms: u64, } +/// One inventory entry. Parents precede children. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "camelCase")] -pub enum TransferContent { - Directory, - File { data: ByteChunk, executable: bool }, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct InventoryEntry { + /// Relative to the selected root; `.` is the root itself. + pub path: TreePath, + /// A file entry always carries its `digest`. + pub content: TreeContent, } +/// `transfer/begin` params. Idempotent: a repeat with identical input +/// returns the operation's status; different input under the same id is +/// `failedPrecondition` (`operationIdInUse`). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TransferEntry { - /// Root is "". Others are strict relative '/' paths; parents must precede children. - /// UTF-8 names only; no empty, '.', '..', NUL or backslash components. - pub path: String, - pub content: TransferContent, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BeginTransferParams { + pub operation_id: TransferOperationId, + pub selection: TransferSelection, + pub limits: TransferLimits, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum TransferOnExisting { - Error, - #[default] - Replace, +/// Params of `transfer/advance`, `transfer/getStatus`, `transfer/commit` +/// and `transfer/abort`: each names only the operation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TransferOperationParams { + pub operation_id: TransferOperationId, } +/// The operation's status: the response of `transfer/begin`, +/// `transfer/advance`, `transfer/getStatus`, `transfer/append`, +/// `transfer/commit` and `transfer/abort`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TransferStatus { + pub operation_id: TransferOperationId, + pub phase: TransferPhase, + /// Entries in the inventory so far. + pub entry_count: u32, + /// File bytes in the inventory so far. + pub total_bytes: u64, + /// Content bytes moved over the wire. + pub transferred_bytes: u64, + /// Content bytes satisfied from files already at the destination. + pub reused_bytes: u64, + /// The first + /// [`MAX_LISTED_SKIPS`](crate::limits::MAX_LISTED_SKIPS) entries a + /// capture left out under its policies. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub skipped: Vec, + /// Total number of entries left out, listed or not; present whenever it + /// is not zero. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_entry_count: u32, + /// Why the operation failed; present exactly when `phase` is `failed`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + /// When `maxDurationMs` runs out for an active operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deadline_at_ms: Option, + /// When the receipt of a finished (complete, aborted or failed) + /// operation stops being answered; afterwards calls are `notFound` + /// (`expired`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub receipt_expires_at_ms: Option, +} + +/// Where an operation stands; see the module docs for the order per +/// selection kind. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct MaterializeParams { - pub destination: EnvironmentPath, - pub entries: Vec, - pub limits: TransferLimits, - #[serde(default)] - pub on_existing: TransferOnExisting, +pub enum TransferPhase { + /// The daemon walks a tree: the capture source, or the destination of a + /// materialize. Advance until the phase changes. + Scanning, + /// Capture: the inventory is sealed and pageable. Materialize: waiting + /// for appended entries. + Inventory, + /// Waiting for the content of `transfer/listMissing` digests. + Content, + /// The daemon assembles the staged tree. Advance until `ready`. + Staging, + /// Ready for `transfer/commit`. + Ready, + Complete, + Aborted, + /// Stopped by an error; see `error`. + Failed, } +impl TransferPhase { + pub fn is_finished(self) -> bool { + matches!(self, Self::Complete | Self::Aborted | Self::Failed) + } +} + +/// An entry a capture left out. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TransferSkip { + /// Absent when the entry's name is not UTF-8. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + pub reason: TransferSkipReason, +} + +/// Why a capture left an entry out. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct MaterializeResponse { - pub destination: EnvironmentPath, - pub entries: u32, - pub bytes: u64, - /// Reserved for compatibility with early daemons. Current daemons clean retired - /// staging trees asynchronously and return None. - pub retired_directory: Option, +pub enum TransferSkipReason { + /// `symlinks: skip`, or a dangling link under `symlinks: expandInside`. + Symlink, + /// `specialFiles: skip`. + SpecialFile, + /// The name is not UTF-8; such entries are always skipped. + NonUtf8Name, } +/// `transfer/readInventory` params: page a sealed inventory by index. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CaptureParams { - pub source: EnvironmentPath, - pub limits: TransferLimits, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadTransferInventoryParams { + pub operation_id: TransferOperationId, + /// Index of the first entry. + pub index: u32, + /// 1 to `limits.filesystem.transfer.maxInventoryPageEntries`. + pub limit: u32, } +/// `transfer/readInventory` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CaptureResponse { - pub source: EnvironmentPath, - /// Complete selected tree only; failures never return a partial inventory. - pub entries: Vec, - pub bytes: u64, -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn binary_inventory_roundtrips_and_legacy_capabilities_default_false() { - let entry = TransferEntry { - path: "".into(), - content: TransferContent::File { - data: ByteChunk::from(vec![0, 255, 128]), - executable: true, - }, - }; - let json = serde_json::to_value(&entry).unwrap(); - assert_eq!(json["content"]["data"], "AP+A"); - assert_eq!( - serde_json::from_value::(json).unwrap(), - entry - ); - let capabilities: crate::shared::EnvironmentCapabilities = - serde_json::from_str("{}").unwrap(); - assert!(!capabilities.filesystem_capture); - assert!(!capabilities.filesystem_materialize); - assert!(serde_json::from_str::(r#"{"source":"."}"#).is_err()); - } +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadTransferInventoryResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub entries: Vec, + /// Index of the next page; absent after the last entry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_index: Option, +} + +/// `transfer/append` params: add incoming inventory entries in order. +/// Retrying an identical page is safe. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AppendTransferInventoryParams { + pub operation_id: TransferOperationId, + /// Index of the page's first entry: the number of entries appended + /// before it. + pub index: u32, + /// At most `limits.filesystem.transfer.maxInventoryPageEntries`. + pub entries: Vec, + /// This page completes the inventory. + #[serde(default, skip_serializing_if = "is_false")] + pub last: bool, +} + +/// `transfer/listMissing` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListTransferMissingParams { + pub operation_id: TransferOperationId, + /// Most digests to return, 1 to + /// `limits.filesystem.transfer.maxInventoryPageEntries`. + pub limit: u32, +} + +/// `transfer/listMissing` response: digests whose content the daemon still +/// needs. The set shrinks as uploads complete; call again rather than +/// paging. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListTransferMissingResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub digests: Vec, + /// Missing digests beyond this page. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub remaining_count: u32, +} + +/// `transfer/read` params: read a captured file's content by digest. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadTransferContentParams { + pub operation_id: TransferOperationId, + pub digest: ContentDigest, + pub offset: u64, + /// 1 to `limits.filesystem.transfer.maxContentChunkBytes`. + pub max_bytes: u64, +} + +/// `transfer/read` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadTransferContentResponse { + pub data: ByteChunk, + /// Bytes exist after `offset` plus the length of `data`, as for + /// `fs/readFile`. + #[serde(default, skip_serializing_if = "is_false")] + pub truncated: bool, +} + +/// `transfer/write` params: upload content for a missing digest. Chunks for +/// one digest arrive in order; an identical chunk may be retried. +/// Different digests may be uploaded concurrently. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WriteTransferContentParams { + pub operation_id: TransferOperationId, + pub digest: ContentDigest, + /// Must equal the digest's next expected offset. + pub offset: u64, + /// At most `limits.filesystem.transfer.maxContentChunkBytes`. + pub data: ByteChunk, +} + +/// `transfer/write` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WriteTransferContentResponse { + /// The digest's next expected offset; equal to its size once complete + /// and verified. + pub next_offset: u64, } diff --git a/crates/environment-protocol/src/data/transfer_session.rs b/crates/environment-protocol/src/data/transfer_session.rs deleted file mode 100644 index a1d93ad67..000000000 --- a/crates/environment-protocol/src/data/transfer_session.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! One logical transfer, many bounded exchanges. No storage or VFS identities cross this boundary. -use super::{ - inventory::{InventoryEntry, InventoryLimits}, - transfer::TransferOnExisting, -}; -use crate::shared::{ByteChunk, EnvironmentPath}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "direction", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum TransferSelection { - Capture { - source: EnvironmentPath, - }, - Materialize { - destination: EnvironmentPath, - #[serde(default)] - on_existing: TransferOnExisting, - }, - /// Write exactly one file, rejecting directories and preserving existing - /// file permissions. Uses the same bounded chunks and durable receipts. - /// A separate direction makes older endpoints reject unsupported semantics. - WriteFile { - destination: EnvironmentPath, - }, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "action", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum TransferRequest { - /// Idempotent until operation expiry, including completed receipts after daemon restart. - /// IDs must not be reused for different input. - Begin { - operation_id: String, - selection: TransferSelection, - #[serde(default)] - limits: InventoryLimits, - }, - /// Advances bounded scan/staging work. Inspect status until ready, then commit. - Advance { - operation_id: String, - }, - Inventory { - operation_id: String, - offset: u32, - }, - /// Parent-before-child inventory pages. Retrying the identical page is safe. - Append { - operation_id: String, - offset: u32, - entries: Vec, - last: bool, - }, - Missing { - operation_id: String, - offset: u32, - }, - Read { - operation_id: String, - digest: String, - offset: u64, - }, - /// Sequential chunks; an identical chunk may be retried. Only complete verified files are reusable. - Write { - operation_id: String, - digest: String, - offset: u64, - data: ByteChunk, - }, - Commit { - operation_id: String, - }, - Status { - operation_id: String, - }, - Abort { - operation_id: String, - }, -} -impl TransferRequest { - pub fn operation_id(&self) -> &str { - match self { - Self::Begin { operation_id, .. } - | Self::Advance { operation_id } - | Self::Inventory { operation_id, .. } - | Self::Append { operation_id, .. } - | Self::Missing { operation_id, .. } - | Self::Read { operation_id, .. } - | Self::Write { operation_id, .. } - | Self::Commit { operation_id } - | Self::Status { operation_id } - | Self::Abort { operation_id } => operation_id, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub enum TransferPhase { - Scanning, - Inventory, - Content, - Staging, - Ready, - Complete, - Aborted, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub struct TransferStatus { - pub operation_id: String, - pub phase: TransferPhase, - pub entries: u32, - pub bytes: u64, - pub transferred_bytes: u64, - pub reused_bytes: u64, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "result", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum TransferResponse { - Status(TransferStatus), - Inventory { - entries: Vec, - next_offset: Option, - }, - Missing { - digests: Vec, - next_offset: Option, - }, - Chunk { - data: ByteChunk, - eof: bool, - }, - Written { - next_offset: u64, - }, -} diff --git a/crates/environment-protocol/src/error.rs b/crates/environment-protocol/src/error.rs index f2e29b069..8cb49c8d1 100644 --- a/crates/environment-protocol/src/error.rs +++ b/crates/environment-protocol/src/error.rs @@ -1,38 +1,858 @@ -//! Shared error payloads for environment protocol methods. +//! The one error vocabulary of every plane. +//! +//! An error travels as a JSON-RPC error object whose `code` is a fixed +//! function of its kind and whose `data` carries the kind and its typed +//! fields: +//! +//! ```json +//! {"code": -32004, "message": "unknown process id: p1", "data": {"kind": "notFound", "reason": "expired"}} +//! ``` +//! +//! `data.kind` is authoritative: receivers branch on it, never on the code +//! or the message. Retryability is a property of the kind +//! ([`ProtocolErrorKind::is_retryable`]) and is never sent. The same error +//! object is embedded in results that report per-entry failures (`job/read` +//! entries, `fs/scan` roots, a failed transfer's status). + +use std::fmt; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use thiserror::Error; -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +use crate::limits::MAX_ERROR_MESSAGE_BYTES; + +/// Closed set of error kinds. Each kind has one meaning and one code. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ProtocolErrorKind { + /// The frame is not JSON. Answered with a null id. + ParseError, + /// The frame is JSON but not a valid request: wrong `jsonrpc`, a batch, + /// a missing or malformed member. A second `initialize` is also + /// `invalidRequest`. + InvalidRequest, + /// No method of that name exists on this plane. + MethodNotFound, + /// The params do not decode, or a value is out of range or above an + /// advertised ceiling. + InvalidParams, + /// A method other than `initialize` arrived before a successful + /// `initialize`. + NotInitialized, + /// The peer stated another protocol version. + VersionMismatch, + /// The named handle, path or target does not exist. + NotFound, + /// The thing to create already exists. + AlreadyExists, + /// The request is well-formed but the current state forbids it; `reason` + /// says which state. + FailedPrecondition, + /// The caller may not do this: confinement, credentials, signatures. + PermissionDenied, + /// The endpoint lacks the capability the request needs; an endpoint + /// never silently ignores a parameter it cannot honour. + CapabilityUnavailable, + /// A limit on concurrent work was reached; retry later. + Busy, + /// A server-side duration bound ran out. + DeadlineExceeded, + /// A credential or token is past its validity. + Expired, + /// The operation stopped before completing: cancelled, or its input + /// changed underneath it. Retrying from the start may succeed. + Aborted, + /// A message, payload or result is larger than allowed. + PayloadTooLarge, + /// A process could not be started or supervised. + ProcessFailed, + /// A dependency of the server is temporarily unreachable. + Unavailable, + /// A defect or unexpected failure inside the server. + Internal, +} + +impl ProtocolErrorKind { + /// The JSON-RPC `code` of this kind. Framing kinds use the reserved + /// codes; domain kinds reuse the public API's numbers where the meaning + /// matches (`permissionDenied`, `notFound`, `alreadyExists`, + /// `payloadTooLarge`). + pub fn code(self) -> i32 { + match self { + Self::ParseError => -32700, + Self::InvalidRequest => -32600, + Self::MethodNotFound => -32601, + Self::InvalidParams => -32602, + Self::Internal => -32603, + Self::NotInitialized => -32002, + Self::PermissionDenied => -32003, + Self::NotFound => -32004, + Self::VersionMismatch => -32005, + Self::CapabilityUnavailable => -32006, + Self::FailedPrecondition => -32007, + Self::Busy => -32008, + Self::AlreadyExists => -32009, + Self::PayloadTooLarge => -32013, + Self::DeadlineExceeded => -32015, + Self::Expired => -32016, + Self::Aborted => -32017, + Self::ProcessFailed => -32018, + Self::Unavailable => -32019, + } + } + + /// The kind a code stands for, for error objects that carry no `data` + /// (a non-conforming peer or an intermediary). + pub fn from_code(code: i64) -> Option { + Self::ALL + .into_iter() + .find(|kind| i64::from(kind.code()) == code) + } + + /// True when retrying the same request later can succeed without any + /// change: `busy`, `unavailable` and `aborted`. + pub fn is_retryable(self) -> bool { + matches!(self, Self::Busy | Self::Unavailable | Self::Aborted) + } + + pub fn as_str(self) -> &'static str { + match self { + Self::ParseError => "parseError", + Self::InvalidRequest => "invalidRequest", + Self::MethodNotFound => "methodNotFound", + Self::InvalidParams => "invalidParams", + Self::NotInitialized => "notInitialized", + Self::VersionMismatch => "versionMismatch", + Self::NotFound => "notFound", + Self::AlreadyExists => "alreadyExists", + Self::FailedPrecondition => "failedPrecondition", + Self::PermissionDenied => "permissionDenied", + Self::CapabilityUnavailable => "capabilityUnavailable", + Self::Busy => "busy", + Self::DeadlineExceeded => "deadlineExceeded", + Self::Expired => "expired", + Self::Aborted => "aborted", + Self::PayloadTooLarge => "payloadTooLarge", + Self::ProcessFailed => "processFailed", + Self::Unavailable => "unavailable", + Self::Internal => "internal", + } + } + + pub const ALL: [Self; 19] = [ + Self::ParseError, + Self::InvalidRequest, + Self::MethodNotFound, + Self::InvalidParams, + Self::NotInitialized, + Self::VersionMismatch, + Self::NotFound, + Self::AlreadyExists, + Self::FailedPrecondition, + Self::PermissionDenied, + Self::CapabilityUnavailable, + Self::Busy, + Self::DeadlineExceeded, + Self::Expired, + Self::Aborted, + Self::PayloadTooLarge, + Self::ProcessFailed, + Self::Unavailable, + Self::Internal, + ]; +} + +impl fmt::Display for ProtocolErrorKind { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Why a handle, path or identity was not found. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum NotFoundReason { + /// Its retention window ended; a tombstone remembers it. + Expired, + /// It belonged to an earlier daemon instance (see `daemonInstanceId`). + DaemonRestarted, + /// A daemon identity the gateway has never admitted presented no + /// registration key. + UnknownDaemon, +} + +/// The state that forbids a `failedPrecondition` request. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum PreconditionReason { + /// The process has no open input: started without `pipe` or `pty`, + /// input already closed, or exited. + InputClosed, + /// The file's current `version` differs from `ifMatch`. + IfMatchFailed, + /// `create: ifAbsent` met an existing file, or `create: ifPresent` a + /// missing one. + CreateCondition, + /// The directory is not empty and `recursive` was not set. + NotEmpty, + /// The path names a directory where a file is required. + IsDirectory, + /// A path component or the path names a non-directory where a directory + /// is required. + NotDirectory, + /// The path names a device, FIFO, socket or other non-regular file. + NotRegularFile, + /// A symlink chain loops. + SymlinkLoop, + /// A capture met a symlink that leaves the selection, or one the + /// selection's symlink policy rejects. + SymlinkRejected, + /// The filesystem root itself cannot be replaced or removed. + FilesystemRoot, + /// Rename across filesystems. + CrossDevice, + /// No space left on the device. + NoSpace, + /// A job id is in use by a job with another `requestId` or spec; choose + /// a new job id. + JobIdInUse, + /// A transfer `operationId` is in use for different input. + OperationIdInUse, + /// The transfer action is not valid in the operation's current phase. + TransferPhase, + /// A transfer write's `offset` is not the digest's next expected offset. + TransferOffset, + /// Uploaded bytes do not hash to their digest. + DigestMismatch, + /// The environment is closed; its identity or target is spent. + EnvironmentClosed, + /// The daemon identity is bound to another universe. + IdentityInUse, + /// The target belongs to another owner, or (`target/adopt`) the owner + /// already holds a different target. + OwnerMismatch, + /// The template is deprecated and creates no new targets. + TemplateDeprecated, + /// `ingress/ensure` on a target whose template has no public ingress. + IngressUnsupported, + /// A power change on a target whose `phase` is not `active` + /// (provisioning, closing, closed or failed). + TargetNotActive, + /// `target/adopt`: the source target exists but the provider cannot take + /// it over, for example an instance of a kind it does not manage. + NotAdoptable, +} + +/// Why a `permissionDenied` request was refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum PermissionReason { + /// The path, or a symlink it passes through, resolves outside the + /// daemon's filesystem root. + OutsideRoot, + /// The path names or contains the daemon's state directory. + DaemonState, + /// The operating system refused access. + FilePermission, + /// The registration signature does not verify. + InvalidSignature, + /// The registration key is unknown. + InvalidRegistrationKey, + /// The registration key was revoked. + RegistrationKeyRevoked, +} + +/// What limit a `busy` request hit. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum BusyReason { + /// The connection's concurrent request limit. + RequestLimit, + /// The daemon's limit on active transfer operations. + TransferCapacity, + /// A rate limit on the caller's identity. + RateLimited, + /// A registration key's limit on active environments. + CapacityExhausted, +} + +/// Why an operation was `aborted`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct EnvironmentProtocolError { - pub code: EnvironmentProtocolErrorCode, +pub enum AbortReason { + /// `$/cancelRequest` named the request. + Cancelled, + /// A file or directory changed while it was being read. + SourceChanged, +} + +/// A capability a request needed and the endpoint lacks. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum Capability { + /// The `filesystem` family. + Filesystem, + /// `capabilities.filesystem.write`: the filesystem is writable. + FilesystemWrite, + /// The `process` family. + Process, + /// `capabilities.process.pty`. + ProcessPty, + /// The `job` family. + Job, + /// The controller's `adoptTarget` capability. + AdoptTarget, + /// The controller's `ingress` capability. + Ingress, +} + +/// The `data` member of an error object: the kind and its typed fields. +/// +/// Kinds without fields are empty struct variants rather than unit +/// variants: serde does not apply `deny_unknown_fields` to the unit +/// variants of an internally tagged enum, and every kind must refuse +/// unknown members. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ErrorData { + ParseError {}, + InvalidRequest {}, + MethodNotFound {}, + InvalidParams { + /// Wire path of the offending field, such as `limits.maxDurationMs`. + #[serde(default, skip_serializing_if = "Option::is_none")] + field: Option, + /// The advertised ceiling the value exceeded. + #[serde(default, skip_serializing_if = "Option::is_none")] + ceiling: Option, + }, + NotInitialized {}, + VersionMismatch { + /// The version of the peer that sent this error. + expected: u32, + /// The version the requester stated. + received: u32, + }, + NotFound { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + }, + AlreadyExists {}, + FailedPrecondition { + reason: PreconditionReason, + }, + PermissionDenied { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + }, + CapabilityUnavailable { + capability: Capability, + }, + Busy { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + /// Earliest useful retry, when the server knows it. + #[serde(default, skip_serializing_if = "Option::is_none")] + retry_after_ms: Option, + }, + DeadlineExceeded {}, + Expired {}, + Aborted { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + }, + PayloadTooLarge { + /// The byte limit the payload exceeded. + #[serde(default, skip_serializing_if = "Option::is_none")] + limit_bytes: Option, + /// The entry-count limit the result exceeded. + #[serde(default, skip_serializing_if = "Option::is_none")] + limit_entries: Option, + }, + ProcessFailed {}, + Unavailable { + #[serde(default, skip_serializing_if = "Option::is_none")] + retry_after_ms: Option, + }, + Internal {}, +} + +impl ErrorData { + pub fn kind(&self) -> ProtocolErrorKind { + match self { + Self::ParseError {} => ProtocolErrorKind::ParseError, + Self::InvalidRequest {} => ProtocolErrorKind::InvalidRequest, + Self::MethodNotFound {} => ProtocolErrorKind::MethodNotFound, + Self::InvalidParams { .. } => ProtocolErrorKind::InvalidParams, + Self::NotInitialized {} => ProtocolErrorKind::NotInitialized, + Self::VersionMismatch { .. } => ProtocolErrorKind::VersionMismatch, + Self::NotFound { .. } => ProtocolErrorKind::NotFound, + Self::AlreadyExists {} => ProtocolErrorKind::AlreadyExists, + Self::FailedPrecondition { .. } => ProtocolErrorKind::FailedPrecondition, + Self::PermissionDenied { .. } => ProtocolErrorKind::PermissionDenied, + Self::CapabilityUnavailable { .. } => ProtocolErrorKind::CapabilityUnavailable, + Self::Busy { .. } => ProtocolErrorKind::Busy, + Self::DeadlineExceeded {} => ProtocolErrorKind::DeadlineExceeded, + Self::Expired {} => ProtocolErrorKind::Expired, + Self::Aborted { .. } => ProtocolErrorKind::Aborted, + Self::PayloadTooLarge { .. } => ProtocolErrorKind::PayloadTooLarge, + Self::ProcessFailed {} => ProtocolErrorKind::ProcessFailed, + Self::Unavailable { .. } => ProtocolErrorKind::Unavailable, + Self::Internal {} => ProtocolErrorKind::Internal, + } + } + + /// The fieldless data of a kind, for kinds whose fields are all + /// optional. Kinds with required fields get their typed constructor on + /// [`ProtocolError`] instead. + fn bare(kind: ProtocolErrorKind) -> Option { + Some(match kind { + ProtocolErrorKind::ParseError => Self::ParseError {}, + ProtocolErrorKind::InvalidRequest => Self::InvalidRequest {}, + ProtocolErrorKind::MethodNotFound => Self::MethodNotFound {}, + ProtocolErrorKind::InvalidParams => Self::InvalidParams { + field: None, + ceiling: None, + }, + ProtocolErrorKind::NotInitialized => Self::NotInitialized {}, + ProtocolErrorKind::NotFound => Self::NotFound { reason: None }, + ProtocolErrorKind::AlreadyExists => Self::AlreadyExists {}, + ProtocolErrorKind::PermissionDenied => Self::PermissionDenied { reason: None }, + ProtocolErrorKind::Busy => Self::Busy { + reason: None, + retry_after_ms: None, + }, + ProtocolErrorKind::DeadlineExceeded => Self::DeadlineExceeded {}, + ProtocolErrorKind::Expired => Self::Expired {}, + ProtocolErrorKind::Aborted => Self::Aborted { reason: None }, + ProtocolErrorKind::ProcessFailed => Self::ProcessFailed {}, + ProtocolErrorKind::Unavailable => Self::Unavailable { + retry_after_ms: None, + }, + ProtocolErrorKind::Internal => Self::Internal {}, + ProtocolErrorKind::PayloadTooLarge => Self::PayloadTooLarge { + limit_bytes: None, + limit_entries: None, + }, + ProtocolErrorKind::VersionMismatch + | ProtocolErrorKind::FailedPrecondition + | ProtocolErrorKind::CapabilityUnavailable => return None, + }) + } +} + +/// A JSON-RPC error object as it appears on the wire. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ErrorObject { + /// Fixed by `data.kind`; see [`ProtocolErrorKind::code`]. + pub code: i32, + /// Human-readable description. Never parsed. pub message: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub details: Option, + pub data: ErrorData, } -impl EnvironmentProtocolError { - pub fn new(code: EnvironmentProtocolErrorCode, message: impl Into) -> Self { +/// A protocol error: one kind, its typed data and a message. +/// +/// Serializes as an [`ErrorObject`]. Displays as `{kind}: {message}`. +#[derive(Clone, Debug, PartialEq, Eq, Error, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "schema", schemars(with = "ErrorObject"))] +#[serde(from = "ErrorObject", into = "ErrorObject")] +#[error("{}: {message}", self.kind())] +pub struct ProtocolError { + pub data: ErrorData, + pub message: String, +} + +impl From for ProtocolError { + /// `data.kind` is authoritative; `code` is not consulted. + fn from(object: ErrorObject) -> Self { Self { - code, - message: message.into(), - details: None, + data: object.data, + message: object.message, } } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum EnvironmentProtocolErrorCode { - InvalidRequest, - Unauthorized, - Forbidden, - NotFound, - Conflict, - Unsupported, - CapabilityUnavailable, - ProcessFailed, - Timeout, - TransportClosed, - Internal, +impl From for ErrorObject { + fn from(error: ProtocolError) -> Self { + Self { + code: error.code(), + message: error.message, + data: error.data, + } + } +} + +impl ProtocolError { + /// An error of this kind. A message longer than + /// [`MAX_ERROR_MESSAGE_BYTES`] is cut on a character boundary, so errors + /// embedded in results stay bounded. + pub fn new(data: ErrorData, message: impl Into) -> Self { + let mut message = message.into(); + let max = MAX_ERROR_MESSAGE_BYTES as usize; + if message.len() > max { + let mut end = max; + while !message.is_char_boundary(end) { + end -= 1; + } + message.truncate(end); + } + Self { data, message } + } + + pub fn kind(&self) -> ProtocolErrorKind { + self.data.kind() + } + + pub fn code(&self) -> i32 { + self.kind().code() + } + + pub fn is_retryable(&self) -> bool { + self.kind().is_retryable() + } + + /// Decode an error object tolerantly: a conforming object keeps its + /// typed data; an object without valid `data` (from a non-conforming + /// peer or an intermediary) maps its code to a kind, falling back to + /// `internal`. + pub fn from_error_value(value: &Value) -> Self { + if let Ok(object) = serde_json::from_value::(value.clone()) { + return object.into(); + } + let message = value + .get("message") + .and_then(Value::as_str) + .unwrap_or("JSON-RPC error") + .to_owned(); + let data = value + .get("code") + .and_then(Value::as_i64) + .and_then(ProtocolErrorKind::from_code) + .and_then(ErrorData::bare) + .unwrap_or(ErrorData::Internal {}); + Self { data, message } + } + + pub fn parse_error(message: impl Into) -> Self { + Self::new(ErrorData::ParseError {}, message) + } + + pub fn invalid_request(message: impl Into) -> Self { + Self::new(ErrorData::InvalidRequest {}, message) + } + + pub fn method_not_found(method: &str) -> Self { + Self::new( + ErrorData::MethodNotFound {}, + format!("method not found: {method}"), + ) + } + + pub fn invalid_params(message: impl Into) -> Self { + Self::new( + ErrorData::InvalidParams { + field: None, + ceiling: None, + }, + message, + ) + } + + /// `invalidParams` naming the offending field by its wire path. + pub fn invalid_field(field: impl Into, message: impl Into) -> Self { + Self::new( + ErrorData::InvalidParams { + field: Some(field.into()), + ceiling: None, + }, + message, + ) + } + + /// `invalidParams` for a value above an advertised ceiling. + pub fn above_ceiling(field: impl Into, value: u64, ceiling: u64) -> Self { + let field = field.into(); + let message = format!("{field} is {value}; the ceiling is {ceiling}"); + Self::new( + ErrorData::InvalidParams { + field: Some(field), + ceiling: Some(ceiling), + }, + message, + ) + } + + pub fn not_initialized() -> Self { + Self::new( + ErrorData::NotInitialized {}, + "initialize must be the first request on a connection", + ) + } + + pub fn version_mismatch(expected: u32, received: u32) -> Self { + Self::new( + ErrorData::VersionMismatch { expected, received }, + format!("protocol version {received} is not supported; this peer speaks {expected}"), + ) + } + + pub fn not_found(message: impl Into) -> Self { + Self::new(ErrorData::NotFound { reason: None }, message) + } + + pub fn not_found_because(reason: NotFoundReason, message: impl Into) -> Self { + Self::new( + ErrorData::NotFound { + reason: Some(reason), + }, + message, + ) + } + + pub fn already_exists(message: impl Into) -> Self { + Self::new(ErrorData::AlreadyExists {}, message) + } + + pub fn failed_precondition(reason: PreconditionReason, message: impl Into) -> Self { + Self::new(ErrorData::FailedPrecondition { reason }, message) + } + + pub fn permission_denied(reason: Option, message: impl Into) -> Self { + Self::new(ErrorData::PermissionDenied { reason }, message) + } + + pub fn capability_unavailable(capability: Capability, message: impl Into) -> Self { + Self::new(ErrorData::CapabilityUnavailable { capability }, message) + } + + pub fn busy( + reason: Option, + retry_after_ms: Option, + message: impl Into, + ) -> Self { + Self::new( + ErrorData::Busy { + reason, + retry_after_ms, + }, + message, + ) + } + + pub fn deadline_exceeded(message: impl Into) -> Self { + Self::new(ErrorData::DeadlineExceeded {}, message) + } + + pub fn expired(message: impl Into) -> Self { + Self::new(ErrorData::Expired {}, message) + } + + pub fn aborted(reason: Option, message: impl Into) -> Self { + Self::new(ErrorData::Aborted { reason }, message) + } + + /// `aborted` for a request named by `$/cancelRequest`. + pub fn cancelled() -> Self { + Self::aborted(Some(AbortReason::Cancelled), "request cancelled") + } + + pub fn payload_too_large(limit_bytes: u64, message: impl Into) -> Self { + Self::new( + ErrorData::PayloadTooLarge { + limit_bytes: Some(limit_bytes), + limit_entries: None, + }, + message, + ) + } + + /// `payloadTooLarge` for a result with more entries than allowed. + pub fn too_many_entries(limit_entries: u32, message: impl Into) -> Self { + Self::new( + ErrorData::PayloadTooLarge { + limit_bytes: None, + limit_entries: Some(limit_entries), + }, + message, + ) + } + + pub fn process_failed(message: impl Into) -> Self { + Self::new(ErrorData::ProcessFailed {}, message) + } + + pub fn unavailable(message: impl Into) -> Self { + Self::new( + ErrorData::Unavailable { + retry_after_ms: None, + }, + message, + ) + } + + pub fn internal(message: impl Into) -> Self { + Self::new(ErrorData::Internal {}, message) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn errors_serialize_as_json_rpc_error_objects_with_the_kind_code() { + let error = ProtocolError::not_found_because(NotFoundReason::Expired, "unknown process"); + let value = serde_json::to_value(&error).unwrap(); + assert_eq!( + value, + json!({"code": -32004, "message": "unknown process", "data": {"kind": "notFound", "reason": "expired"}}) + ); + assert_eq!( + serde_json::from_value::(value).unwrap(), + error + ); + assert_eq!(error.to_string(), "notFound: unknown process"); + } + + #[test] + fn every_kind_has_a_distinct_code_that_maps_back() { + let mut codes = std::collections::BTreeSet::new(); + for kind in ProtocolErrorKind::ALL { + assert!(codes.insert(kind.code()), "duplicate code for {kind}"); + assert_eq!(ProtocolErrorKind::from_code(kind.code().into()), Some(kind)); + } + } + + #[test] + fn retryability_follows_the_kind() { + let retryable: Vec<_> = ProtocolErrorKind::ALL + .into_iter() + .filter(|kind| kind.is_retryable()) + .collect(); + assert_eq!( + retryable, + [ + ProtocolErrorKind::Busy, + ProtocolErrorKind::Aborted, + ProtocolErrorKind::Unavailable + ] + ); + } + + #[test] + fn foreign_error_objects_map_reserved_codes_and_fall_back_to_internal() { + let parsed = ProtocolError::from_error_value(&json!({"code": -32601, "message": "nope"})); + assert_eq!(parsed.kind(), ProtocolErrorKind::MethodNotFound); + let parsed = ProtocolError::from_error_value(&json!({"code": "weird"})); + assert_eq!(parsed.kind(), ProtocolErrorKind::Internal); + assert_eq!(parsed.message, "JSON-RPC error"); + } + + #[test] + fn unknown_data_members_are_rejected() { + let value = + json!({"code": -32004, "message": "m", "data": {"kind": "notFound", "extra": 1}}); + assert!(serde_json::from_value::(value).is_err()); + } + + /// One error of every kind, with every optional field set where the kind + /// has one. + fn one_of_each_kind() -> Vec { + vec![ + ProtocolError::parse_error("m"), + ProtocolError::invalid_request("m"), + ProtocolError::method_not_found("x/y"), + ProtocolError::above_ceiling("waitMs", 2, 1), + ProtocolError::not_initialized(), + ProtocolError::version_mismatch(3, 2), + ProtocolError::not_found_because(NotFoundReason::DaemonRestarted, "m"), + ProtocolError::already_exists("m"), + ProtocolError::failed_precondition(PreconditionReason::TargetNotActive, "m"), + ProtocolError::permission_denied(Some(PermissionReason::OutsideRoot), "m"), + ProtocolError::capability_unavailable(Capability::ProcessPty, "m"), + ProtocolError::busy(Some(BusyReason::RateLimited), Some(500), "m"), + ProtocolError::deadline_exceeded("m"), + ProtocolError::expired("m"), + ProtocolError::aborted(Some(AbortReason::SourceChanged), "m"), + ProtocolError::new( + ErrorData::PayloadTooLarge { + limit_bytes: Some(1), + limit_entries: Some(2), + }, + "m", + ), + ProtocolError::process_failed("m"), + ProtocolError::new( + ErrorData::Unavailable { + retry_after_ms: Some(10), + }, + "m", + ), + ProtocolError::internal("m"), + ] + } + + #[test] + fn every_kind_round_trips_with_its_code() { + let errors = one_of_each_kind(); + let kinds: Vec<_> = errors.iter().map(ProtocolError::kind).collect(); + assert_eq!(kinds, ProtocolErrorKind::ALL); + for error in errors { + let value = serde_json::to_value(&error).unwrap(); + assert_eq!(value["code"], json!(error.code())); + assert_eq!(value["data"]["kind"], json!(error.kind().as_str())); + assert_eq!( + serde_json::from_value::(value).unwrap(), + error + ); + } + } + + #[test] + fn every_kind_rejects_unknown_data_members() { + for error in one_of_each_kind() { + let mut value = serde_json::to_value(&error).unwrap(); + value["data"]["unexpectedMember"] = json!(1); + assert!( + serde_json::from_value::(value).is_err(), + "{} accepts an unknown member", + error.kind() + ); + } + } + + #[test] + fn fieldless_kinds_serialize_as_the_kind_alone() { + assert_eq!( + serde_json::to_value(ErrorData::NotInitialized {}).unwrap(), + json!({"kind": "notInitialized"}) + ); + } + + #[test] + fn long_messages_are_cut_on_a_character_boundary() { + let error = ProtocolError::internal("\u{e9}".repeat(MAX_ERROR_MESSAGE_BYTES as usize)); + assert!(error.message.len() <= MAX_ERROR_MESSAGE_BYTES as usize); + assert!(error.message.len() > MAX_ERROR_MESSAGE_BYTES as usize - 2); + } } diff --git a/crates/environment-protocol/src/gateway.rs b/crates/environment-protocol/src/gateway.rs deleted file mode 100644 index 93445e36f..000000000 --- a/crates/environment-protocol/src/gateway.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Stable environment gateway route paths. - -pub const ROUTE_PATH_PREFIX: &str = "/environment-gateway/routes"; - -/// Appended to a provider controller base URL for Lightspeed-initiated data -/// connections. The remainder is -/// `/{universe}/{binding}/{environment}/{incarnation}/{target}`. -pub const PROVIDER_DATA_PATH_PREFIX: &str = "/routes"; diff --git a/crates/environment-protocol/src/handshake.rs b/crates/environment-protocol/src/handshake.rs new file mode 100644 index 000000000..ccc6ba291 --- /dev/null +++ b/crates/environment-protocol/src/handshake.rs @@ -0,0 +1,257 @@ +//! Values every handshake shares. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use thiserror::Error; + +#[cfg(feature = "schema")] +use crate::ids::NO_CONTROL_PATTERN; +use crate::limits::MAX_IMPLEMENTATION_FIELD_BYTES; + +/// The plane a connection serves. Each `initialize` response names its +/// plane, so a client that reached the wrong socket fails at the handshake. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum Plane { + /// envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`. + Data, + /// A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`. + Controller, + /// The registration channel between a daemon and the gateway, versioned + /// by `ENVD_PROTOCOL_VERSION`. + Registration, +} + +impl Plane { + pub fn as_str(self) -> &'static str { + match self { + Self::Data => "data", + Self::Controller => "controller", + Self::Registration => "registration", + } + } +} + +impl std::fmt::Display for Plane { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// What a peer says about its own build: `clientInfo`, `serverInfo` and a +/// daemon's registration `implementation`. Descriptive only; the protocol +/// version alone decides whether two peers can talk, and nothing here +/// selects, authenticates or gates. Every field is 1 to 256 bytes without control characters, +/// checked on decode and by every constructor, so a value that would be +/// refused can never be built or sent. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ImplementationInfo { + /// Product name, such as `lightspeed-envd`. + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 256), pattern(NO_CONTROL_PATTERN)) + )] + name: String, + /// Release version, such as `0.5.0`. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 256), pattern(NO_CONTROL_PATTERN)) + )] + version: Option, + /// Full or abbreviated git commit of the build. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 256), pattern(NO_CONTROL_PATTERN)) + )] + git_sha: Option, + /// Rust target triple of the build. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 256), pattern(NO_CONTROL_PATTERN)) + )] + target: Option, +} + +/// The decoded shape before the bounds are checked. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ImplementationInfoFields { + name: String, + #[serde(default)] + version: Option, + #[serde(default)] + git_sha: Option, + #[serde(default)] + target: Option, +} + +impl ImplementationInfoFields { + fn into_info(self) -> Result { + let info = ImplementationInfo { + name: self.name, + version: self.version, + git_sha: self.git_sha, + target: self.target, + }; + check_field("name", Some(&info.name))?; + check_field("version", info.version.as_deref())?; + check_field("gitSha", info.git_sha.as_deref())?; + check_field("target", info.target.as_deref())?; + Ok(info) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +#[error("{field} must be 1..={max} bytes without control characters", max = MAX_IMPLEMENTATION_FIELD_BYTES)] +pub struct ImplementationInfoError { + /// Wire name of the offending member. + pub field: &'static str, +} + +impl ImplementationInfo { + pub fn new(name: impl Into) -> Result { + let name = name.into(); + check_field("name", Some(&name))?; + Ok(Self { + name, + version: None, + git_sha: None, + target: None, + }) + } + + pub fn with_version( + mut self, + version: impl Into, + ) -> Result { + let version = version.into(); + check_field("version", Some(&version))?; + self.version = Some(version); + Ok(self) + } + + pub fn with_git_sha( + mut self, + git_sha: impl Into, + ) -> Result { + let git_sha = git_sha.into(); + check_field("gitSha", Some(&git_sha))?; + self.git_sha = Some(git_sha); + Ok(self) + } + + pub fn with_target( + mut self, + target: impl Into, + ) -> Result { + let target = target.into(); + check_field("target", Some(&target))?; + self.target = Some(target); + Ok(self) + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn version(&self) -> Option<&str> { + self.version.as_deref() + } + + pub fn git_sha(&self) -> Option<&str> { + self.git_sha.as_deref() + } + + pub fn target(&self) -> Option<&str> { + self.target.as_deref() + } + + /// For a value that has the shape of an `ImplementationInfo` but breaks + /// its bounds, the offending member. `None` when the value decodes, or + /// when it does not even have the shape (a strict decode reports that). + pub fn bounds_violation(value: &Value) -> Option { + ImplementationInfoFields::deserialize(value) + .ok()? + .into_info() + .err() + } +} + +impl<'de> Deserialize<'de> for ImplementationInfo { + fn deserialize>(deserializer: D) -> Result { + ImplementationInfoFields::deserialize(deserializer)? + .into_info() + .map_err(serde::de::Error::custom) + } +} + +fn check_field(field: &'static str, value: Option<&str>) -> Result<(), ImplementationInfoError> { + match value { + Some(value) + if value.is_empty() + || value.len() > MAX_IMPLEMENTATION_FIELD_BYTES as usize + || value.chars().any(char::is_control) => + { + Err(ImplementationInfoError { field }) + } + _ => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn implementation_info_is_bounded_on_decode() { + let ok = serde_json::json!({"name": "lightspeed-envd", "version": "0.5.0"}); + assert!(serde_json::from_value::(ok).is_ok()); + for bad in [ + serde_json::json!({"name": ""}), + serde_json::json!({"name": "x", "version": "a\nb"}), + serde_json::json!({"name": "x", "gitSha": "a".repeat(257)}), + serde_json::json!({"name": "x", "extra": 1}), + ] { + assert!(serde_json::from_value::(bad).is_err()); + } + } + + #[test] + fn constructors_enforce_the_decode_bounds() { + let info = ImplementationInfo::new("lightspeed-envd") + .and_then(|info| info.with_version("0.5.0")) + .unwrap(); + assert_eq!(info.version(), Some("0.5.0")); + assert_eq!( + ImplementationInfo::new(""), + Err(ImplementationInfoError { field: "name" }) + ); + assert_eq!( + info.with_git_sha("a\nb"), + Err(ImplementationInfoError { field: "gitSha" }) + ); + } + + #[test] + fn bounds_violations_are_told_apart_from_shape_errors() { + let long = serde_json::json!({"name": "x", "version": "v".repeat(257)}); + assert_eq!( + ImplementationInfo::bounds_violation(&long), + Some(ImplementationInfoError { field: "version" }) + ); + assert_eq!( + ImplementationInfo::bounds_violation(&serde_json::json!({"name": 1})), + None + ); + assert_eq!( + ImplementationInfo::bounds_violation(&serde_json::json!({"name": "ok"})), + None + ); + } +} diff --git a/crates/environment-protocol/src/ids.rs b/crates/environment-protocol/src/ids.rs new file mode 100644 index 000000000..f94d60a61 --- /dev/null +++ b/crates/environment-protocol/src/ids.rs @@ -0,0 +1,334 @@ +//! Typed identifiers. +//! +//! Every id space has its own type with a private field and a validating +//! decoder. Two grammars exist: +//! +//! - **General** (Lightspeed-assigned ids): 1 to 128 bytes, an ASCII letter +//! or digit first, then ASCII letters, digits, `_`, `-`, `.` or `:`. +//! - **Opaque** (provider-assigned ids and daemon-minted tokens): 1 to 255 +//! bytes without control characters. Container, Kubernetes and cloud ids +//! such as `default/pod-abc` or an ARN are valid as they are. +//! +//! [`JobName`] is a third, caller-chosen name space with its own rule. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +pub const GENERAL_ID_MAX_BYTES: u32 = 128; +pub const OPAQUE_ID_MAX_BYTES: u32 = 255; +pub const JOB_NAME_MAX_BYTES: u32 = 128; + +/// JSON Schema pattern of the general grammar. +pub const GENERAL_ID_PATTERN: &str = "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$"; +/// JSON Schema pattern of a string without control characters (Unicode +/// category Cc), the character rule of opaque ids and job names. +pub const NO_CONTROL_PATTERN: &str = "^[^\\u0000-\\u001F\\u007F-\\u009F]+$"; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum IdError { + #[error("{kind} must not be empty")] + Empty { kind: &'static str }, + + #[error("{kind} is {actual} bytes; the maximum is {max}")] + TooLong { + kind: &'static str, + max: u32, + actual: usize, + }, + + #[error("{kind} must start with an ASCII letter or digit")] + InvalidStart { kind: &'static str }, + + #[error( + "{kind} contains {character:?} at byte {index}; allowed are ASCII letters, digits, '_', '-', '.', ':'" + )] + InvalidCharacter { + kind: &'static str, + index: usize, + character: char, + }, + + #[error("{kind} contains a control character at byte {index}")] + ControlCharacter { kind: &'static str, index: usize }, +} + +/// Check the general id grammar. +pub fn validate_general_id(kind: &'static str, value: &str) -> Result<(), IdError> { + check_length(kind, value, GENERAL_ID_MAX_BYTES)?; + if !value.as_bytes()[0].is_ascii_alphanumeric() { + return Err(IdError::InvalidStart { kind }); + } + if let Some((index, character)) = value + .char_indices() + .find(|(_, character)| !(character.is_ascii_alphanumeric() || "_-.:".contains(*character))) + { + return Err(IdError::InvalidCharacter { + kind, + index, + character, + }); + } + Ok(()) +} + +/// Check the opaque id grammar. +pub fn validate_opaque_id(kind: &'static str, value: &str) -> Result<(), IdError> { + check_length(kind, value, OPAQUE_ID_MAX_BYTES)?; + check_no_control(kind, value) +} + +/// Check the job name rule: 1 to [`JOB_NAME_MAX_BYTES`] bytes without +/// control characters. +pub fn validate_job_name(kind: &'static str, value: &str) -> Result<(), IdError> { + check_length(kind, value, JOB_NAME_MAX_BYTES)?; + check_no_control(kind, value) +} + +fn check_no_control(kind: &'static str, value: &str) -> Result<(), IdError> { + if let Some((index, _)) = value + .char_indices() + .find(|(_, character)| character.is_control()) + { + return Err(IdError::ControlCharacter { kind, index }); + } + Ok(()) +} + +fn check_length(kind: &'static str, value: &str, max: u32) -> Result<(), IdError> { + if value.is_empty() { + return Err(IdError::Empty { kind }); + } + if value.len() > max as usize { + return Err(IdError::TooLong { + kind, + max, + actual: value.len(), + }); + } + Ok(()) +} + +macro_rules! protocol_id { + ($(#[$doc:meta])* $name:ident, general) => { + protocol_id!( + $(#[$doc])* $name, + validate_general_id, + length(min = 1, max = 128), + pattern(GENERAL_ID_PATTERN) + ); + }; + ($(#[$doc:meta])* $name:ident, opaque) => { + protocol_id!( + $(#[$doc])* $name, + validate_opaque_id, + length(min = 1, max = 255), + pattern(NO_CONTROL_PATTERN) + ); + }; + ($(#[$doc:meta])* $name:ident, $validate:ident, $($schema:tt)*) => { + $(#[$doc])* + #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] + pub struct $name(#[cfg_attr(feature = "schema", schemars($($schema)*))] String); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + $validate(stringify!($name), &value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } + } + + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } + } + + impl FromStr for $name { + type Err = IdError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } + } + + impl TryFrom for $name { + type Error = IdError; + + fn try_from(value: String) -> Result { + Self::new(value) + } + } + + impl TryFrom<&str> for $name { + type Error = IdError; + + fn try_from(value: &str) -> Result { + Self::new(value) + } + } + + impl AsRef for $name { + fn as_ref(&self) -> &str { + &self.0 + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } + } + }; +} + +protocol_id!( + /// `processId`: caller-chosen handle of a process on one daemon. + ProcessId, + general +); +protocol_id!( + /// `jobId`: caller-chosen name of one job run on one daemon. + JobId, + general +); +protocol_id!( + /// `requestId` of a `job/start` call. Part of each job's identity: a + /// retried start with the same request id and spec replays the job; any + /// other reuse of the job id is refused. + JobRequestId, + general +); +protocol_id!( + /// `queueKey`: jobs sharing a key run one at a time, in start order. + QueueKey, + general +); +protocol_id!( + /// `operationId` of one transfer operation. + TransferOperationId, + general +); +protocol_id!( + /// `daemonInstanceId`: random per daemon start. A change between two + /// connections means the daemon restarted and every process handle from + /// the earlier instance is gone. + DaemonInstanceId, + general +); +protocol_id!( + /// `universeId` of a Lightspeed universe. + UniverseId, + general +); +protocol_id!( + /// `bindingId` of a provider binding within a universe. + BindingId, + general +); +protocol_id!( + /// `environmentId` of a Lightspeed environment. + EnvironmentId, + general +); +protocol_id!( + /// `incarnationId` of one incarnation of an environment. + IncarnationId, + general +); +protocol_id!( + /// `daemonId` the gateway assigns to an admitted daemon identity. + DaemonId, + general +); +protocol_id!( + /// `controlConnectionId` the gateway assigns to one control connection. + ControlConnectionId, + general +); +protocol_id!( + /// `targetId`: the provider's own id for a target, opaque to Lightspeed. + ProviderTargetId, + opaque +); +protocol_id!( + /// `templateId`: the provider's own id for a template. + TemplateId, + opaque +); +protocol_id!( + /// `sourceTarget` of `target/adopt`: a provider-native reference to an + /// existing target, in a format the provider defines. + ProviderNativeRef, + opaque +); +protocol_id!( + /// `name` of a job and `dependsOn {kind: name}`: a caller-chosen name, + /// unique within one `job/start`, 1 to 128 bytes without control + /// characters. + JobName, + validate_job_name, + length(min = 1, max = 128), + pattern(NO_CONTROL_PATTERN) +); +protocol_id!( + /// `cursor` and `nextCursor` of `fs/readDirectory`: an opaque position + /// the daemon mints, echoed back unchanged. + DirectoryCursor, + opaque +); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn general_ids_validate_on_decode() { + assert!(serde_json::from_str::("\"build.1:a-b_c\"").is_ok()); + for bad in ["\"\"", "\"-lead\"", "\"has space\"", "\"é\""] { + assert!(serde_json::from_str::(bad).is_err(), "{bad}"); + } + let long = format!("\"{}\"", "a".repeat(GENERAL_ID_MAX_BYTES as usize + 1)); + assert!(serde_json::from_str::(&long).is_err()); + } + + #[test] + fn opaque_ids_accept_natural_provider_ids() { + for good in [ + "default/pod-abc", + "arn:aws:ec2:eu-west-1:123:instance/i-0abc", + "ls-3f2a", + ] { + assert!(ProviderTargetId::new(good).is_ok(), "{good}"); + } + assert!(ProviderTargetId::new("a\nb").is_err()); + assert!(ProviderTargetId::new("").is_err()); + assert!(ProviderTargetId::new("x".repeat(OPAQUE_ID_MAX_BYTES as usize + 1)).is_err()); + } + + #[test] + fn job_names_allow_spaces_but_not_control_characters() { + assert!(JobName::new("build & test").is_ok()); + assert!(JobName::new("a\tb").is_err()); + assert!(JobName::new("").is_err()); + assert!(JobName::new("x".repeat(JOB_NAME_MAX_BYTES as usize + 1)).is_err()); + } +} diff --git a/crates/environment-protocol/src/jsonrpc.rs b/crates/environment-protocol/src/jsonrpc.rs new file mode 100644 index 000000000..617d17906 --- /dev/null +++ b/crates/environment-protocol/src/jsonrpc.rs @@ -0,0 +1,601 @@ +//! The JSON-RPC 2.0 envelope every socket uses: the data plane, the +//! controller plane and the registration channel. +//! +//! Rules, applied by [`parse`] and the constructors here: +//! +//! - One message per WebSocket text frame, at most +//! [`MAX_MESSAGE_BYTES`]. Binary frames +//! are not part of any plane. +//! - `jsonrpc` must be `"2.0"`. +//! - Params are named: an object, or absent, which means `{}`. +//! - A request has a string or integer `id`; a notification has none and is +//! never answered, not even with an error. A malformed frame that has a +//! string `method` and no `id` is a malformed notification: [`parse`] +//! rejects it, and a server drops it silently +//! ([`Rejection::into_response`] returns `None`). +//! - Batches (JSON arrays) are not supported and are answered with one +//! `invalidRequest` error with a null id. +//! - A frame that is not JSON is answered with `parseError` and a null id; +//! the connection stays open. +//! - A server answers every other rejection, with the request's id when it +//! could be read and a null id otherwise; a client that rejects a frame +//! closes the connection. +//! - Requests may be pipelined. Responses arrive in any order and are +//! matched by id. +//! - [`CANCEL_REQUEST_METHOD`] asks the server to abort a pending request. + +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; +use thiserror::Error; + +use crate::{ + error::{ErrorObject, ProtocolError}, + limits::MAX_MESSAGE_BYTES, +}; + +pub const JSONRPC_VERSION: &str = "2.0"; + +/// Client-to-server notification asking the server to abort the request +/// with this id. Read-only waits stop and answer `aborted` with reason +/// `cancelled`; methods with side effects run to a consistent state and +/// answer normally. An unknown or finished id is ignored. +pub const CANCEL_REQUEST_METHOD: &str = "$/cancelRequest"; + +/// A request id: an integer or a string, echoed in the response. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub enum Id { + Number(i64), + String(String), +} + +impl From for Id { + fn from(value: i64) -> Self { + Self::Number(value) + } +} + +impl std::fmt::Display for Id { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Number(number) => number.fmt(formatter), + Self::String(text) => text.fmt(formatter), + } + } +} + +/// `$/cancelRequest` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CancelRequestParams { + /// Id of the request to abort. + pub id: Id, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Request { + pub id: Id, + pub method: String, + /// Always a JSON object. + pub params: Value, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Notification { + pub method: String, + /// Always a JSON object. + pub params: Value, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Response { + /// Absent (null on the wire) only when the request's id could not be + /// read. + pub id: Option, + pub result: Result, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Message { + Request(Request), + Notification(Notification), + Response(Response), +} + +/// A frame that cannot be accepted, with the error to answer it with. +/// +/// A server answers a rejection with [`into_response`](Self::into_response) +/// unless the frame was a malformed notification, which is dropped +/// silently. A client that rejects a frame closes the connection. +#[derive(Clone, Debug, PartialEq)] +pub struct Rejection { + /// The rejected request's id; absent when it could not be read. + pub id: Option, + pub error: ProtocolError, + /// The frame had a string `method` and no `id`. + notification: bool, +} + +impl Rejection { + fn of_frame(error: ProtocolError) -> Self { + Self { + id: None, + error, + notification: false, + } + } + + /// True when the frame was a malformed notification: it must not be + /// answered. + pub fn is_notification(&self) -> bool { + self.notification + } + + /// The error response a server sends for this rejection, or `None` for + /// a malformed notification, which is never answered. + pub fn into_response(self) -> Option { + (!self.notification).then_some(Response { + id: self.id, + result: Err(self.error), + }) + } +} + +/// Classify one text frame. +pub fn parse(text: &str) -> Result { + let value: Value = serde_json::from_str(text).map_err(|error| { + Rejection::of_frame(ProtocolError::parse_error(format!( + "frame is not JSON: {error}" + ))) + })?; + parse_value(value) +} + +/// Classify one decoded frame. +pub fn parse_value(value: Value) -> Result { + let mut object = match value { + Value::Object(object) => object, + Value::Array(_) => { + return Err(Rejection::of_frame(ProtocolError::invalid_request( + "batches are not supported", + ))); + } + _ => { + return Err(Rejection::of_frame(ProtocolError::invalid_request( + "a message must be a JSON object", + ))); + } + }; + let notification = + !object.contains_key("id") && matches!(object.get("method"), Some(Value::String(_))); + let id = match object.remove("id") { + None => None, + Some(value) => match serde_json::from_value::(value.clone()) { + Ok(id) => Some(id), + Err(_) if value.is_null() && object.contains_key("error") => None, + Err(_) => { + return Err(Rejection::of_frame(ProtocolError::invalid_request( + "id must be a string or an integer", + ))); + } + }, + }; + let reject = |error: ProtocolError| Rejection { + id: id.clone(), + error, + notification, + }; + match object.remove("jsonrpc") { + Some(Value::String(version)) if version == JSONRPC_VERSION => {} + _ => { + return Err(reject(ProtocolError::invalid_request( + "jsonrpc must be \"2.0\"", + ))); + } + } + + if let Some(method) = object.remove("method") { + let Value::String(method) = method else { + return Err(reject(ProtocolError::invalid_request( + "method must be a string", + ))); + }; + let params = match object.remove("params") { + None => Value::Object(Map::new()), + Some(params @ Value::Object(_)) => params, + Some(_) => { + return Err(reject(ProtocolError::invalid_params( + "params must be an object", + ))); + } + }; + if let Some(member) = object.keys().next() { + return Err(reject(ProtocolError::invalid_request(format!( + "unknown member {member:?}" + )))); + } + return Ok(match id { + Some(id) => Message::Request(Request { id, method, params }), + None => Message::Notification(Notification { method, params }), + }); + } + + let result = match (object.remove("result"), object.remove("error")) { + (Some(result), None) => Ok(result), + (None, Some(error)) => Err(ProtocolError::from_error_value(&error)), + _ => { + return Err(reject(ProtocolError::invalid_request( + "a response carries exactly one of result and error", + ))); + } + }; + if let Some(member) = object.keys().next() { + return Err(reject(ProtocolError::invalid_request(format!( + "unknown member {member:?}" + )))); + } + if id.is_none() && result.is_ok() { + return Err(reject(ProtocolError::invalid_request( + "a result needs an id", + ))); + } + Ok(Message::Response(Response { id, result })) +} + +impl Request { + pub fn new(id: Id, method: &str, params: &P) -> Result { + Ok(Self { + id, + method: method.to_owned(), + params: params_value(params)?, + }) + } + + /// Decode the params strictly; a failure is `invalidParams`. + pub fn params(&self) -> Result { + decode_params(&self.params) + } + + pub fn to_value(&self) -> Value { + serde_json::json!({ + "jsonrpc": JSONRPC_VERSION, + "id": self.id, + "method": self.method, + "params": self.params, + }) + } + + pub fn to_text(&self) -> String { + self.to_value().to_string() + } +} + +impl Notification { + pub fn new(method: &str, params: &P) -> Result { + Ok(Self { + method: method.to_owned(), + params: params_value(params)?, + }) + } + + /// Decode the params strictly; a failure is `invalidParams`. + pub fn params(&self) -> Result { + decode_params(&self.params) + } + + pub fn to_value(&self) -> Value { + serde_json::json!({ + "jsonrpc": JSONRPC_VERSION, + "method": self.method, + "params": self.params, + }) + } + + pub fn to_text(&self) -> String { + self.to_value().to_string() + } +} + +impl Response { + pub fn success(id: Id, result: &T) -> Self { + let result = serde_json::to_value(result) + .map_err(|error| ProtocolError::internal(format!("encode result: {error}"))); + Self { + id: Some(id), + result, + } + } + + pub fn failure(id: Option, error: ProtocolError) -> Self { + Self { + id, + result: Err(error), + } + } + + pub fn from_result(id: Id, result: Result) -> Self { + match result { + Ok(value) => Self::success(id, &value), + Err(error) => Self::failure(Some(id), error), + } + } + + /// Decode a successful result strictly. The two failures stay apart: the + /// peer answered with an error ([`ResponseError::Failed`]), or its result + /// breaks the contract ([`ResponseError::Undecodable`]). + pub fn decode(self) -> Result { + let value = self.result.map_err(ResponseError::Failed)?; + serde_path_to_error::deserialize(&value) + .map_err(|error| ResponseError::Undecodable(describe_path_error(&error))) + } + + pub fn to_value(&self) -> Value { + let id = self.id.as_ref().map_or(Value::Null, |id| { + serde_json::to_value(id).unwrap_or(Value::Null) + }); + match &self.result { + Ok(result) => { + serde_json::json!({"jsonrpc": JSONRPC_VERSION, "id": id, "result": result}) + } + Err(error) => serde_json::json!({ + "jsonrpc": JSONRPC_VERSION, + "id": id, + "error": ErrorObject::from(error.clone()), + }), + } + } + + /// Encode for sending. A response larger than + /// [`MAX_MESSAGE_BYTES`] is replaced by a `payloadTooLarge` error for the + /// same id, so a server never sends a frame its peer must drop. + pub fn to_text(&self) -> String { + let text = self.to_value().to_string(); + if (text.len() as u64) <= MAX_MESSAGE_BYTES { + return text; + } + Self::failure( + self.id.clone(), + ProtocolError::payload_too_large( + MAX_MESSAGE_BYTES, + format!("response of {} bytes exceeds the message limit", text.len()), + ), + ) + .to_value() + .to_string() + } +} + +/// Why a response yielded no typed result. +#[derive(Clone, Debug, PartialEq, Error)] +pub enum ResponseError { + /// The peer answered with this error. + #[error(transparent)] + Failed(ProtocolError), + + /// The result does not decode as the method's response type: the peer + /// broke the contract. + #[error("result does not decode: {0}")] + Undecodable(String), +} + +/// Check an encoded outgoing message against the message limit. +pub fn check_message_size(text: &str) -> Result<(), ProtocolError> { + let length = text.len() as u64; + if length > MAX_MESSAGE_BYTES { + return Err(ProtocolError::payload_too_large( + MAX_MESSAGE_BYTES, + format!("message of {length} bytes exceeds the {MAX_MESSAGE_BYTES}-byte limit"), + )); + } + Ok(()) +} + +fn params_value(params: &P) -> Result { + match serde_json::to_value(params) { + Ok(value @ Value::Object(_)) => Ok(value), + Ok(_) => Err(ProtocolError::internal( + "params must serialize as an object", + )), + Err(error) => Err(ProtocolError::internal(format!("encode params: {error}"))), + } +} + +/// Strict decode; a failure is `invalidParams` naming the offending field by +/// its wire path when the decoder knows it. +pub fn decode_params(params: &Value) -> Result { + serde_path_to_error::deserialize(params).map_err(|error| { + let field = path_of(&error); + let message = describe_path_error(&error); + match field { + Some(field) => ProtocolError::invalid_field(field, message), + None => ProtocolError::invalid_params(message), + } + }) +} + +fn path_of(error: &serde_path_to_error::Error) -> Option { + let path = error.path(); + path.iter().next().is_some().then(|| path.to_string()) +} + +fn describe_path_error(error: &serde_path_to_error::Error) -> String { + match path_of(error) { + Some(path) => format!("{path}: {}", error.inner()), + None => error.inner().to_string(), + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::error::{ErrorData, ProtocolErrorKind}; + + fn rejected(text: &str) -> Rejection { + parse(text).expect_err("rejected") + } + + #[test] + fn missing_params_mean_an_empty_object() { + let Message::Request(request) = + parse(r#"{"jsonrpc":"2.0","id":1,"method":"env/getIdle"}"#).expect("request") + else { + panic!("expected a request"); + }; + assert_eq!(request.params, json!({})); + assert_eq!(request.id, Id::Number(1)); + } + + #[test] + fn framing_errors_use_reserved_kinds_and_null_ids() { + let parse_error = rejected("not json"); + assert_eq!(parse_error.error.kind(), ProtocolErrorKind::ParseError); + assert_eq!(parse_error.id, None); + let batch = rejected(r#"[{"jsonrpc":"2.0","id":1,"method":"x"}]"#); + assert_eq!(batch.error.kind(), ProtocolErrorKind::InvalidRequest); + assert_eq!(batch.id, None); + let version = rejected(r#"{"jsonrpc":"1.0","id":7,"method":"x"}"#); + assert_eq!(version.error.kind(), ProtocolErrorKind::InvalidRequest); + assert_eq!(version.id, Some(Id::Number(7))); + let params = rejected(r#"{"jsonrpc":"2.0","id":"a","method":"x","params":[1]}"#); + assert_eq!(params.error.kind(), ProtocolErrorKind::InvalidParams); + let member = rejected(r#"{"jsonrpc":"2.0","id":1,"method":"x","extra":true}"#); + assert_eq!(member.error.kind(), ProtocolErrorKind::InvalidRequest); + } + + #[test] + fn notifications_carry_no_id() { + let message = parse(r#"{"jsonrpc":"2.0","method":"$/cancelRequest","params":{"id":3}}"#) + .expect("notification"); + let Message::Notification(notification) = message else { + panic!("expected a notification"); + }; + let params: CancelRequestParams = notification.params().expect("params"); + assert_eq!(params.id, Id::Number(3)); + } + + #[test] + fn error_responses_round_trip_with_null_ids() { + let response = Response::failure(None, ProtocolError::parse_error("bad")); + let text = response.to_text(); + let Message::Response(parsed) = parse(&text).expect("response") else { + panic!("expected a response"); + }; + assert_eq!(parsed, response); + assert_eq!( + serde_json::from_str::(&text).unwrap()["id"], + Value::Null + ); + } + + #[test] + fn oversized_responses_become_payload_too_large() { + let big = "x".repeat(MAX_MESSAGE_BYTES as usize + 1); + let text = Response::success(Id::Number(4), &json!({"data": big})).to_text(); + let Message::Response(parsed) = parse(&text).expect("response") else { + panic!("expected a response"); + }; + assert_eq!(parsed.id, Some(Id::Number(4))); + assert_eq!( + parsed.result.unwrap_err().kind(), + ProtocolErrorKind::PayloadTooLarge + ); + } + + #[test] + fn params_decode_failures_are_invalid_params_naming_the_field() { + let request = Request::new(Id::Number(1), "x", &json!({"id": true})).unwrap(); + let error = request.params::().unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + assert!(matches!( + error.data, + ErrorData::InvalidParams { field: Some(ref field), .. } if field == "id" + )); + } + + #[test] + fn malformed_notifications_are_never_answered() { + for text in [ + r#"{"jsonrpc":"2.0","method":"$/cancelRequest","params":[3]}"#, + r#"{"jsonrpc":"2.0","method":"$/cancelRequest","params":{"id":3},"extra":1}"#, + r#"{"jsonrpc":"1.0","method":"$/cancelRequest","params":{"id":3}}"#, + r#"{"method":"$/cancelRequest"}"#, + ] { + let rejection = rejected(text); + assert!(rejection.is_notification(), "{text}"); + assert_eq!(rejection.into_response(), None, "{text}"); + } + } + + #[test] + fn malformed_requests_and_frames_are_answered() { + for (text, id) in [ + ("not json", None), + ("[]", None), + ("7", None), + ( + r#"{"jsonrpc":"2.0","id":5,"method":"x","params":[1]}"#, + Some(Id::Number(5)), + ), + (r#"{"jsonrpc":"2.0","id":{},"method":"x"}"#, None), + (r#"{"jsonrpc":"2.0","method":7}"#, None), + ( + r#"{"jsonrpc":"2.0","id":"r","result":1,"error":{}}"#, + Some(Id::String("r".to_owned())), + ), + ] { + let rejection = rejected(text); + assert!(!rejection.is_notification(), "{text}"); + let response = rejection.into_response().expect("answered"); + assert_eq!(response.id, id, "{text}"); + assert!(response.result.is_err()); + } + } + + #[test] + fn responses_parse_strictly() { + let Message::Response(ok) = + parse(r#"{"jsonrpc":"2.0","id":1,"result":{"idleForMs":0}}"#).expect("response") + else { + panic!("expected a response"); + }; + assert_eq!(ok.id, Some(Id::Number(1))); + for bad in [ + r#"{"jsonrpc":"2.0","id":null,"result":{}}"#, + r#"{"jsonrpc":"2.0","result":{}}"#, + r#"{"id":1,"result":{}}"#, + r#"{"jsonrpc":"2.0","id":1}"#, + r#"{"jsonrpc":"2.0","id":1,"result":{},"extra":true}"#, + ] { + assert!(parse(bad).is_err(), "{bad}"); + } + let Message::Response(failed) = parse( + r#"{"jsonrpc":"2.0","id":null,"error":{"code":-32700,"message":"m","data":{"kind":"parseError"}}}"#, + ) + .expect("error response") else { + panic!("expected a response"); + }; + assert_eq!(failed.id, None); + assert_eq!( + failed.result.unwrap_err().kind(), + ProtocolErrorKind::ParseError + ); + } + + #[test] + fn decode_tells_a_failed_call_from_a_broken_result() { + let failed = Response::failure(Some(Id::Number(1)), ProtocolError::internal("x")); + assert!(matches!( + failed.decode::(), + Err(ResponseError::Failed(_)) + )); + let broken = Response::success(Id::Number(1), &json!({"id": [1]})); + assert!(matches!( + broken.decode::(), + Err(ResponseError::Undecodable(_)) + )); + } +} diff --git a/crates/environment-protocol/src/lib.rs b/crates/environment-protocol/src/lib.rs index 6882d6c69..a0cbca96f 100644 --- a/crates/environment-protocol/src/lib.rs +++ b/crates/environment-protocol/src/lib.rs @@ -1,12 +1,100 @@ -//! Protocol types for environment execution targets. +//! Wire types of the environment protocol. //! -//! This crate is intentionally transport-free. It defines the stable -//! request/response records used by clients, provider controllers, and environment -//! implementations. +//! Three planes share one JSON-RPC 2.0 envelope ([`jsonrpc`]) and one error +//! vocabulary ([`error`]): +//! +//! - the **data plane** ([`data`]): the runtime drives envd's filesystem, +//! processes and jobs; versioned by [`ENVD_PROTOCOL_VERSION`]; +//! - the **controller plane** ([`control`]): the runtime drives a provider +//! controller's targets; versioned by [`CONTROLLER_PROTOCOL_VERSION`]; +//! - the **registration channel** ([`registration`]): a daemon registers +//! with the gateway and is asked for reverse-dialed streams; versioned by +//! [`ENVD_PROTOCOL_VERSION`]. +//! +//! [`manifest`] lists every method with its types. The crate is +//! transport-free: it defines messages and the rules both sides follow, +//! not sockets. +//! +//! # Conventions +//! +//! These rules hold for every plane; the frozen bootstrap surface +//! ([`registration::ChallengeParams`], [`bootstrap`]) is the one exception +//! to rule 13. +//! +//! 1. **Framing.** JSON-RPC 2.0 over WebSocket text frames; `initialize` +//! first on the data and controller planes; requests may be pipelined +//! and are matched by id. +//! 2. **Errors.** Integer codes fixed by kind, `data.kind` authoritative, +//! typed fields instead of free-form details; [`error::ProtocolError`] +//! displays as `{kind}: {message}`. +//! 3. **Methods.** `domain/verb[Object]` in lowerCamel with singular +//! domains, batch-ness in params, no plane prefix; every method is a +//! manifest entry; types are `Params` and +//! `Response`. One exception: methods of a family that +//! all act on one thing and answer its state share that state as their +//! response type, and share one params type when they name only that +//! thing (`TransferStatus` and `TransferOperationParams` for +//! `transfer/*`, `TargetResponse` for `target/*` and `ingress/*`). +//! 4. **Fields.** camelCase; plural nouns for lists; `Count` +//! for counts, `Bytes` for sizes, `…Ms` for durations, +//! `…AtMs` for instants, `…Id`/`…Ids` for ids; payload bytes are always +//! `data`; one name per relation (`dependsOn`, `truncated`). Positions +//! name their unit: `nextOffset` (bytes of one file), `nextIndex` +//! (entries), `nextCursor` (opaque); an output page's `next` and `end` +//! are byte offsets into the handle's output stream. +//! 5. **Optionality.** No `null` except JSON-RPC's own `id`; absent options +//! and empty collections are skipped; flags are `bool`, default false, +//! skipped when false; no `Option`, no empty-string sentinels. +//! 6. **Numbers.** `u64` for bytes, offsets and durations; `u32` for counts, +//! indices and depths; `i64` for instants; no `usize` on the wire; every +//! value below 2^53. +//! 7. **Unions.** Data unions are tagged `kind`; variant fields are renamed +//! only through enum-level `rename_all_fields`; operations are methods, +//! never tagged actions. +//! 8. **Ids.** Validated newtypes ([`ids`]): the general grammar for +//! Lightspeed-assigned ids, the opaque rule for provider-assigned ids. +//! 9. **Bytes.** Padded standard base64 for payloads; canonical lowercase +//! hex for fixed-size crypto values; [`paths::ContentDigest`] as +//! `sha256:`; opaque fingerprints carry their own prefix. +//! 10. **Paths.** [`paths::EnvironmentPath`] (absolute) for endpoint paths, +//! [`paths::TreePath`] for relative tree entries. +//! 11. **Limits.** Caller quotas are required per request; endpoint ceilings +//! are advertised in `initialize`; the message size is a protocol +//! constant ([`limits`]); out-of-range values are rejected, never +//! clamped. Ceilings are chosen so every response stays below the +//! message limit: payload ceilings fit after base64, lists have length +//! ceilings, walks stop at [`limits::MAX_RESULT_BYTES`], descriptive +//! strings are bounded. [`jsonrpc::Response::to_text`]'s +//! `payloadTooLarge` replacement is a defect backstop, not paging. +//! 12. **Capabilities.** In each plane's handshake only, as instance facts +//! grouped by family; an absent family or fact is +//! `capabilityUnavailable`; an endpoint never silently ignores a +//! parameter. +//! 13. **Decoding.** Strict on every plane: unknown members are rejected, +//! and defaults exist only for documented optional knobs. +//! 14. **Docs and fixtures.** Doc comments use wire names; every params and +//! response type and every enum is documented; `fixtures/` holds one +//! params, response and error fixture per method. +//! 15. **Boundaries.** Wire types are not persistence, workflow or +//! model-facing types; each layer maps at its own boundary. +pub mod bootstrap; +pub mod bytes; +#[cfg(feature = "schema")] +pub mod contract; pub mod control; pub mod data; pub mod error; -pub mod gateway; +pub mod handshake; +pub mod ids; +pub mod jsonrpc; +pub mod limits; +pub mod manifest; +pub mod metadata; +pub mod paths; pub mod registration; -pub mod shared; +pub mod routes; +mod serde_util; +mod version; + +pub use version::{CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION}; diff --git a/crates/environment-protocol/src/limits.rs b/crates/environment-protocol/src/limits.rs new file mode 100644 index 000000000..7f3767f10 --- /dev/null +++ b/crates/environment-protocol/src/limits.rs @@ -0,0 +1,80 @@ +//! Protocol constants: the limits every hop enforces without reading a +//! handshake. +//! +//! Operational ceilings that a deployment configures (request concurrency, +//! retention, wait, scan and transfer maxima) are not constants: the data +//! plane advertises them in `initialize` as +//! [`DataLimits`](crate::data::handshake::DataLimits). Caller quotas are +//! required fields on each request, and a value above an advertised ceiling +//! is refused with `invalidParams` naming the field, never clamped. + +/// Largest WebSocket message, and therefore frame, that any hop sends or +/// accepts on any plane. Every reader and writer configures it explicitly, +/// with the frame limit equal to the message limit. A sender that would +/// exceed it fails with `payloadTooLarge` instead of sending. +pub const MAX_MESSAGE_BYTES: u64 = 16 * 1024 * 1024; + +/// A peer waiting for a response sends a WebSocket ping at least this often +/// while any request it sent is unanswered. +pub const KEEPALIVE_INTERVAL_MS: u64 = 20_000; + +/// Intermediaries (gateways, relays, proxies) forward pings and pongs, count +/// them as activity, and never close a connection for idleness sooner than +/// this. +pub const MIN_INTERMEDIARY_IDLE_MS: u64 = 60_000; + +/// Every data and controller server accepts at least this many requests in +/// flight on one connection before it answers `busy`. +pub const MIN_CONCURRENT_REQUESTS: u32 = 16; + +/// A registration-channel peer that receives no frame for this many +/// `heartbeatIntervalMs` treats the connection as dead. +pub const HEARTBEAT_MISSED_INTERVALS: u32 = 3; + +/// Longest [`EnvironmentPath`](crate::paths::EnvironmentPath) or +/// [`TreePath`](crate::paths::TreePath) in bytes. +pub const MAX_PATH_BYTES: u32 = 4096; + +/// Longest [`DescriptiveText`](crate::metadata::DescriptiveText) in bytes: +/// a target's `message`, a template's `description`, a +/// `registration/closed` message. +pub const MAX_DESCRIPTIVE_TEXT_BYTES: u32 = 1024; + +/// Longest URL a peer advertises or reports (`routes.data`, an ingress +/// `endpoint`), in bytes. +pub const MAX_URL_BYTES: u32 = 2048; + +/// Longest error `message` in bytes. [`ProtocolError::new`] cuts longer +/// messages on a character boundary; the message is never parsed. +/// +/// [`ProtocolError::new`]: crate::error::ProtocolError::new +pub const MAX_ERROR_MESSAGE_BYTES: u32 = 4096; + +/// Most chunks plus gaps in one `process/read` page, and across all entries +/// of one `job/read` response. A page that reaches it ends early and loses +/// nothing: `next` says where to continue. +pub const MAX_PAGE_CHUNKS: u32 = 4096; + +/// Budget, in encoded bytes, for the variable part of a walk's result: the +/// matches of `fs/searchText` and `fs/globFiles`, and the entries, skips +/// and base64 content of all roots of one `fs/scan`. A walk stops (search +/// and glob: `stopped: responseLimit`) or a scan root fails +/// (`payloadTooLarge`) before its result would exceed it, so every response +/// stays below [`MAX_MESSAGE_BYTES`] with room for its fixed members. +pub const MAX_RESULT_BYTES: u64 = 12 * 1024 * 1024; + +/// Most skipped entries a scan root or transfer lists by path; the total is +/// always counted in `skippedEntryCount`. +pub const MAX_LISTED_SKIPS: u32 = 100; + +/// Most leftover processes a report lists; the rest are counted in +/// `omittedProcessCount`. +pub const MAX_LEFTOVER_PROCESSES: u32 = 64; + +/// Longest leftover `command` in bytes; longer command lines are cut on a +/// character boundary. +pub const MAX_LEFTOVER_COMMAND_BYTES: u32 = 1024; + +/// Longest field of an +/// [`ImplementationInfo`](crate::handshake::ImplementationInfo) in bytes. +pub const MAX_IMPLEMENTATION_FIELD_BYTES: u32 = 256; diff --git a/crates/environment-protocol/src/manifest.rs b/crates/environment-protocol/src/manifest.rs new file mode 100644 index 000000000..c894b09d9 --- /dev/null +++ b/crates/environment-protocol/src/manifest.rs @@ -0,0 +1,651 @@ +//! The method manifest: one table of every method on every plane, with its +//! params and response types and the facts servers and clients act on. +//! +//! envd dispatch and idle classification, the client's typed calls, the +//! contract export and the fixture tests all read this table, so a method +//! exists exactly when it has an entry here. Each entry also has a marker +//! type implementing [`RequestMethod`] or [`NotificationMethod`], which +//! binds the method name to its Rust types: +//! +//! ``` +//! use environment_protocol::manifest::{ProcessRead, RequestMethod}; +//! assert_eq!(ProcessRead::NAME, "process/read"); +//! ``` + +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; + +use crate::{ + control::{ + handshake::{ControllerInitializeParams, ControllerInitializeResponse}, + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + AdoptTargetParams, CloseTargetParams, CreateTargetParams, GetTargetParams, + SetTargetPowerParams, TargetResponse, + }, + template::{ListTemplatesParams, ListTemplatesResponse}, + }, + data::{ + fs::{ + CreateDirectoryParams, CreateDirectoryResponse, GetMetadataParams, GetMetadataResponse, + GlobFilesParams, GlobFilesResponse, ReadDirectoryParams, ReadDirectoryResponse, + ReadFileParams, ReadFileResponse, RemoveParams, RemoveResponse, RenameParams, + RenameResponse, SearchTextParams, SearchTextResponse, WriteFileParams, + WriteFileResponse, + }, + handshake::{InitializeParams, InitializeResponse}, + idle::{GetIdleParams, GetIdleResponse}, + job::{ + CancelJobsParams, CancelJobsResponse, ReadJobsParams, ReadJobsResponse, + StartJobsParams, StartJobsResponse, + }, + process::{ + ReadProcessParams, ReadProcessResponse, StartProcessParams, StartProcessResponse, + TerminateProcessParams, TerminateProcessResponse, WriteProcessParams, + WriteProcessResponse, + }, + scan::{ScanParams, ScanResponse}, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, ListTransferMissingParams, + ListTransferMissingResponse, ReadTransferContentParams, ReadTransferContentResponse, + ReadTransferInventoryParams, ReadTransferInventoryResponse, TransferOperationParams, + TransferStatus, WriteTransferContentParams, WriteTransferContentResponse, + }, + }, + error::Capability, + handshake::Plane, + jsonrpc::CancelRequestParams, + registration::{ + ChallengeParams, ClosedParams, GoodbyeParams, OpenStreamParams, OpenStreamResponse, + RegisterParams, RegisterResponse, + }, +}; + +/// A request method: its name, plane and the types of its params and +/// result. +pub trait RequestMethod { + const NAME: &'static str; + const PLANE: Plane; + type Params: Serialize + DeserializeOwned + Send + Sync + 'static; + type Response: Serialize + DeserializeOwned + Send + Sync + 'static; +} + +/// A notification method: never answered. +pub trait NotificationMethod { + const NAME: &'static str; + const PLANE: Plane; + type Params: Serialize + DeserializeOwned + Send + Sync + 'static; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum MessageKind { + Request, + Notification, +} + +impl MessageKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Request => "request", + Self::Notification => "notification", + } + } +} + +/// Who sends a method. The client is the peer that dialed: the runtime on +/// the data and controller planes, the daemon on the registration channel. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Direction { + ClientToServer, + ServerToClient, +} + +impl Direction { + pub fn as_str(self) -> &'static str { + match self { + Self::ClientToServer => "clientToServer", + Self::ServerToClient => "serverToClient", + } + } +} + +/// Method family: the method name's domain. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Family { + Handshake, + Envelope, + Env, + Filesystem, + Transfer, + Process, + Job, + Template, + Target, + Ingress, + Registration, +} + +impl Family { + pub fn as_str(self) -> &'static str { + match self { + Self::Handshake => "handshake", + Self::Envelope => "envelope", + Self::Env => "env", + Self::Filesystem => "filesystem", + Self::Transfer => "transfer", + Self::Process => "process", + Self::Job => "job", + Self::Template => "template", + Self::Target => "target", + Self::Ingress => "ingress", + Self::Registration => "registration", + } + } +} + +/// Whether a data-plane method counts toward the idle clock. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Activity { + /// Real work: resets `idleForMs`. + Use, + /// Looking, not using: handshakes, idle reports, discovery scans. + Observation, +} + +impl Activity { + pub fn as_str(self) -> &'static str { + match self { + Self::Use => "use", + Self::Observation => "observation", + } + } +} + +/// Schemas of one method, registered in a shared generator. +#[cfg(feature = "schema")] +pub struct MethodSchemas { + pub params: schemars::Schema, + pub response: Option, +} + +/// Re-encode a value through a wire type: decoding enforces the contract +/// strictly, encoding yields the canonical form. +pub type Normalize = fn(&Value) -> Result; + +/// One manifest entry. +pub struct MethodSpec { + pub name: &'static str, + pub plane: Plane, + pub kind: MessageKind, + pub direction: Direction, + pub family: Family, + /// Data-plane methods only. + pub activity: Option, + /// The capability a request needs; a server without it answers + /// `capabilityUnavailable` before doing anything. + pub capability: Option, + /// `$/cancelRequest` aborts it: a read-only wait or walk. Every other + /// method runs to a consistent state. + pub abortable: bool, + pub params_type: &'static str, + /// Absent for notifications. + pub response_type: Option<&'static str>, + pub summary: &'static str, + pub normalize_params: Normalize, + pub normalize_response: Option, + #[cfg(feature = "schema")] + pub schemas: fn(&mut schemars::SchemaGenerator) -> MethodSchemas, +} + +impl std::fmt::Debug for MethodSpec { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("MethodSpec") + .field("name", &self.name) + .field("plane", &self.plane) + .field("kind", &self.kind) + .finish_non_exhaustive() + } +} + +fn normalize(value: &Value) -> Result { + serde_json::to_value(T::deserialize(value)?) +} + +macro_rules! requests { + ($( + $marker:ident = $name:literal { + plane: $plane:ident, + direction: $direction:ident, + family: $family:ident, + activity: $activity:expr, + capability: $capability:expr, + abortable: $abortable:expr, + params: $params:ty, + response: $response:ty, + summary: $summary:literal $(,)? + } + )+) => { + $( + #[doc = concat!("`", $name, "`: ", $summary)] + #[derive(Clone, Copy, Debug)] + pub struct $marker; + + impl $marker { + pub const NAME: &'static str = $name; + } + + impl RequestMethod for $marker { + const NAME: &'static str = $name; + const PLANE: Plane = Plane::$plane; + type Params = $params; + type Response = $response; + } + )+ + + /// Every request method. + pub static REQUESTS: &[MethodSpec] = &[ + $( + MethodSpec { + name: $name, + plane: Plane::$plane, + kind: MessageKind::Request, + direction: Direction::$direction, + family: Family::$family, + activity: $activity, + capability: $capability, + abortable: $abortable, + params_type: stringify!($params), + response_type: Some(stringify!($response)), + summary: $summary, + normalize_params: normalize::<$params>, + normalize_response: Some(normalize::<$response>), + #[cfg(feature = "schema")] + schemas: |generator| MethodSchemas { + params: generator.subschema_for::<$params>(), + response: Some(generator.subschema_for::<$response>()), + }, + }, + )+ + ]; + }; +} + +macro_rules! notifications { + ($( + $marker:ident = $name:literal { + plane: $plane:ident, + direction: $direction:ident, + family: $family:ident, + activity: $activity:expr, + params: $params:ty, + summary: $summary:literal $(,)? + } + )+) => { + $( + #[doc = concat!("`", $name, "`: ", $summary)] + #[derive(Clone, Copy, Debug)] + pub struct $marker; + + impl $marker { + pub const NAME: &'static str = $name; + } + + impl NotificationMethod for $marker { + const NAME: &'static str = $name; + const PLANE: Plane = Plane::$plane; + type Params = $params; + } + )+ + + /// Every notification method. + pub static NOTIFICATIONS: &[MethodSpec] = &[ + $( + MethodSpec { + name: $name, + plane: Plane::$plane, + kind: MessageKind::Notification, + direction: Direction::$direction, + family: Family::$family, + activity: $activity, + capability: None, + abortable: false, + params_type: stringify!($params), + response_type: None, + summary: $summary, + normalize_params: normalize::<$params>, + normalize_response: None, + #[cfg(feature = "schema")] + schemas: |generator| MethodSchemas { + params: generator.subschema_for::<$params>(), + response: None, + }, + }, + )+ + ]; + }; +} + +const USE: Option = Some(Activity::Use); +const OBSERVATION: Option = Some(Activity::Observation); +const FILESYSTEM: Option = Some(Capability::Filesystem); +const WRITE: Option = Some(Capability::FilesystemWrite); +const PROCESS: Option = Some(Capability::Process); +const JOB: Option = Some(Capability::Job); + +requests! { + DataInitialize = "initialize" { + plane: Data, direction: ClientToServer, family: Handshake, activity: OBSERVATION, + capability: None, abortable: false, + params: InitializeParams, response: InitializeResponse, + summary: "Check the protocol version and read the daemon's instance id, capabilities and ceilings.", + } + EnvGetIdle = "env/getIdle" { + plane: Data, direction: ClientToServer, family: Env, activity: OBSERVATION, + capability: None, abortable: false, + params: GetIdleParams, response: GetIdleResponse, + summary: "Report how long the environment has been idle.", + } + FsReadFile = "fs/readFile" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ReadFileParams, response: ReadFileResponse, + summary: "Read a byte range of a regular file.", + } + FsWriteFile = "fs/writeFile" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: WRITE, abortable: false, + params: WriteFileParams, response: WriteFileResponse, + summary: "Write a file atomically, optionally conditioned on its version.", + } + FsCreateDirectory = "fs/createDirectory" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: WRITE, abortable: false, + params: CreateDirectoryParams, response: CreateDirectoryResponse, + summary: "Create a directory.", + } + FsGetMetadata = "fs/getMetadata" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: false, + params: GetMetadataParams, response: GetMetadataResponse, + summary: "Describe one entry.", + } + FsReadDirectory = "fs/readDirectory" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ReadDirectoryParams, response: ReadDirectoryResponse, + summary: "List one page of a directory's entries.", + } + FsRemove = "fs/remove" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: WRITE, abortable: false, + params: RemoveParams, response: RemoveResponse, + summary: "Remove a file, symlink or directory.", + } + FsRename = "fs/rename" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: WRITE, abortable: false, + params: RenameParams, response: RenameResponse, + summary: "Rename an entry atomically.", + } + FsSearchText = "fs/searchText" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: true, + params: SearchTextParams, response: SearchTextResponse, + summary: "Search file contents below a root with a regular expression.", + } + FsGlobFiles = "fs/globFiles" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: true, + params: GlobFilesParams, response: GlobFilesResponse, + summary: "List files below a root that match patterns.", + } + FsScan = "fs/scan" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: OBSERVATION, + capability: FILESYSTEM, abortable: true, + params: ScanParams, response: ScanResponse, + summary: "Observe small trees below several roots, each reported on its own.", + } + TransferBegin = "transfer/begin" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: BeginTransferParams, response: TransferStatus, + summary: "Start or resume a transfer operation.", + } + TransferAdvance = "transfer/advance" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: TransferOperationParams, response: TransferStatus, + summary: "Do a bounded step of scanning or staging work.", + } + TransferGetStatus = "transfer/getStatus" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: TransferOperationParams, response: TransferStatus, + summary: "Read an operation's status or receipt.", + } + TransferReadInventory = "transfer/readInventory" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ReadTransferInventoryParams, response: ReadTransferInventoryResponse, + summary: "Read one page of a sealed inventory.", + } + TransferAppend = "transfer/append" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: AppendTransferInventoryParams, response: TransferStatus, + summary: "Append incoming inventory entries.", + } + TransferListMissing = "transfer/listMissing" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ListTransferMissingParams, response: ListTransferMissingResponse, + summary: "List digests whose content the daemon still needs.", + } + TransferRead = "transfer/read" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ReadTransferContentParams, response: ReadTransferContentResponse, + summary: "Read captured content by digest.", + } + TransferWrite = "transfer/write" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: WriteTransferContentParams, response: WriteTransferContentResponse, + summary: "Upload content for a missing digest.", + } + TransferCommit = "transfer/commit" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: TransferOperationParams, response: TransferStatus, + summary: "Finish a ready operation: publish a staged tree, or release a capture.", + } + TransferAbort = "transfer/abort" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: TransferOperationParams, response: TransferStatus, + summary: "Abandon an operation and discard its staging.", + } + ProcessStart = "process/start" { + plane: Data, direction: ClientToServer, family: Process, activity: USE, + capability: PROCESS, abortable: false, + params: StartProcessParams, response: StartProcessResponse, + summary: "Start a process under a caller-chosen handle.", + } + ProcessRead = "process/read" { + plane: Data, direction: ClientToServer, family: Process, activity: USE, + capability: PROCESS, abortable: true, + params: ReadProcessParams, response: ReadProcessResponse, + summary: "Wait for and read a page of a process's output and its state.", + } + ProcessWrite = "process/write" { + plane: Data, direction: ClientToServer, family: Process, activity: USE, + capability: PROCESS, abortable: false, + params: WriteProcessParams, response: WriteProcessResponse, + summary: "Queue bytes for a process's input pipe or PTY.", + } + ProcessTerminate = "process/terminate" { + plane: Data, direction: ClientToServer, family: Process, activity: USE, + capability: PROCESS, abortable: false, + params: TerminateProcessParams, response: TerminateProcessResponse, + summary: "Interrupt, terminate or kill a process's group.", + } + JobStart = "job/start" { + plane: Data, direction: ClientToServer, family: Job, activity: USE, + capability: JOB, abortable: false, + params: StartJobsParams, response: StartJobsResponse, + summary: "Start jobs, or replay an identical earlier start.", + } + JobRead = "job/read" { + plane: Data, direction: ClientToServer, family: Job, activity: OBSERVATION, + capability: JOB, abortable: true, + params: ReadJobsParams, response: ReadJobsResponse, + summary: "Wait for and read jobs' summaries and output pages.", + } + JobCancel = "job/cancel" { + plane: Data, direction: ClientToServer, family: Job, activity: USE, + capability: JOB, abortable: false, + params: CancelJobsParams, response: CancelJobsResponse, + summary: "Cancel jobs and optionally their dependents.", + } + ControllerInitialize = "initialize" { + plane: Controller, direction: ClientToServer, family: Handshake, activity: None, + capability: None, abortable: false, + params: ControllerInitializeParams, response: ControllerInitializeResponse, + summary: "Check the protocol version and read the controller's capabilities and routes.", + } + TemplateList = "template/list" { + plane: Controller, direction: ClientToServer, family: Template, activity: None, + capability: None, abortable: false, + params: ListTemplatesParams, response: ListTemplatesResponse, + summary: "List the templates a binding may create targets from.", + } + TargetCreate = "target/create" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: None, abortable: false, + params: CreateTargetParams, response: TargetResponse, + summary: "Create the owner's target, or observe the one it has.", + } + TargetAdopt = "target/adopt" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: Some(Capability::AdoptTarget), abortable: false, + params: AdoptTargetParams, response: TargetResponse, + summary: "Take ownership of an existing provider target.", + } + TargetGet = "target/get" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: None, abortable: false, + params: GetTargetParams, response: TargetResponse, + summary: "Observe a target without changing it.", + } + TargetSetPower = "target/setPower" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: None, abortable: false, + params: SetTargetPowerParams, response: TargetResponse, + summary: "Converge a target toward a power state.", + } + TargetClose = "target/close" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: None, abortable: false, + params: CloseTargetParams, response: TargetResponse, + summary: "Destroy the owner's target and everything it holds.", + } + IngressEnsure = "ingress/ensure" { + plane: Controller, direction: ClientToServer, family: Ingress, activity: None, + capability: Some(Capability::Ingress), abortable: false, + params: EnsureIngressParams, response: TargetResponse, + summary: "Route public ingress to a target.", + } + IngressRemove = "ingress/remove" { + plane: Controller, direction: ClientToServer, family: Ingress, activity: None, + capability: Some(Capability::Ingress), abortable: false, + params: RemoveIngressParams, response: TargetResponse, + summary: "Stop routing public ingress to a target that stays.", + } + RegistrationRegister = "registration/register" { + plane: Registration, direction: ClientToServer, family: Registration, activity: None, + capability: None, abortable: false, + params: RegisterParams, response: RegisterResponse, + summary: "Prove the daemon identity and be admitted to its environment.", + } + RegistrationOpenStream = "registration/openStream" { + plane: Registration, direction: ServerToClient, family: Registration, activity: None, + capability: None, abortable: false, + params: OpenStreamParams, response: OpenStreamResponse, + summary: "Ask the daemon to dial one reverse stream.", + } +} + +notifications! { + CancelRequest = "$/cancelRequest" { + plane: Data, direction: ClientToServer, family: Envelope, activity: OBSERVATION, + params: CancelRequestParams, + summary: "Abort a pending abortable request.", + } + RegistrationChallenge = "registration/challenge" { + plane: Registration, direction: ServerToClient, family: Registration, activity: None, + params: ChallengeParams, + summary: "The frozen first frame: the gateway's version and nonce.", + } + RegistrationGoodbye = "registration/goodbye" { + plane: Registration, direction: ClientToServer, family: Registration, activity: None, + params: GoodbyeParams, + summary: "The daemon is stopping deliberately.", + } + RegistrationClosed = "registration/closed" { + plane: Registration, direction: ServerToClient, family: Registration, activity: None, + params: ClosedParams, + summary: "The gateway ends the session for good.", + } +} + +/// Every method of every plane: requests, then notifications. +pub fn methods() -> impl Iterator { + REQUESTS.iter().chain(NOTIFICATIONS.iter()) +} + +/// Every method of one plane. +pub fn plane_methods(plane: Plane) -> impl Iterator { + methods().filter(move |spec| spec.plane == plane) +} + +/// The method of this name on this plane. +pub fn find(plane: Plane, name: &str) -> Option<&'static MethodSpec> { + plane_methods(plane).find(|spec| spec.name == name) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use super::*; + + #[test] + fn method_names_are_unique_per_plane() { + let mut seen = BTreeSet::new(); + for spec in methods() { + assert!( + seen.insert((spec.plane.as_str(), spec.name)), + "duplicate {} on {}", + spec.name, + spec.plane + ); + } + } + + #[test] + fn activity_is_declared_for_data_methods_only() { + for spec in methods() { + assert_eq!( + spec.activity.is_some(), + spec.plane == Plane::Data, + "{}", + spec.name + ); + } + assert_eq!( + find(Plane::Data, FsScan::NAME).unwrap().activity, + Some(Activity::Observation) + ); + } +} diff --git a/crates/environment-protocol/src/metadata.rs b/crates/environment-protocol/src/metadata.rs new file mode 100644 index 000000000..918e60336 --- /dev/null +++ b/crates/environment-protocol/src/metadata.rs @@ -0,0 +1,400 @@ +//! Bounds on descriptive values: display names, correlation metadata and +//! short human-readable texts. +//! +//! These values are descriptive: they never select, authenticate or gate. +//! The same bounds apply wherever Lightspeed accepts them (daemon +//! registration, provider observations and templates, environment and +//! session metadata, profile documents), so a session and the environment it +//! ran in accept the same keys. On the wire they are validated on decode by +//! [`DisplayName`], [`Metadata`] and [`DescriptiveText`]. + +use std::{collections::BTreeMap, fmt}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +use crate::limits::MAX_DESCRIPTIVE_TEXT_BYTES; + +pub const MAX_DISPLAY_NAME_BYTES: u32 = 128; +pub const MAX_METADATA_ENTRIES: u32 = 32; +pub const MAX_METADATA_KEY_BYTES: u32 = 64; +pub const MAX_METADATA_VALUE_BYTES: u32 = 256; +/// Keys under this prefix are written by Lightspeed only. +pub const RESERVED_METADATA_PREFIX: &str = "lightspeed."; + +/// JSON Schema pattern of [`DescriptiveText`]: no control characters except +/// line feed and tab. +pub const DESCRIPTIVE_TEXT_PATTERN: &str = "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$"; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum MetadataError { + #[error("display name must be 1..={MAX_DISPLAY_NAME_BYTES} bytes without control characters")] + DisplayName, + + #[error("metadata has more than {MAX_METADATA_ENTRIES} entries")] + TooManyEntries, + + #[error("metadata key {key:?} must be 1..={MAX_METADATA_KEY_BYTES} bytes")] + KeyBytes { key: String }, + + #[error("metadata value for {key:?} must be 1..={MAX_METADATA_VALUE_BYTES} bytes")] + ValueBytes { key: String }, + + #[error("metadata entry {key:?} contains a control character")] + ControlCharacter { key: String }, + + #[error("metadata key {key:?} uses the reserved {RESERVED_METADATA_PREFIX} prefix")] + ReservedPrefix { key: String }, + + #[error( + "text must be 1..={MAX_DESCRIPTIVE_TEXT_BYTES} bytes without control characters other than line feed and tab" + )] + DescriptiveText, +} + +pub fn validate_display_name(name: &str) -> Result<(), MetadataError> { + if name.is_empty() + || name.len() > MAX_DISPLAY_NAME_BYTES as usize + || name.chars().any(char::is_control) + { + return Err(MetadataError::DisplayName); + } + Ok(()) +} + +/// A metadata map a caller supplies: at most [`MAX_METADATA_ENTRIES`] +/// bounded entries and no key under [`RESERVED_METADATA_PREFIX`]. +pub fn validate_caller_metadata(metadata: &BTreeMap) -> Result<(), MetadataError> { + if metadata.len() > MAX_METADATA_ENTRIES as usize { + return Err(MetadataError::TooManyEntries); + } + for (key, value) in metadata { + validate_metadata_entry(key, value)?; + if key.starts_with(RESERVED_METADATA_PREFIX) { + return Err(MetadataError::ReservedPrefix { key: key.clone() }); + } + } + Ok(()) +} + +/// A stored metadata map: caller keys obey [`validate_caller_metadata`]'s +/// count, and Lightspeed's own reserved keys are bounded per entry but not +/// counted. +pub fn validate_stored_metadata(metadata: &BTreeMap) -> Result<(), MetadataError> { + let caller_entries = metadata + .keys() + .filter(|key| !key.starts_with(RESERVED_METADATA_PREFIX)) + .count(); + if caller_entries > MAX_METADATA_ENTRIES as usize { + return Err(MetadataError::TooManyEntries); + } + for (key, value) in metadata { + validate_metadata_entry(key, value)?; + } + Ok(()) +} + +/// Key and value bytes and control characters of one entry. +pub fn validate_metadata_entry(key: &str, value: &str) -> Result<(), MetadataError> { + if key.is_empty() || key.len() > MAX_METADATA_KEY_BYTES as usize { + return Err(MetadataError::KeyBytes { + key: key.to_owned(), + }); + } + if value.is_empty() || value.len() > MAX_METADATA_VALUE_BYTES as usize { + return Err(MetadataError::ValueBytes { + key: key.to_owned(), + }); + } + if key.chars().any(char::is_control) || value.chars().any(char::is_control) { + return Err(MetadataError::ControlCharacter { + key: key.to_owned(), + }); + } + Ok(()) +} + +/// Check [`DescriptiveText`]'s rule. +pub fn validate_descriptive_text(text: &str) -> Result<(), MetadataError> { + if text.is_empty() + || text.len() > MAX_DESCRIPTIVE_TEXT_BYTES as usize + || text.chars().any(is_disallowed_text_control) + { + return Err(MetadataError::DescriptiveText); + } + Ok(()) +} + +fn is_disallowed_text_control(character: char) -> bool { + character.is_control() && character != '\n' && character != '\t' +} + +/// A display name: 1 to 128 bytes without control characters. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct DisplayName( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 128), pattern(crate::ids::NO_CONTROL_PATTERN)) + )] + String, +); + +impl DisplayName { + pub fn new(name: impl Into) -> Result { + let name = name.into(); + validate_display_name(&name)?; + Ok(Self(name)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +/// A short human-readable text: 1 to 1024 bytes without control characters +/// other than line feed and tab. Never parsed. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct DescriptiveText( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 1024), pattern(DESCRIPTIVE_TEXT_PATTERN)) + )] + String, +); + +impl DescriptiveText { + pub fn new(text: impl Into) -> Result { + let text = text.into(); + validate_descriptive_text(&text)?; + Ok(Self(text)) + } + + /// Build a valid text from any string, for senders: disallowed control + /// characters become spaces and the text is cut to the bound on a + /// character boundary. `None` when nothing is left. + pub fn lossy(text: &str) -> Option { + let mut cleaned: String = text + .chars() + .map(|character| { + if is_disallowed_text_control(character) { + ' ' + } else { + character + } + }) + .collect(); + let max = MAX_DESCRIPTIVE_TEXT_BYTES as usize; + if cleaned.len() > max { + let mut end = max; + while !cleaned.is_char_boundary(end) { + end -= 1; + } + cleaned.truncate(end); + } + (!cleaned.is_empty()).then_some(Self(cleaned)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +macro_rules! text_impls { + ($name:ident, $error:ty) => { + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } + } + + impl TryFrom for $name { + type Error = $error; + + fn try_from(value: String) -> Result { + Self::new(value) + } + } + + impl TryFrom<&str> for $name { + type Error = $error; + + fn try_from(value: &str) -> Result { + Self::new(value) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } + } + }; +} + +text_impls!(DisplayName, MetadataError); +text_impls!(DescriptiveText, MetadataError); + +/// Caller-supplied metadata on the wire: at most [`MAX_METADATA_ENTRIES`] +/// bounded entries and no key under [`RESERVED_METADATA_PREFIX`] +/// ([`validate_caller_metadata`]), checked on decode. Serialized as a JSON +/// object of strings. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Metadata(BTreeMap); + +impl Metadata { + pub fn new(entries: BTreeMap) -> Result { + validate_caller_metadata(&entries)?; + Ok(Self(entries)) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.0.get(key).map(String::as_str) + } + + pub fn iter(&self) -> impl Iterator { + self.0 + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + } + + pub fn as_map(&self) -> &BTreeMap { + &self.0 + } + + pub fn into_map(self) -> BTreeMap { + self.0 + } +} + +impl TryFrom> for Metadata { + type Error = MetadataError; + + fn try_from(entries: BTreeMap) -> Result { + Self::new(entries) + } +} + +impl Serialize for Metadata { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Metadata { + fn deserialize>(deserializer: D) -> Result { + let entries = BTreeMap::::deserialize(deserializer)?; + Self::new(entries).map_err(D::Error::custom) + } +} + +#[cfg(feature = "schema")] +impl schemars::JsonSchema for Metadata { + fn schema_name() -> std::borrow::Cow<'static, str> { + "Metadata".into() + } + + fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "type": "object", + "maxProperties": MAX_METADATA_ENTRIES, + "propertyNames": { + "minLength": 1, + "maxLength": MAX_METADATA_KEY_BYTES, + "pattern": crate::ids::NO_CONTROL_PATTERN, + "not": {"pattern": "^lightspeed\\."}, + }, + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": MAX_METADATA_VALUE_BYTES, + "pattern": crate::ids::NO_CONTROL_PATTERN, + }, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn caller_metadata_is_bounded_and_keeps_out_of_the_reserved_prefix() { + let mut metadata = BTreeMap::new(); + metadata.insert("harbor.trialId".to_owned(), "trial-1".to_owned()); + assert_eq!(validate_caller_metadata(&metadata), Ok(())); + metadata.insert("lightspeed.x".to_owned(), "y".to_owned()); + assert_eq!( + validate_caller_metadata(&metadata), + Err(MetadataError::ReservedPrefix { + key: "lightspeed.x".to_owned() + }) + ); + assert_eq!(validate_stored_metadata(&metadata), Ok(())); + metadata.clear(); + metadata.insert( + "k".to_owned(), + "a".repeat(MAX_METADATA_VALUE_BYTES as usize + 1), + ); + assert!(matches!( + validate_caller_metadata(&metadata), + Err(MetadataError::ValueBytes { .. }) + )); + assert_eq!( + validate_display_name("bad\nname"), + Err(MetadataError::DisplayName) + ); + assert_eq!(validate_display_name(""), Err(MetadataError::DisplayName)); + } + + #[test] + fn stored_metadata_does_not_count_reserved_keys() { + let mut metadata: BTreeMap = (0..MAX_METADATA_ENTRIES as usize) + .map(|index| (format!("k{index}"), "v".to_owned())) + .collect(); + metadata.insert("lightspeed.envd.version".to_owned(), "0.5.0".to_owned()); + assert_eq!(validate_stored_metadata(&metadata), Ok(())); + assert!(validate_caller_metadata(&metadata).is_err()); + } + + #[test] + fn wire_values_validate_on_decode() { + assert!(serde_json::from_str::("\"runner 7\"").is_ok()); + assert!(serde_json::from_str::("\"\"").is_err()); + assert!(serde_json::from_str::(r#"{"lightspeed.x": "y"}"#).is_err()); + assert!(serde_json::from_str::(r#"{"k": "v"}"#).is_ok()); + assert!(serde_json::from_str::("\"line one\\nline two\"").is_ok()); + assert!(serde_json::from_str::("\"bell\\u0007\"").is_err()); + let long = "x".repeat(MAX_DESCRIPTIVE_TEXT_BYTES as usize + 1); + assert!(DescriptiveText::new(long.clone()).is_err()); + let cut = DescriptiveText::lossy(&format!("a\u{7}{long}")).unwrap(); + assert_eq!(cut.as_str().len(), MAX_DESCRIPTIVE_TEXT_BYTES as usize); + assert!(cut.as_str().starts_with("a ")); + assert!(DescriptiveText::lossy("").is_none()); + } +} diff --git a/crates/environment-protocol/src/paths.rs b/crates/environment-protocol/src/paths.rs new file mode 100644 index 000000000..27242259d --- /dev/null +++ b/crates/environment-protocol/src/paths.rs @@ -0,0 +1,475 @@ +//! Path and digest values. +//! +//! **Path model.** A wire path is an absolute path in the environment's own +//! filesystem namespace, the one its processes see, not a path relative to +//! the daemon's filesystem root: with the root narrowed to `/workspace`, +//! `/workspace/src` names that directory and `/etc` is outside the root. +//! The root is not a chroot. +//! +//! **Confinement.** One rule for every data-plane method, a command's `cwd` +//! included. A path that is not the filesystem root or below it is refused +//! with `permissionDenied` (reason `outsideRoot`). Otherwise the daemon +//! strips the root prefix and resolves the remaining segments one by one +//! beneath the root's directory descriptor. A symlink met on the way is +//! followed only while its target stays inside the root; a link that leaves +//! it is refused the same way. No method addresses the daemon's state +//! directory: a path that names it, lies inside it or (for removes, renames, +//! writes and captures) contains it is refused with `permissionDenied` +//! (reason `daemonState`), and walks prune it. +//! +//! **Names.** Paths are UTF-8 and POSIX: `/` is the only separator, and +//! every other character except NUL, the backslash included, may appear in +//! a name. An entry whose name is not UTF-8 is skipped and counted wherever +//! a method enumerates entries, never returned lossily. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +use crate::{bytes::decode_hex_array, limits::MAX_PATH_BYTES}; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum PathError { + #[error("path is empty")] + Empty, + + #[error("path is {actual} bytes; the maximum is {max}")] + TooLong { actual: usize, max: u32 }, + + #[error("path must be absolute: {path}")] + NotAbsolute { path: String }, + + #[error("path must be relative: {path}")] + NotRelative { path: String }, + + #[error("path contains a NUL byte")] + NulByte, + + #[error("path has an empty, '.' or '..' segment or a trailing '/': {path}")] + NotNormalized { path: String }, + + #[error("entry name must be one segment other than '.' and '..': {name}")] + NotAName { name: String }, +} + +/// JSON Schema pattern of [`EnvironmentPath`]. +pub const ENVIRONMENT_PATH_PATTERN: &str = "^(/|(/(?!\\.\\.?(/|$))[^/\\u0000]+)+)$"; +/// JSON Schema pattern of [`TreePath`]. +pub const TREE_PATH_PATTERN: &str = + "^(\\.|(?!\\.\\.?(/|$))[^/\\u0000]+(/(?!\\.\\.?(/|$))[^/\\u0000]+)*)$"; +/// JSON Schema pattern of [`EntryName`]. +pub const ENTRY_NAME_PATTERN: &str = "^(?!\\.\\.?$)[^/\\u0000]+$"; +/// JSON Schema pattern of [`ContentDigest`]. +pub const CONTENT_DIGEST_PATTERN: &str = "^sha256:[0-9a-f]{64}$"; + +/// An absolute, normalized POSIX path in the environment: `/`, or `/` and +/// one or more segments joined by `/`. Empty, `.` and `..` segments and a +/// trailing `/` are rejected, never rewritten. At most 4096 bytes. +/// `defaultCwd`, `homeDirectory`, every `cwd` and every path parameter use +/// this type; see the module's path model and confinement rule. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct EnvironmentPath( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 4096), pattern(ENVIRONMENT_PATH_PATTERN)) + )] + String, +); + +impl EnvironmentPath { + pub fn new(path: impl Into) -> Result { + let path = path.into(); + check_common(&path)?; + if !path.starts_with('/') { + return Err(PathError::NotAbsolute { path }); + } + if path != "/" && !segments_normalized(&path[1..]) { + return Err(PathError::NotNormalized { path }); + } + Ok(Self(path)) + } + + pub fn root() -> Self { + Self("/".to_owned()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn is_root(&self) -> bool { + self.0 == "/" + } + + /// Path segments below the root. + pub fn segments(&self) -> impl Iterator { + self.0.split('/').filter(|segment| !segment.is_empty()) + } + + pub fn file_name(&self) -> Option<&str> { + self.segments().last() + } + + pub fn parent(&self) -> Option { + if self.is_root() { + return None; + } + let index = self.0.rfind('/').unwrap_or(0); + Some(if index == 0 { + Self::root() + } else { + Self(self.0[..index].to_owned()) + }) + } + + /// Append a tree path. An empty tree path returns `self`. + pub fn join(&self, tree: &TreePath) -> Result { + if tree.is_root() { + return Ok(self.clone()); + } + let joined = if self.is_root() { + format!("/{}", tree.as_str()) + } else { + format!("{}/{}", self.0, tree.as_str()) + }; + Self::new(joined) + } + + /// True when `self` is `base` or lies below it. + pub fn starts_with(&self, base: &EnvironmentPath) -> bool { + base.is_root() + || self.0 == base.0 + || (self.0.starts_with(&base.0) && self.0.as_bytes().get(base.0.len()) == Some(&b'/')) + } + + /// The tree path of `self` below `base`, or `None` when it is outside. + pub fn relative_to(&self, base: &EnvironmentPath) -> Option { + if !self.starts_with(base) { + return None; + } + let rest = if base.is_root() { + &self.0[1..] + } else { + self.0[base.0.len()..].trim_start_matches('/') + }; + Some(if rest.is_empty() { + TreePath::root() + } else { + TreePath(rest.to_owned()) + }) + } +} + +/// A relative, normalized path below a selected root: segments joined by +/// `/`, without empty, `.` or `..` segments. The single segment `.` names +/// the root itself. At most 4096 bytes. Used for scan, inventory and +/// transfer entries. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct TreePath( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 4096), pattern(TREE_PATH_PATTERN)) + )] + String, +); + +impl Default for TreePath { + fn default() -> Self { + Self::root() + } +} + +impl TreePath { + /// Wire spelling of the root. + pub const ROOT: &'static str = "."; + + pub fn new(path: impl Into) -> Result { + let path = path.into(); + if path == Self::ROOT { + return Ok(Self(path)); + } + check_common(&path)?; + if path.starts_with('/') { + return Err(PathError::NotRelative { path }); + } + if !segments_normalized(&path) { + return Err(PathError::NotNormalized { path }); + } + Ok(Self(path)) + } + + pub fn root() -> Self { + Self(Self::ROOT.to_owned()) + } + + /// The wire spelling: `.` for the root. + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn is_root(&self) -> bool { + self.0 == Self::ROOT + } + + /// Segments below the root; none for the root itself. + pub fn segments(&self) -> impl Iterator { + let path = if self.is_root() { "" } else { self.0.as_str() }; + path.split('/').filter(|segment| !segment.is_empty()) + } + + /// Number of segments: the root has depth 0, its direct children 1. + pub fn depth(&self) -> u32 { + self.segments().count() as u32 + } + + pub fn join(&self, segment: &str) -> Result { + if self.is_root() { + Self::new(segment) + } else { + Self::new(format!("{}/{segment}", self.0)) + } + } +} + +fn check_common(path: &str) -> Result<(), PathError> { + if path.is_empty() { + return Err(PathError::Empty); + } + if path.len() > MAX_PATH_BYTES as usize { + return Err(PathError::TooLong { + actual: path.len(), + max: MAX_PATH_BYTES, + }); + } + if path.contains('\0') { + return Err(PathError::NulByte); + } + Ok(()) +} + +fn segments_normalized(path: &str) -> bool { + path.split('/') + .all(|segment| !matches!(segment, "" | "." | "..")) +} + +macro_rules! path_string_impls { + ($name:ident) => { + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } + } + + impl FromStr for $name { + type Err = PathError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } + } + + impl TryFrom<&str> for $name { + type Error = PathError; + + fn try_from(value: &str) -> Result { + Self::new(value) + } + } + + impl TryFrom for $name { + type Error = PathError; + + fn try_from(value: String) -> Result { + Self::new(value) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } + } + }; +} + +/// One entry name: a single path segment, neither `.` nor `..`, without +/// `/` or NUL, at most 4096 bytes. `fs/readDirectory` entries use it. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct EntryName( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 4096), pattern(ENTRY_NAME_PATTERN)) + )] + String, +); + +impl EntryName { + pub fn new(name: impl Into) -> Result { + let name = name.into(); + check_common(&name)?; + if name.contains('/') || name == "." || name == ".." { + return Err(PathError::NotAName { name }); + } + Ok(Self(name)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +path_string_impls!(EnvironmentPath); +path_string_impls!(TreePath); +path_string_impls!(EntryName); + +/// `sha256:` followed by 64 lowercase hex digits: the digest of a file's raw +/// bytes, independent of its path and mode. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct ContentDigest( + #[cfg_attr( + feature = "schema", + schemars(with = "String", pattern(CONTENT_DIGEST_PATTERN)) + )] + [u8; 32], +); + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +#[error("content digest must be sha256: followed by 64 lowercase hex digits")] +pub struct DigestError; + +impl ContentDigest { + pub const PREFIX: &'static str = "sha256:"; + + pub fn from_sha256(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + pub fn sha256(&self) -> &[u8; 32] { + &self.0 + } + + /// The 64 lowercase hex digits without the prefix. + pub fn hex(&self) -> String { + crate::bytes::encode_hex(&self.0) + } + + pub fn parse(value: &str) -> Result { + let hex = value.strip_prefix(Self::PREFIX).ok_or(DigestError)?; + decode_hex_array::<32>(hex) + .map(Self) + .map_err(|_| DigestError) + } +} + +impl fmt::Display for ContentDigest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}{}", Self::PREFIX, self.hex()) + } +} + +impl FromStr for ContentDigest { + type Err = DigestError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } +} + +impl Serialize for ContentDigest { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for ContentDigest { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::parse(&value).map_err(D::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn environment_paths_are_rejected_rather_than_rewritten() { + for good in ["/", "/workspace", "/a/b.c/d"] { + assert_eq!(EnvironmentPath::new(good).unwrap().as_str(), good); + } + for bad in ["", "relative", "/../etc", "/a/./b", "/a//b", "/a/", "/a\0b"] { + assert!(EnvironmentPath::new(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn backslashes_are_ordinary_name_bytes() { + assert_eq!( + EnvironmentPath::new("/a\\b").unwrap().file_name(), + Some("a\\b") + ); + assert!(EntryName::new("a\\b").is_ok()); + assert!(TreePath::new("dir/a\\b").is_ok()); + } + + #[test] + fn tree_paths_are_relative_and_normalized() { + let root = TreePath::new(".").unwrap(); + assert!(root.is_root()); + assert_eq!(root.depth(), 0); + assert_eq!(serde_json::to_value(TreePath::root()).unwrap(), "."); + assert_eq!(TreePath::root().join("a").unwrap().as_str(), "a"); + assert_eq!(TreePath::new("a/b").unwrap().depth(), 2); + for bad in ["", "/a", "a/../b", "a//b", "./a", "a/", "a/."] { + assert!(TreePath::new(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn entry_names_are_one_segment() { + assert!(EntryName::new("notes.md").is_ok()); + for bad in ["", ".", "..", "a/b"] { + assert!(EntryName::new(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn paths_join_and_relativize() { + let base = EnvironmentPath::new("/workspace").unwrap(); + let tree = TreePath::new("src/lib.rs").unwrap(); + let joined = base.join(&tree).unwrap(); + assert_eq!(joined.as_str(), "/workspace/src/lib.rs"); + assert_eq!(joined.relative_to(&base), Some(tree)); + assert_eq!(base.relative_to(&base), Some(TreePath::root())); + assert_eq!(base.join(&TreePath::root()).unwrap(), base); + assert!( + EnvironmentPath::new("/workspace2") + .unwrap() + .relative_to(&base) + .is_none() + ); + assert_eq!(joined.parent().unwrap().as_str(), "/workspace/src"); + assert_eq!(base.parent().unwrap(), EnvironmentPath::root()); + } + + #[test] + fn digests_are_canonical_lowercase() { + let text = format!("sha256:{}", "ab".repeat(32)); + let digest: ContentDigest = serde_json::from_value(serde_json::json!(text)).unwrap(); + assert_eq!(digest.to_string(), text); + assert!(ContentDigest::parse(&format!("sha256:{}", "AB".repeat(32))).is_err()); + assert!(ContentDigest::parse(&"ab".repeat(32)).is_err()); + } +} diff --git a/crates/environment-protocol/src/registration.rs b/crates/environment-protocol/src/registration.rs index 8ee167db9..ff7adf296 100644 --- a/crates/environment-protocol/src/registration.rs +++ b/crates/environment-protocol/src/registration.rs @@ -1,306 +1,471 @@ -//! Key-based outbound registration: the control-channel messages an `envd` -//! exchanges with the environment gateway after dialing out. +//! The registration channel: the control connection an envd dials to the +//! gateway at [`CONNECT_PATH`](crate::routes::CONNECT_PATH) to register and +//! to be asked for reverse-dialed streams. It runs the shared JSON-RPC +//! envelope; the daemon is the client (it dialed), the gateway the server. +//! Both directions carry requests and notifications. It never carries +//! data-plane messages: every stream is a separate socket the daemon dials +//! on request. //! -//! The control connection carries registration, liveness, and requests to -//! open data connections. It never carries environment-protocol frames: -//! every worker route is served by a separate, reverse-dialed data socket -//! that runs the ordinary data protocol. +//! Sequence: +//! +//! 1. Gateway → daemon, notification [`registration/challenge`]: always the +//! first frame (see [`ChallengeParams`]). +//! 2. Daemon → gateway, request `registration/register`: signs the nonce +//! bound to the gateway origin. The response carries the receipt and +//! `heartbeatIntervalMs`; an error refuses admission and the gateway +//! closes the socket. +//! 3. Gateway → daemon, request `registration/openStream`, any number of +//! times: the daemon dials +//! [`gateway_stream_url`](crate::routes::gateway_stream_url) with the +//! token and answers once the stream socket is upgraded, or with an +//! error. +//! 4. Daemon → gateway, notification `registration/goodbye` before a +//! deliberate shutdown; gateway → daemon, notification +//! `registration/closed` before the gateway ends the session for good. +//! +//! **Frame rules.** +//! +//! - Text frames only. The gateway answers a binary frame as the envelope +//! says (`invalidRequest`, null id); the daemon closes the connection on +//! one. +//! - An unknown request is answered with `methodNotFound`; an unknown +//! notification is ignored and logged. A response to an id the receiver +//! never sent is ignored and logged. +//! - Before admission the gateway accepts only `registration/register`: +//! any other request is answered with `notInitialized` and the gateway +//! closes the connection; any notification closes it without an answer. +//! A `registration/register` after admission is `invalidRequest`, and +//! the connection stays open. +//! - Before it has the register response, the daemon accepts only the +//! challenge (first) and that response. Any request or other +//! notification from the gateway in that window is a protocol violation: +//! the daemon closes the connection and reconnects with backoff. The +//! gateway sends `registration/openStream` and `registration/closed` only +//! after it sent the register response. +//! +//! **Liveness.** The gateway pings every `heartbeatIntervalMs`; the daemon +//! answers. A peer that receives no frame for +//! [`HEARTBEAT_MISSED_INTERVALS`](crate::limits::HEARTBEAT_MISSED_INTERVALS) +//! intervals treats the connection as dead and closes it; the daemon then +//! reconnects with backoff. +//! +//! [`registration/challenge`]: CHALLENGE_METHOD use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest as _, Sha256}; +use thiserror::Error; -use crate::shared::{CURRENT_PROTOCOL_VERSION, ImplementationInfo, SecretString}; - -/// Public route an `envd` dials to register and hold its control connection. -pub const CONNECT_PATH: &str = "/environment-gateway/connect"; - -/// Public route an `envd` dials, presenting a one-time token as a bearer -/// header, to serve one worker route. -pub const DATA_PATH: &str = "/environment-gateway/data"; +use crate::{ + bytes::{SecretString, hex_value}, + error::ProtocolError, + handshake::ImplementationInfo, + ids::{ControlConnectionId, DaemonId, EnvironmentId, IncarnationId}, + jsonrpc::decode_params, + metadata::{ + DescriptiveText, DisplayName, Metadata, validate_caller_metadata, validate_display_name, + }, + routes::CanonicalOrigin, +}; -/// Domain separator the daemon signs together with the gateway's nonce, so a -/// signature over a nonce can never be confused with any other Ed25519 use -/// of the same key. -pub const REGISTRATION_SIGNATURE_DOMAIN: &[u8] = b"lightspeed-envd-registration/v1"; +/// Method name of the frozen first frame. +pub const CHALLENGE_METHOD: &str = "registration/challenge"; -/// Bytes of the gateway's per-connection challenge nonce. -pub const REGISTRATION_NONCE_BYTES: usize = 32; +/// Domain separator of the signed registration message. Its suffix numbers +/// the signed-message layout, not the protocol. +pub const REGISTRATION_SIGNATURE_DOMAIN: &[u8] = b"lightspeed-envd-registration/v2"; -pub const MAX_DISPLAY_NAME_BYTES: usize = 128; -pub const MAX_METADATA_ENTRIES: usize = 32; -pub const MAX_METADATA_KEY_BYTES: usize = 64; -pub const MAX_METADATA_VALUE_BYTES: usize = 256; -/// Metadata keys under this prefix are written by Lightspeed only. -pub const RESERVED_METADATA_PREFIX: &str = "lightspeed."; +hex_value!( + /// `nonce`: the gateway's 32 random bytes for one control connection. + ChallengeNonce, + 32, + "^[0-9a-f]{64}$" +); +hex_value!( + /// `daemonPublicKey`: a raw Ed25519 public key. + DaemonPublicKey, + 32, + "^[0-9a-f]{64}$" +); +hex_value!( + /// `signature`: an Ed25519 signature over + /// [`signed_registration_message`]. + Ed25519Signature, + 64, + "^[0-9a-f]{128}$" +); -/// Messages the gateway sends on the control connection. +/// `registration/challenge` params. +/// +/// **Frozen.** This is the first frame of every control connection and the +/// only frame a daemon of any version must parse to learn it is stale: it +/// only ever gains fields, is decoded tolerantly (unknown members are +/// ignored), and a receiver reads `protocolVersion` with +/// [`challenge_protocol_version`] before decoding anything else. The whole +/// frame is +/// `{"jsonrpc":"2.0","method":"registration/challenge","params":{"protocolVersion":N,"nonce":"<64 hex>"}}`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "type", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum GatewayControlMessage { - /// First frame after the upgrade. `nonce` is hex; the daemon signs - /// [`signed_registration_message`] of its raw bytes. - Challenge { - protocol_version: u32, - nonce: String, - }, - /// The daemon is admitted; the receipt identifies the environment it - /// now serves. `heartbeatIntervalMs` tells the daemon how often the - /// gateway pings. - Accepted { - #[serde(flatten)] - receipt: RegistrationReceipt, - heartbeat_interval_ms: u64, - }, - /// Registration failed. The gateway closes the socket after this frame. - Rejected { - code: RegistrationRejectionCode, - message: String, - }, - /// A worker route is waiting: dial `dataUrl` with `token` as a bearer - /// header and run the data protocol on that socket. - OpenData { - token: SecretString, - data_url: String, - }, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct ChallengeParams { + /// The gateway's `ENVD_PROTOCOL_VERSION`. + pub protocol_version: u32, + pub nonce: ChallengeNonce, } -/// Messages the daemon sends on the control connection. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "type", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum DaemonControlMessage { - Register(RegisterParams), +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum BootstrapError { + #[error("first frame is not JSON")] + NotJson, + #[error("first frame is not a registration/challenge notification")] + NotChallenge, + #[error("challenge carries no integer protocolVersion")] + NoVersion, +} + +/// Read the gateway's protocol version from a first frame without +/// decoding anything else, so a daemon of any version can tell it is stale +/// before it fails on fields it does not know. +pub fn challenge_protocol_version(frame: &str) -> Result { + let value: Value = serde_json::from_str(frame).map_err(|_| BootstrapError::NotJson)?; + if value.get("method").and_then(Value::as_str) != Some(CHALLENGE_METHOD) { + return Err(BootstrapError::NotChallenge); + } + value + .get("params") + .and_then(|params| params.get("protocolVersion")) + .and_then(Value::as_u64) + .and_then(|version| u32::try_from(version).ok()) + .ok_or(BootstrapError::NoVersion) } +/// `registration/register` params. `registrationKey`, `displayName` and +/// `metadata` are read only at a daemon identity's first admission; a +/// daemon sends the key until it is admitted once and never afterwards. +/// +/// The gateway decodes them with [`RegisterParams::decode`]: build facts, +/// a display name or metadata out of bounds refuse admission with +/// `invalidRequest`; anything else that does not decode is +/// `invalidParams`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RegisterParams { - #[serde(default = "default_protocol_version")] + /// The daemon's `ENVD_PROTOCOL_VERSION`. pub protocol_version: u32, - /// Required for a first-seen daemon public key; ignored for a known one. #[serde(default, skip_serializing_if = "Option::is_none")] pub registration_key: Option, - /// Lowercase hex of the raw 32-byte Ed25519 public key. - pub daemon_public_key: String, - /// Lowercase hex of the 64-byte Ed25519 signature over - /// [`signed_registration_message`] of the challenge nonce. - pub signature: String, + pub daemon_public_key: DaemonPublicKey, + /// Over [`signed_registration_message`] of the challenge nonce and the + /// canonical origin of the URL the daemon dialed. + pub signature: Ed25519Signature, #[serde(default, skip_serializing_if = "Option::is_none")] - pub display_name: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, + pub display_name: Option, + #[serde(default, skip_serializing_if = "Metadata::is_empty")] + pub metadata: Metadata, pub implementation: ImplementationInfo, } +impl RegisterParams { + /// Decode `registration/register` params the way the gateway answers + /// them. Descriptive values that have the right shape but break their + /// bounds (`implementation`, `displayName`, `metadata`) are + /// `invalidRequest`, naming the member in the message: admission is + /// refused, never granted with the value dropped. Every other decode + /// failure is `invalidParams` naming the field. + pub fn decode(params: &Value) -> Result { + if let Some(implementation) = params.get("implementation") + && let Some(error) = ImplementationInfo::bounds_violation(implementation) + { + return Err(ProtocolError::invalid_request(format!( + "implementation.{error}" + ))); + } + if let Some(Value::String(name)) = params.get("displayName") + && let Err(error) = validate_display_name(name) + { + return Err(ProtocolError::invalid_request(format!( + "displayName: {error}" + ))); + } + if let Some(metadata) = params.get("metadata") + && let Ok(entries) = BTreeMap::::deserialize(metadata) + && let Err(error) = validate_caller_metadata(&entries) + { + return Err(ProtocolError::invalid_request(format!("metadata: {error}"))); + } + decode_params(params) + } +} + +/// `registration/register` response: the daemon is admitted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RegisterResponse { + pub receipt: RegistrationReceipt, + /// How often the gateway pings; see the liveness rule. + pub heartbeat_interval_ms: u64, +} + /// What the daemon learns about the environment it serves. Written to the -/// receipt file and logged once; contains no secret material. +/// receipt file as is; contains no secret. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RegistrationReceipt { - pub environment_id: String, - pub incarnation_id: String, - pub daemon_id: String, - pub connection_id: String, - pub identity_mode: String, + pub environment_id: EnvironmentId, + pub incarnation_id: IncarnationId, + pub daemon_id: DaemonId, + /// The gateway's id for this control connection. + pub control_connection_id: ControlConnectionId, + pub identity_mode: RegistrationIdentityMode, + pub admission: Admission, } -/// Why a registration was refused. Terminal codes mean the daemon must not -/// retry with the same identity and configuration; the others are transient. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Registration-key policy for the daemon's environment. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub enum RegistrationRejectionCode { - UnsupportedProtocol, - InvalidRequest, - InvalidSignature, - InvalidRegistrationKey, - RegistrationKeyRevoked, - RegistrationKeyExpired, - /// A daemon identity the gateway has never seen presented no key. - UnknownDaemon, - /// The identity's environment was closed; the identity is spent. - EnvironmentClosed, - /// The identity is already bound in another universe. - IdentityInUse, - CapacityExhausted, - RateLimited, - Unavailable, +pub enum RegistrationIdentityMode { + /// The environment outlives disconnects. + Persistent, + /// The environment closes after it stays disconnected past its grace. + Ephemeral, } -impl RegistrationRejectionCode { - pub fn is_terminal(self) -> bool { - !matches!( - self, - Self::CapacityExhausted | Self::RateLimited | Self::Unavailable - ) - } +/// Whether this admission created the environment or rejoined it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum Admission { + /// First admission of this daemon identity: a new environment. + Created, + /// A known identity reconnected to its environment. + Reconnected, } -/// The bytes a daemon signs for one challenge. -pub fn signed_registration_message(nonce: &[u8]) -> Vec { - let mut message = Vec::with_capacity(REGISTRATION_SIGNATURE_DOMAIN.len() + nonce.len()); - message.extend_from_slice(REGISTRATION_SIGNATURE_DOMAIN); - message.extend_from_slice(nonce); - message +/// `registration/openStream` params: the gateway asks the daemon to dial +/// one stream. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OpenStreamParams { + /// One-time bearer token for the stream dial; short-lived. + pub token: SecretString, + pub stream: StreamSpec, } -/// Check the bounds on caller-supplied correlation metadata and display -/// name. Metadata is descriptive only; it never selects or authenticates. -/// Caller-supplied maps may not use the reserved prefix; stored records -/// may, because Lightspeed annotates them itself (see -/// [`validate_metadata_bounds`]). Sessions share this validator so a -/// session and the environment it ran in accept the same keys. -pub fn validate_registration_metadata( - display_name: Option<&str>, - metadata: &BTreeMap, -) -> Result<(), String> { - if let Some(name) = display_name - && (name.is_empty() - || name.len() > MAX_DISPLAY_NAME_BYTES - || name.chars().any(char::is_control)) - { - return Err(format!( - "display name must be 1..={MAX_DISPLAY_NAME_BYTES} bytes without control characters" - )); - } - validate_metadata_bounds(metadata)?; - if let Some(key) = metadata - .keys() - .find(|key| key.starts_with(RESERVED_METADATA_PREFIX)) - { - return Err(format!( - "metadata key {key:?} uses the reserved {RESERVED_METADATA_PREFIX} prefix" - )); - } - Ok(()) +/// What the dialed stream carries. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum StreamSpec { + /// The data plane, served by the daemon as on any data connection. + DataProtocol {}, } -/// Entry count, key and value bytes, and control characters: the bounds -/// every stored metadata map obeys, including the keys under the reserved -/// prefix that Lightspeed writes itself. -pub fn validate_metadata_bounds(metadata: &BTreeMap) -> Result<(), String> { - if metadata.len() > MAX_METADATA_ENTRIES { - return Err(format!( - "metadata has more than {MAX_METADATA_ENTRIES} entries" - )); - } - for (key, value) in metadata { - validate_metadata_entry(key, value)?; - } - Ok(()) +/// `registration/openStream` response, sent once the stream socket is +/// upgraded and served. A failed dial answers `unavailable`; the gateway +/// fails the waiting route at once. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OpenStreamResponse {} + +/// `registration/goodbye` params: the daemon is stopping deliberately. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GoodbyeParams { + pub reason: GoodbyeReason, } -/// Key and value bytes and control characters for one entry, independent of -/// how many entries a map may hold. -pub fn validate_metadata_entry(key: &str, value: &str) -> Result<(), String> { - if key.is_empty() || key.len() > MAX_METADATA_KEY_BYTES || key.chars().any(char::is_control) { - return Err(format!( - "metadata key {key:?} must be 1..={MAX_METADATA_KEY_BYTES} bytes without control characters" - )); - } - if value.is_empty() - || value.len() > MAX_METADATA_VALUE_BYTES - || value.chars().any(char::is_control) - { - return Err(format!( - "metadata value for {key:?} must be 1..={MAX_METADATA_VALUE_BYTES} bytes without control characters" - )); - } - Ok(()) +/// Why the daemon is leaving. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum GoodbyeReason { + /// `SIGTERM` or `SIGINT`. + Shutdown, } -pub fn encode_hex(bytes: &[u8]) -> String { - use std::fmt::Write as _; - let mut hex = String::with_capacity(bytes.len() * 2); - for byte in bytes { - let _ = write!(hex, "{byte:02x}"); - } - hex +/// `registration/closed` params: the gateway ends the session for good and +/// closes the socket after this frame. The daemon exits and does not +/// reconnect. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ClosedParams { + pub reason: ClosedReason, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, } -pub fn decode_hex(value: &str) -> Option> { - if !value.len().is_multiple_of(2) { - return None; - } - (0..value.len()) - .step_by(2) - .map(|index| u8::from_str_radix(&value[index..index + 2], 16).ok()) - .collect() +/// Why the gateway ended the session for good. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ClosedReason { + /// Another process with this daemon identity connected; the newest + /// connection wins and this daemon exits with its own status naming the + /// shared identity. + Superseded, + /// The identity's environment was closed; the identity is spent. + EnvironmentClosed, } -fn default_protocol_version() -> u32 { - CURRENT_PROTOCOL_VERSION +/// The bytes a daemon signs for one challenge: +/// [`REGISTRATION_SIGNATURE_DOMAIN`] ‖ nonce ‖ sha256(origin), where +/// `origin` is the [`canonical_origin`](crate::routes::canonical_origin) of +/// the gateway URL the daemon dialed; the gateway verifies against the +/// canonical origin of each public URL it is configured with. Binding the +/// origin stops a gateway the daemon was pointed at from relaying another +/// gateway's nonce. +pub fn signed_registration_message(nonce: &ChallengeNonce, origin: &CanonicalOrigin) -> Vec { + let origin_hash = Sha256::digest(origin.as_str().as_bytes()); + let mut message = Vec::with_capacity(REGISTRATION_SIGNATURE_DOMAIN.len() + 32 + 32); + message.extend_from_slice(REGISTRATION_SIGNATURE_DOMAIN); + message.extend_from_slice(nonce.as_bytes()); + message.extend_from_slice(&origin_hash); + message } #[cfg(test)] mod tests { + use serde_json::json; + use super::*; #[test] - fn control_messages_are_tagged_and_flatten_the_receipt() { - let accepted = GatewayControlMessage::Accepted { - receipt: RegistrationReceipt { - environment_id: "environment_a".to_owned(), - incarnation_id: "incarnation_a".to_owned(), - daemon_id: "daemon_a".to_owned(), - connection_id: "connection_a".to_owned(), - identity_mode: "ephemeral".to_owned(), - }, - heartbeat_interval_ms: 30_000, + fn the_challenge_version_is_read_before_anything_else() { + let future = json!({ + "jsonrpc": "2.0", + "method": CHALLENGE_METHOD, + "params": {"protocolVersion": 9, "nonce": 7, "newField": {"x": 1}}, + }); + assert_eq!(challenge_protocol_version(&future.to_string()), Ok(9)); + assert_eq!( + challenge_protocol_version("{\"method\":\"other\"}"), + Err(BootstrapError::NotChallenge) + ); + assert_eq!( + challenge_protocol_version("garbage"), + Err(BootstrapError::NotJson) + ); + } + + #[test] + fn the_challenge_decodes_tolerantly() { + let params: ChallengeParams = serde_json::from_value(json!({ + "protocolVersion": 3, + "nonce": "00".repeat(32), + "addedLater": true, + })) + .unwrap(); + assert_eq!(params.protocol_version, 3); + } + + #[test] + fn crypto_values_are_canonical_hex() { + let key = format!("\"{}\"", "ab".repeat(32)); + assert!(serde_json::from_str::(&key).is_ok()); + for bad in [ + format!("\"{}\"", "AB".repeat(32)), + format!("\"{}\"", "ab".repeat(31)), + format!("\"{}\u{e9}\"", "a".repeat(62)), + ] { + assert!( + serde_json::from_str::(&bad).is_err(), + "{bad}" + ); + } + } + + #[test] + fn the_signed_message_binds_the_origin() { + let nonce = ChallengeNonce::from_bytes([9; 32]); + let origin = |url| crate::routes::canonical_origin(url).unwrap(); + let first = signed_registration_message(&nonce, &origin("wss://a.example/x")); + let second = signed_registration_message(&nonce, &origin("wss://b.example")); + assert_eq!( + first, + signed_registration_message(&nonce, &origin("WSS://A.example:443/y")) + ); + assert!(first.starts_with(REGISTRATION_SIGNATURE_DOMAIN)); + assert_ne!(first, second); + assert_eq!(first.len(), REGISTRATION_SIGNATURE_DOMAIN.len() + 64); + } + + #[test] + fn stream_tokens_are_redacted_in_debug_output() { + let params = OpenStreamParams { + token: SecretString::new("secret-token"), + stream: StreamSpec::DataProtocol {}, }; - let json = serde_json::to_value(&accepted).expect("serialize"); - assert_eq!(json["type"], "accepted"); - assert_eq!(json["environmentId"], "environment_a"); - assert_eq!(json["heartbeatIntervalMs"], 30_000); - let parsed: GatewayControlMessage = serde_json::from_value(json).expect("parse"); - assert_eq!(parsed, accepted); - - let open = serde_json::json!({"type": "openData", "token": "t", "dataUrl": "wss://g/d"}); - let parsed: GatewayControlMessage = serde_json::from_value(open).expect("parse"); - assert!(matches!(parsed, GatewayControlMessage::OpenData { .. })); - assert!(!format!("{parsed:?}").contains("\"t\"")); + assert!(!format!("{params:?}").contains("secret-token")); + assert_eq!( + serde_json::to_value(¶ms).unwrap(), + json!({"token": "secret-token", "stream": {"kind": "dataProtocol"}}) + ); + } + + fn register_params() -> Value { + json!({ + "protocolVersion": 3, + "daemonPublicKey": "ab".repeat(32), + "signature": "cd".repeat(64), + "implementation": {"name": "lightspeed-envd", "version": "0.5.0"}, + }) } #[test] - fn rejection_codes_split_terminal_from_retryable() { - assert!(RegistrationRejectionCode::EnvironmentClosed.is_terminal()); - assert!(RegistrationRejectionCode::InvalidSignature.is_terminal()); - assert!(RegistrationRejectionCode::RegistrationKeyRevoked.is_terminal()); - assert!(!RegistrationRejectionCode::CapacityExhausted.is_terminal()); - assert!(!RegistrationRejectionCode::RateLimited.is_terminal()); - assert!(!RegistrationRejectionCode::Unavailable.is_terminal()); + fn out_of_bounds_descriptive_values_refuse_admission() { + assert!(RegisterParams::decode(®ister_params()).is_ok()); + let mut implementation = register_params(); + implementation["implementation"]["version"] = json!("v\u{7}"); + let mut name = register_params(); + name["displayName"] = json!("x".repeat(200)); + let mut metadata = register_params(); + metadata["metadata"] = json!({"lightspeed.daemon": "spoofed"}); + for params in [implementation, name, metadata] { + assert_eq!( + RegisterParams::decode(¶ms).unwrap_err().kind(), + crate::error::ProtocolErrorKind::InvalidRequest, + "{params}" + ); + } } #[test] - fn metadata_bounds_and_reserved_prefix_are_enforced() { - let mut metadata = BTreeMap::new(); - metadata.insert("harbor.trialId".to_owned(), "trial-1".to_owned()); - assert!(validate_registration_metadata(Some("worker"), &metadata).is_ok()); - metadata.insert("lightspeed.x".to_owned(), "y".to_owned()); - assert!(validate_registration_metadata(None, &metadata).is_err()); - metadata.clear(); - metadata.insert("k".to_owned(), "a".repeat(MAX_METADATA_VALUE_BYTES + 1)); - assert!(validate_registration_metadata(None, &metadata).is_err()); - assert!(validate_registration_metadata(Some("bad\nname"), &BTreeMap::new()).is_err()); - assert!(validate_registration_metadata(Some(""), &BTreeMap::new()).is_err()); + fn other_register_decode_failures_are_invalid_params() { + let mut key = register_params(); + key["daemonPublicKey"] = json!("AB".repeat(32)); + let mut extra = register_params(); + extra["dataUrl"] = json!("wss://elsewhere"); + let mut shape = register_params(); + shape["implementation"] = json!({"name": 7}); + for params in [key, extra, shape] { + assert_eq!( + RegisterParams::decode(¶ms).unwrap_err().kind(), + crate::error::ProtocolErrorKind::InvalidParams, + "{params}" + ); + } } #[test] - fn hex_round_trips_and_signed_message_is_domain_separated() { - let bytes = [0u8, 1, 0xab, 0xff]; - assert_eq!(encode_hex(&bytes), "0001abff"); - assert_eq!(decode_hex("0001abff").as_deref(), Some(&bytes[..])); - assert_eq!(decode_hex("abc"), None); - assert_eq!(decode_hex("zz"), None); - let message = signed_registration_message(&[9u8; 4]); - assert!(message.starts_with(REGISTRATION_SIGNATURE_DOMAIN)); - assert!(message.ends_with(&[9u8; 4])); + fn fieldless_stream_kinds_reject_unknown_members() { + assert!(serde_json::from_value::(json!({"kind": "dataProtocol"})).is_ok()); + assert!( + serde_json::from_value::(json!({"kind": "dataProtocol", "x": 1})).is_err() + ); } } diff --git a/crates/environment-protocol/src/routes.rs b/crates/environment-protocol/src/routes.rs new file mode 100644 index 000000000..70bef6aff --- /dev/null +++ b/crates/environment-protocol/src/routes.rs @@ -0,0 +1,464 @@ +//! Public route paths and the URL rules both peers must compute the same +//! way. + +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +use crate::{control::target::TargetOwner, ids::ProviderTargetId, limits::MAX_URL_BYTES}; + +/// Gateway route an envd dials to register and hold its control connection. +/// A gateway may be served under a path prefix (`wss://host/ls` plus this +/// path); the daemon is configured with the full connect URL. +pub const CONNECT_PATH: &str = "/environment-gateway/connect"; + +/// Gateway route an envd dials, with a stream token as a bearer header, to +/// serve one reverse-dialed stream. Always derived from the connect URL by +/// replacing its [`CONNECT_PATH`] suffix ([`gateway_stream_url`]), so it +/// shares the connect URL's origin and path prefix. +pub const DATA_PATH: &str = "/environment-gateway/data"; + +/// Gateway route serving the envd discovery document. +pub const DISCOVERY_PATH: &str = "/.well-known/lightspeed-envd"; + +/// JSON Schema pattern of [`DataRouteBase`]. +pub const DATA_ROUTE_BASE_PATTERN: &str = "^wss?://[^/?#@\\s]+(/[^?#\\s]*[^/?#\\s])?$"; + +/// JSON Schema pattern of [`PublicUrl`]. +pub const PUBLIC_URL_PATTERN: &str = "^https?://[^/?#@\\s]+([/?][^#\\s]*)?$"; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum UrlError { + #[error("URL has an unsupported scheme: {0}")] + Scheme(String), + + #[error("URL has no host: {0}")] + NoHost(String), + + #[error("URL must not carry user information: {0}")] + UserInfo(String), + + #[error("URL has an invalid port: {0}")] + Port(String), + + #[error("URL must not carry a query or fragment: {0}")] + QueryOrFragment(String), + + #[error("URL must not end with '/': {0}")] + TrailingSlash(String), + + #[error("URL is longer than {MAX_URL_BYTES} bytes")] + TooLong, + + #[error("URL contains whitespace or a control character: {0}")] + Character(String), + + #[error("gateway connect URL path must end with {CONNECT_PATH}: {0}")] + NotConnectPath(String), + + #[error("a controller reached over wss must advertise a wss data route: {0}")] + Downgrade(String), +} + +/// The parts of an absolute URL the protocol's rules look at. +struct UrlParts<'a> { + /// As written. + scheme: &'a str, + host: &'a str, + port: Option, + path: &'a str, + query: Option<&'a str>, + fragment: Option<&'a str>, +} + +fn split_url(url: &str) -> Result, UrlError> { + if url.len() > MAX_URL_BYTES as usize { + return Err(UrlError::TooLong); + } + if url + .chars() + .any(|character| character.is_whitespace() || character.is_control()) + { + return Err(UrlError::Character(url.to_owned())); + } + let (scheme, rest) = url + .split_once("://") + .ok_or_else(|| UrlError::Scheme(url.to_owned()))?; + let (rest, fragment) = match rest.split_once('#') { + Some((rest, fragment)) => (rest, Some(fragment)), + None => (rest, None), + }; + let (rest, query) = match rest.split_once('?') { + Some((rest, query)) => (rest, Some(query)), + None => (rest, None), + }; + let (authority, path) = match rest.find('/') { + Some(index) => rest.split_at(index), + None => (rest, ""), + }; + if authority.contains('@') { + return Err(UrlError::UserInfo(url.to_owned())); + } + let (host, port) = split_host_port(authority).ok_or_else(|| UrlError::Port(url.to_owned()))?; + if host.is_empty() || host == "[]" { + return Err(UrlError::NoHost(url.to_owned())); + } + Ok(UrlParts { + scheme, + host, + port, + path, + query, + fragment, + }) +} + +/// The canonical WebSocket origin of a URL, the value the registration +/// signature binds. Only [`canonical_origin`] builds one, so the daemon +/// (from the URL it dialed) and the gateway (from its configured public +/// URLs) cannot pass a raw URL where an origin belongs. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CanonicalOrigin(String); + +impl CanonicalOrigin { + pub fn as_str(&self) -> &str { + &self.0 + } + + /// True for a `wss` origin. + pub fn is_secure(&self) -> bool { + self.0.starts_with("wss://") + } +} + +impl fmt::Display for CanonicalOrigin { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +/// The canonical WebSocket origin of a URL: `ws` or `wss` (`http` maps to +/// `ws`, `https` to `wss`, any case), `://`, the lowercase host, and +/// `:port` only when it is not the scheme's default. Path, query and +/// fragment are dropped. +pub fn canonical_origin(url: &str) -> Result { + let parts = split_url(url)?; + let (scheme, default_port) = match parts.scheme.to_ascii_lowercase().as_str() { + "ws" | "http" => ("ws", 80), + "wss" | "https" => ("wss", 443), + _ => return Err(UrlError::Scheme(url.to_owned())), + }; + let host = parts.host.to_ascii_lowercase(); + Ok(CanonicalOrigin(match parts.port { + Some(port) if port != default_port => format!("{scheme}://{host}:{port}"), + _ => format!("{scheme}://{host}"), + })) +} + +/// The URL a daemon dials for a reverse-dialed stream: the canonical origin +/// of its configured gateway connect URL, the connect URL's path with its +/// [`CONNECT_PATH`] suffix replaced by [`DATA_PATH`]. A path prefix in front +/// of the gateway routes is kept. The gateway never names this URL, so a +/// stream can never be pointed at another host or a weaker scheme than the +/// control connection. A connect URL whose path does not end with +/// [`CONNECT_PATH`], or that carries a query or fragment, is refused. +pub fn gateway_stream_url(gateway_connect_url: &str) -> Result { + let parts = split_url(gateway_connect_url)?; + if parts.query.is_some() || parts.fragment.is_some() { + return Err(UrlError::QueryOrFragment(gateway_connect_url.to_owned())); + } + let prefix = parts + .path + .strip_suffix(CONNECT_PATH) + .ok_or_else(|| UrlError::NotConnectPath(gateway_connect_url.to_owned()))?; + Ok(format!( + "{}{prefix}{DATA_PATH}", + canonical_origin(gateway_connect_url)? + )) +} + +/// `routes.data` of a controller's `initialize`: the absolute `ws://` or +/// `wss://` base URL of the provider's data routes, validated on decode and +/// never rewritten. Lowercase scheme, a host and optional port, an optional +/// path that does not end with `/`, no user information, query or +/// fragment, at most 2048 bytes. A controller reached over `wss` must +/// advertise a `wss` base, so the provider credential never travels in +/// cleartext; the runtime checks it with +/// [`DataRouteBase::check_for_controller`]. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct DataRouteBase( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 2048), pattern(DATA_ROUTE_BASE_PATTERN)) + )] + String, +); + +impl DataRouteBase { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let parts = split_url(&value)?; + if !matches!(parts.scheme, "ws" | "wss") { + return Err(UrlError::Scheme(value)); + } + if parts.query.is_some() || parts.fragment.is_some() { + return Err(UrlError::QueryOrFragment(value)); + } + if parts.path.ends_with('/') { + return Err(UrlError::TrailingSlash(value)); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn is_secure(&self) -> bool { + self.0.starts_with("wss://") + } + + /// The downgrade rule: a controller whose URL is `wss` (or `https`) may + /// only advertise a `wss` base. The runtime checks it at every + /// `initialize` before it records the base. + pub fn check_for_controller(&self, controller_url: &str) -> Result<(), UrlError> { + if canonical_origin(controller_url)?.is_secure() && !self.is_secure() { + return Err(UrlError::Downgrade(self.0.clone())); + } + Ok(()) + } +} + +/// A public `http://` or `https://` URL a provider reports, such as a +/// target's ingress `endpoint`: lowercase scheme, a host and optional port, +/// optional path and query, no user information or fragment, at most 2048 +/// bytes. Validated on decode. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct PublicUrl( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 2048), pattern(PUBLIC_URL_PATTERN)) + )] + String, +); + +impl PublicUrl { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let parts = split_url(&value)?; + if !matches!(parts.scheme, "http" | "https") { + return Err(UrlError::Scheme(value)); + } + if parts.fragment.is_some() { + return Err(UrlError::QueryOrFragment(value)); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +macro_rules! url_impls { + ($name:ident) => { + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } + } + + impl std::str::FromStr for $name { + type Err = UrlError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } + } + }; +} + +url_impls!(DataRouteBase); +url_impls!(PublicUrl); + +/// A target's data route under a controller's advertised `routes.data` +/// base: the base followed by +/// `/{universeId}/{bindingId}/{environmentId}/{incarnationId}/{targetId}`, +/// each segment percent-encoded. +pub fn provider_data_route( + base: &DataRouteBase, + owner: &TargetOwner, + target_id: &ProviderTargetId, +) -> String { + format!( + "{}/{}/{}/{}/{}/{}", + base.as_str(), + encode_path_segment(owner.universe_id.as_str()), + encode_path_segment(owner.binding_id.as_str()), + encode_path_segment(owner.environment_id.as_str()), + encode_path_segment(owner.incarnation_id.as_str()), + encode_path_segment(target_id.as_str()), + ) +} + +/// Percent-encode one URL path segment: everything except RFC 3986 +/// unreserved characters. +pub fn encode_path_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +fn split_host_port(authority: &str) -> Option<(&str, Option)> { + if let Some(rest) = authority.strip_prefix('[') { + let (host, after) = rest.split_once(']')?; + let host = &authority[..host.len() + 2]; + return match after { + "" => Some((host, None)), + port => Some((host, Some(port.strip_prefix(':')?.parse().ok()?))), + }; + } + match authority.rsplit_once(':') { + Some((host, port)) => Some((host, Some(port.parse().ok()?))), + None => Some((authority, None)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn origins_are_canonical() { + for (url, origin) in [ + ( + "wss://GW.Example.com/environment-gateway/connect", + "wss://gw.example.com", + ), + ("wss://gw.example.com:443/x?y#z", "wss://gw.example.com"), + ("https://gw.example.com:8443", "wss://gw.example.com:8443"), + ( + "ws://127.0.0.1:18080/environment-gateway/connect", + "ws://127.0.0.1:18080", + ), + ("http://[::1]:80/", "ws://[::1]"), + ] { + assert_eq!(canonical_origin(url).unwrap().as_str(), origin, "{url}"); + } + for bad in [ + "ftp://x", + "wss://", + "wss://user@host", + "wss://host:port", + "host", + "wss://host/a b", + ] { + assert!(canonical_origin(bad).is_err(), "{bad}"); + } + } + + #[test] + fn stream_urls_derive_from_the_connect_url() { + assert_eq!( + gateway_stream_url("wss://gw.example.com/environment-gateway/connect").unwrap(), + "wss://gw.example.com/environment-gateway/data" + ); + assert_eq!( + gateway_stream_url("wss://GW.example.com:443/ls/environment-gateway/connect").unwrap(), + "wss://gw.example.com/ls/environment-gateway/data", + "a path prefix is kept" + ); + assert!(matches!( + gateway_stream_url("wss://gw.example.com/connect"), + Err(UrlError::NotConnectPath(_)) + )); + assert!(matches!( + gateway_stream_url("wss://gw.example.com/environment-gateway/connect?x=1"), + Err(UrlError::QueryOrFragment(_)) + )); + } + + #[test] + fn data_route_bases_are_validated_not_rewritten() { + for good in [ + "wss://p.example/routes", + "ws://127.0.0.1:9000", + "wss://p.example", + ] { + assert_eq!(DataRouteBase::new(good).unwrap().as_str(), good); + } + for bad in [ + "wss://p.example/routes/", + "wss://p.example/routes?x=1", + "WSS://p.example", + "https://p.example", + "wss://u@p.example", + "wss:///routes", + ] { + assert!(DataRouteBase::new(bad).is_err(), "{bad}"); + } + assert!(serde_json::from_str::("\"wss://p.example/\"").is_err()); + } + + #[test] + fn a_wss_controller_cannot_advertise_a_ws_data_route() { + let plain = DataRouteBase::new("ws://p.example/routes").unwrap(); + assert!(matches!( + plain.check_for_controller("wss://p.example/control"), + Err(UrlError::Downgrade(_)) + )); + assert!(plain.check_for_controller("ws://127.0.0.1/control").is_ok()); + let secure = DataRouteBase::new("wss://p.example/routes").unwrap(); + assert!( + secure + .check_for_controller("wss://p.example/control") + .is_ok() + ); + } + + #[test] + fn public_urls_allow_paths_and_queries() { + assert!(PublicUrl::new("https://app.example/x?y=1").is_ok()); + for bad in ["ftp://x", "https://x/#f", "https://u@x", "wss://x"] { + assert!(PublicUrl::new(bad).is_err(), "{bad}"); + } + } + + #[test] + fn data_routes_encode_opaque_target_ids() { + let owner = TargetOwner { + universe_id: "u1".parse().unwrap(), + binding_id: "primary".parse().unwrap(), + environment_id: "env_1".parse().unwrap(), + incarnation_id: "inc_1".parse().unwrap(), + }; + let target = ProviderTargetId::new("default/pod abc").unwrap(); + let base = DataRouteBase::new("wss://p.example/routes").unwrap(); + assert_eq!( + provider_data_route(&base, &owner, &target), + "wss://p.example/routes/u1/primary/env_1/inc_1/default%2Fpod%20abc" + ); + } +} diff --git a/crates/environment-protocol/src/serde_util.rs b/crates/environment-protocol/src/serde_util.rs new file mode 100644 index 000000000..659ebbc82 --- /dev/null +++ b/crates/environment-protocol/src/serde_util.rs @@ -0,0 +1,13 @@ +//! Serialization predicates shared by the wire types. + +pub(crate) fn is_false(value: &bool) -> bool { + !*value +} + +pub(crate) fn is_zero_u32(value: &u32) -> bool { + *value == 0 +} + +pub(crate) fn is_zero_u64(value: &u64) -> bool { + *value == 0 +} diff --git a/crates/environment-protocol/src/shared.rs b/crates/environment-protocol/src/shared.rs deleted file mode 100644 index 3c6f1924d..000000000 --- a/crates/environment-protocol/src/shared.rs +++ /dev/null @@ -1,461 +0,0 @@ -//! Shared protocol primitives. - -use std::{fmt, str::FromStr}; - -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as SerdeError}; -use thiserror::Error; - -pub const CURRENT_PROTOCOL_VERSION: u32 = 2; - -macro_rules! string_id { - ($name:ident) => { - #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] - #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] - #[serde(transparent)] - pub struct $name(pub String); - - impl $name { - pub fn new(value: impl Into) -> Self { - Self(value.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } - } - - impl fmt::Display for $name { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(formatter) - } - } - - impl From<&str> for $name { - fn from(value: &str) -> Self { - Self::new(value) - } - } - - impl From for $name { - fn from(value: String) -> Self { - Self::new(value) - } - } - }; -} - -string_id!(ProviderTargetId); -string_id!(EnvironmentConnectionId); -string_id!(ProcessId); -string_id!(JobId); - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum EnvironmentTransport { - WebSocket, - Http, - Stdio, - Ssh, - Provider { - #[serde(rename = "providerType")] - provider_type: String, - }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentDataConnection { - pub target_id: ProviderTargetId, - pub endpoint: String, - pub transport: EnvironmentTransport, - pub scope: EnvironmentScope, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_cwd: Option, - pub capabilities: EnvironmentCapabilities, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum EnvironmentScope { - Default, - #[serde(rename_all = "camelCase")] - Session { - session_id: String, - }, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentCapabilities { - #[serde(default)] - pub network: bool, - #[serde(default)] - pub filesystem_read: bool, - #[serde(default)] - pub filesystem_write: bool, - /// The environment endpoint implements the bounded `fs/searchText` operation natively, so - /// callers can avoid per-file transfer for recursive text search. - #[serde(default)] - pub filesystem_search: bool, - /// The environment endpoint implements the bounded `fs/globFiles` operation natively, so - /// callers can avoid per-directory transfer for recursive enumeration. - #[serde(default)] - pub filesystem_glob: bool, - /// The environment endpoint honors `offset`/`maxBytes` on `fs/readFile` and reports the - /// true file size, so oversized files are truncated at the source instead - /// of transferred in full. - #[serde(default)] - pub filesystem_ranged_read: bool, - /// Bounded inline capture; absent on older endpoints. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub filesystem_capture: bool, - /// Paged transfer sessions with whole-file digests and bounded byte exchanges. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub filesystem_transfer: bool, - /// Bounded filesystem inventory observations with conditional fingerprints. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub filesystem_scan: bool, - /// Bounded staged materialization; false for read-only endpoints. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub filesystem_materialize: bool, - #[serde(default)] - pub process_start: bool, - #[serde(default)] - pub process_stdin: bool, - #[serde(default)] - pub process_terminate: bool, - #[serde(default)] - pub process_output_polling: bool, - #[serde(default)] - pub process_output_notifications: bool, - #[serde(default)] - pub process_pty: bool, - #[serde(default)] - pub job_start: bool, - #[serde(default)] - pub job_list: bool, - #[serde(default)] - pub job_read: bool, - #[serde(default)] - pub job_cancel: bool, - #[serde(default)] - pub job_wait_hint: bool, - #[serde(default)] - pub job_dependencies: bool, - #[serde(default)] - pub job_queue_keys: bool, -} - -impl EnvironmentCapabilities { - pub fn filesystem(read: bool, write: bool) -> Self { - Self { - filesystem_read: read, - filesystem_write: write, - ..Self::default() - } - } - - pub fn with_filesystem_search(mut self) -> Self { - self.filesystem_search = true; - self - } - - pub fn with_process(mut self) -> Self { - self.process_start = true; - self.process_stdin = true; - self.process_terminate = true; - self.process_output_polling = true; - self.process_output_notifications = true; - self - } - - pub fn with_jobs(mut self) -> Self { - self.job_start = true; - self.job_list = true; - self.job_read = true; - self.job_cancel = true; - self.job_wait_hint = false; - self.job_dependencies = true; - self.job_queue_keys = true; - self - } -} - -/// What an endpoint says about its own build. Descriptive only: the protocol -/// version decides compatibility, and nothing here selects, authenticates, -/// or gates. The gateway records these facts on the environment row so a -/// deployment can see which build serves each environment. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ImplementationInfo { - pub name: String, - /// Release version of the build, for example `0.1.0`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub version: Option, - /// Full git commit the build was made from. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub git_sha: Option, - /// Rust target triple the build runs on. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub target: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ByteChunk(pub Vec); - -impl ByteChunk { - pub fn new(bytes: impl Into>) -> Self { - Self(bytes.into()) - } - - pub fn into_inner(self) -> Vec { - self.0 - } - - pub fn as_slice(&self) -> &[u8] { - &self.0 - } -} - -impl From> for ByteChunk { - fn from(value: Vec) -> Self { - Self(value) - } -} - -impl From<&[u8]> for ByteChunk { - fn from(value: &[u8]) -> Self { - Self(value.to_vec()) - } -} - -impl Serialize for ByteChunk { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&BASE64_STANDARD.encode(&self.0)) - } -} - -impl<'de> Deserialize<'de> for ByteChunk { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let encoded = String::deserialize(deserializer)?; - BASE64_STANDARD - .decode(encoded) - .map(Self) - .map_err(SerdeError::custom) - } -} - -/// Secret string carried over the environment data protocol. -/// -/// It serializes as a string because it is a wire value, but `Debug` is -/// redacted so ordinary structured logging does not print plaintext. -#[derive(Clone, PartialEq, Eq)] -pub struct SecretString(String); - -impl SecretString { - pub fn new(value: impl Into) -> Self { - Self(value.into()) - } - - pub fn expose(&self) -> &str { - &self.0 - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -impl fmt::Debug for SecretString { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("SecretString()") - } -} - -impl From for SecretString { - fn from(value: String) -> Self { - Self(value) - } -} - -impl From<&str> for SecretString { - fn from(value: &str) -> Self { - Self(value.to_owned()) - } -} - -impl Serialize for SecretString { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.0) - } -} - -impl<'de> Deserialize<'de> for SecretString { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - String::deserialize(deserializer).map(Self) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[cfg_attr(feature = "schema", schemars(with = "String"))] -pub struct EnvironmentPath { - normalized: String, -} - -impl EnvironmentPath { - pub fn new(path: impl AsRef) -> Result { - normalize_path(path.as_ref()).map(|normalized| Self { normalized }) - } - - pub fn root() -> Self { - Self { - normalized: "/".to_owned(), - } - } - - pub fn current_dir() -> Self { - Self { - normalized: ".".to_owned(), - } - } - - pub fn as_str(&self) -> &str { - &self.normalized - } - - pub fn is_absolute(&self) -> bool { - self.normalized.starts_with('/') - } -} - -impl fmt::Display for EnvironmentPath { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - self.normalized.fmt(formatter) - } -} - -impl FromStr for EnvironmentPath { - type Err = EnvironmentPathError; - - fn from_str(value: &str) -> Result { - Self::new(value) - } -} - -impl TryFrom<&str> for EnvironmentPath { - type Error = EnvironmentPathError; - - fn try_from(value: &str) -> Result { - Self::new(value) - } -} - -impl Serialize for EnvironmentPath { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> Deserialize<'de> for EnvironmentPath { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::new(value).map_err(SerdeError::custom) - } -} - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum EnvironmentPathError { - #[error("environment path is empty")] - Empty, - - #[error("environment path contains a NUL byte")] - NulByte, - - #[error("environment path must use '/' separators: {path}")] - NonUnixSeparator { path: String }, -} - -fn normalize_path(path: &str) -> Result { - if path.is_empty() { - return Err(EnvironmentPathError::Empty); - } - if path.contains('\0') { - return Err(EnvironmentPathError::NulByte); - } - if path.contains('\\') { - return Err(EnvironmentPathError::NonUnixSeparator { - path: path.to_owned(), - }); - } - - let absolute = path.starts_with('/'); - let mut segments = Vec::new(); - for segment in path.split('/') { - match segment { - "" | "." => {} - ".." => { - if let Some(last) = segments.last() - && *last != ".." - { - segments.pop(); - continue; - } - if !absolute { - segments.push(".."); - } - } - segment => segments.push(segment), - } - } - - if absolute { - if segments.is_empty() { - Ok("/".to_owned()) - } else { - Ok(format!("/{}", segments.join("/"))) - } - } else if segments.is_empty() { - Ok(".".to_owned()) - } else { - Ok(segments.join("/")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn byte_chunk_serializes_as_base64_string() { - let value = serde_json::to_value(ByteChunk::from(b"hello".as_slice())).expect("serialize"); - - assert_eq!(value, serde_json::json!("aGVsbG8=")); - } - - #[test] - fn environment_path_normalizes_like_agent_fs_path() { - let path = EnvironmentPath::new("src//./lib/../main.rs").expect("path"); - - assert_eq!(path.as_str(), "src/main.rs"); - } -} diff --git a/crates/environment-protocol/src/version.rs b/crates/environment-protocol/src/version.rs new file mode 100644 index 000000000..00a8406a1 --- /dev/null +++ b/crates/environment-protocol/src/version.rs @@ -0,0 +1,17 @@ +//! Protocol version numbers. +//! +//! Each number is matched exactly: a peer that states any other number is +//! refused with `versionMismatch`, and no range or negotiation exists. A +//! change to a plane's wire shape or to the meaning of an existing field or +//! method needs a new number for that plane. `release/metadata.env` carries +//! the same numbers and `scripts/release/verify-metadata.sh` checks them. + +/// Version of the data plane and the registration channel. +/// +/// Both share one number so a data-plane change also changes the number in +/// the gateway's `registration/challenge`, which is what triggers a +/// registered daemon's automatic upgrade. +pub const ENVD_PROTOCOL_VERSION: u32 = 3; + +/// Version of the provider controller plane. +pub const CONTROLLER_PROTOCOL_VERSION: u32 = 3; diff --git a/crates/environment-protocol/tests/contract.rs b/crates/environment-protocol/tests/contract.rs new file mode 100644 index 000000000..e772e0f1b --- /dev/null +++ b/crates/environment-protocol/tests/contract.rs @@ -0,0 +1,47 @@ +//! The committed `contract/` documents match the current wire types. When +//! this fails, regenerate with +//! `cargo run -p environment-protocol --features schema --bin export-contract` +//! and commit the result with the type change. + +use std::{fs, path::PathBuf}; + +use serde_json::Value; + +#[test] +fn committed_contract_is_current() { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("contract"); + for plane in environment_protocol::contract::export() { + for (name, current) in [ + (plane.schema_file_name(), &plane.schema_bundle), + (plane.openrpc_file_name(), &plane.openrpc), + ] { + let path = dir.join(&name); + let text = fs::read_to_string(&path).unwrap_or_else(|error| { + panic!( + "missing {}: {error}; run the export-contract binary", + path.display() + ) + }); + let committed: Value = serde_json::from_str(&text).expect("contract parses"); + assert_eq!( + &committed, current, + "{name} is stale; run the export-contract binary" + ); + } + } +} + +#[test] +fn every_method_appears_in_its_plane_document() { + for plane in environment_protocol::contract::export() { + let names: Vec<&str> = plane.openrpc["methods"] + .as_array() + .expect("methods") + .iter() + .filter_map(|method| method["name"].as_str()) + .collect(); + for spec in environment_protocol::manifest::plane_methods(plane.plane) { + assert!(names.contains(&spec.name), "{} missing", spec.name); + } + } +} diff --git a/crates/environment-protocol/tests/fixtures.rs b/crates/environment-protocol/tests/fixtures.rs new file mode 100644 index 000000000..60d3d7c8e --- /dev/null +++ b/crates/environment-protocol/tests/fixtures.rs @@ -0,0 +1,203 @@ +//! One fixture per method: params, response and an error, round-tripped +//! through the manifest's wire types under the strict decoding policy. + +use std::{collections::BTreeSet, fs, path::PathBuf}; + +use environment_protocol::{ + bootstrap::{BuildFacts, DiscoveryDocument}, + error::ProtocolError, + handshake::Plane, + manifest::{MessageKind, MethodSpec, RegistrationChallenge, methods}, +}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Value, json}; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures") +} + +fn fixture_file(spec: &MethodSpec) -> PathBuf { + let name = match spec.name.strip_prefix("$/") { + Some(rest) => format!("rpc/{rest}"), + None => spec.name.to_owned(), + }; + fixtures_dir() + .join(spec.plane.as_str()) + .join(format!("{}.json", name.replace('/', "."))) +} + +fn load(path: &PathBuf) -> Value { + let text = fs::read_to_string(path) + .unwrap_or_else(|error| panic!("missing fixture {}: {error}", path.display())); + serde_json::from_str(&text).unwrap_or_else(|error| panic!("{}: {error}", path.display())) +} + +fn round_trip(value: &Value) -> Value { + serde_json::to_value(serde_json::from_value::(value.clone()).expect("decode")) + .expect("encode") +} + +#[test] +fn every_method_has_fixtures_that_round_trip() { + for spec in methods() { + let path = fixture_file(spec); + let fixture = load(&path); + let params = &fixture["params"]; + assert_eq!( + (spec.normalize_params)(params) + .unwrap_or_else(|error| panic!("{} params: {error}", spec.name)), + *params, + "{} params are not canonical", + spec.name + ); + match spec.kind { + MessageKind::Request => { + let response = &fixture["response"]; + let normalize = spec.normalize_response.expect("requests have responses"); + assert_eq!( + normalize(response) + .unwrap_or_else(|error| panic!("{} response: {error}", spec.name)), + *response, + "{} response is not canonical", + spec.name + ); + let error = &fixture["error"]; + let decoded: ProtocolError = serde_json::from_value(error.clone()) + .unwrap_or_else(|decode| panic!("{} error: {decode}", spec.name)); + assert_eq!( + error["code"], + json!(decoded.code()), + "{} error code does not match its kind", + spec.name + ); + assert_eq!(serde_json::to_value(&decoded).unwrap(), *error); + } + MessageKind::Notification => { + assert!( + fixture.get("response").is_none() && fixture.get("error").is_none(), + "notification {} is never answered", + spec.name + ); + } + } + } +} + +#[test] +fn every_fixture_belongs_to_a_method() { + let expected: BTreeSet = methods().map(fixture_file).collect(); + for plane in [Plane::Data, Plane::Controller, Plane::Registration] { + for entry in fs::read_dir(fixtures_dir().join(plane.as_str())).expect("fixture dir") { + let path = entry.expect("entry").path(); + assert!( + expected.contains(&path), + "orphan fixture {}", + path.display() + ); + } + } +} + +/// Members whose values are free-form string maps rather than wire +/// structs. +const FREE_FORM_MAPS: &[&str] = &["metadata", "env", "secretEnv"]; + +/// The path of every JSON object inside `value` that is a wire struct, +/// `value` itself included. +fn object_paths(value: &Value, path: &mut Vec, paths: &mut Vec>) { + match value { + Value::Object(map) => { + paths.push(path.clone()); + for (key, child) in map { + if FREE_FORM_MAPS.contains(&key.as_str()) { + continue; + } + path.push(key.clone()); + object_paths(child, path, paths); + path.pop(); + } + } + Value::Array(items) => { + for (index, child) in items.iter().enumerate() { + path.push(index.to_string()); + object_paths(child, path, paths); + path.pop(); + } + } + _ => {} + } +} + +/// `value` with an unknown member added to the object at `path`. +fn with_unknown_member(value: &Value, path: &[String]) -> Value { + let mut value = value.clone(); + let mut target = &mut value; + for segment in path { + target = match target { + Value::Array(items) => &mut items[segment.parse::().expect("index")], + other => &mut other[segment.as_str()], + }; + } + target["unexpectedMember"] = json!(true); + value +} + +/// Every nested object of `value` refuses an unknown member under `decode`. +fn assert_strict(name: &str, part: &str, value: &Value, decode: impl Fn(&Value) -> bool) { + let mut paths = Vec::new(); + object_paths(value, &mut Vec::new(), &mut paths); + assert!(!paths.is_empty(), "{name} {part} has no object"); + for path in paths { + assert!( + !decode(&with_unknown_member(value, &path)), + "{name} {part} accepts an unknown member at /{}", + path.join("/") + ); + } +} + +#[test] +fn versioned_messages_reject_unknown_members_at_every_depth() { + for spec in methods() { + if spec.name == RegistrationChallenge::NAME { + continue; + } + let fixture = load(&fixture_file(spec)); + assert_strict(spec.name, "params", &fixture["params"], |value| { + (spec.normalize_params)(value).is_ok() + }); + if let Some(normalize) = spec.normalize_response { + assert_strict(spec.name, "response", &fixture["response"], |value| { + normalize(value).is_ok() + }); + assert_strict(spec.name, "error", &fixture["error"], |value| { + serde_json::from_value::(value.clone()).is_ok() + }); + } + } +} + +#[test] +fn the_frozen_challenge_tolerates_additions() { + let spec = methods() + .find(|spec| spec.name == RegistrationChallenge::NAME) + .expect("challenge"); + let mut params = load(&fixture_file(spec))["params"].clone(); + params["addedInALaterVersion"] = json!({"any": "shape"}); + assert!((spec.normalize_params)(¶ms).is_ok()); +} + +#[test] +fn bootstrap_documents_round_trip_and_tolerate_additions() { + let dir = fixtures_dir().join("bootstrap"); + let discovery = load(&dir.join("discovery-document.json")); + assert_eq!(round_trip::(&discovery), discovery); + let build = load(&dir.join("build-facts.json")); + assert_eq!(round_trip::(&build), build); + + let mut extended = discovery.clone(); + extended["addedLater"] = json!(1); + extended["artifacts"]["x86_64-unknown-linux-musl"]["url"] = Value::Null; + let decoded: DiscoveryDocument = serde_json::from_value(extended).expect("tolerant"); + assert!(decoded.artifacts["x86_64-unknown-linux-musl"].url.is_none()); +} diff --git a/crates/environment-protocol/tests/serde.rs b/crates/environment-protocol/tests/serde.rs index 2223b5652..a49ad17db 100644 --- a/crates/environment-protocol/tests/serde.rs +++ b/crates/environment-protocol/tests/serde.rs @@ -1,416 +1,182 @@ -use std::collections::BTreeMap; +//! Wire-shape rules that the per-method fixtures do not spell out. use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION, control::{ handshake::ControllerInitializeParams, - methods::{ - CREATE_TARGET_METHOD, INITIALIZE_METHOD as CONTROL_INITIALIZE_METHOD, - LIST_TARGETS_METHOD, SET_TARGET_POWER_METHOD, - }, - targets::{ - CreateTargetParams, CreateTargetResponse, ListTargetsResponse, PowerState, - ProviderBindingContext, ProviderTargetStatus, ProviderTargetSummary, - SetTargetPowerParams, - }, + target::{TargetObservation, TargetPhase, TargetResponse}, }, data::{ - fs::ReadFileResponse, - handshake::InitializeParams, - idle::IdleResponse, - jobs::{ - JobArtifact, JobDependency, JobDependencyPolicy, JobOutputChunk, JobOutputStream, - JobReadResult, JobStartSpec, JobStatus, JobSummary, ListJobsParams, ReadJobsResponse, - StartJobsParams, - }, - methods::{ - ENV_IDLE_METHOD, FS_READ_FILE_METHOD, INITIALIZE_METHOD, JOB_CANCEL_METHOD, - JOB_LIST_METHOD, JOB_READ_METHOD, JOB_START_METHOD, PROCESS_OUTPUT_METHOD, - PROCESS_START_METHOD, - }, - process::{ - LeftoverProcess, ProcessOutputChunk, ProcessOutputStream, ReadProcessResponse, - StartProcessParams, - }, - }, - shared::{ - ByteChunk, CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentConnectionId, - EnvironmentScope, JobId, ProcessId, ProviderTargetId, + exec::{ExecInput, ExitStatus, GapReason, OutputPage}, + handshake::{DataCapabilities, InitializeParams, InitializeResponse}, + job::{JobDependency, JobReadEntry, JobStatus, ReadJobsResponse, StartJobsParams}, + process::{ReadProcessParams, ReadProcessResponse}, + scan::{ScanResponse, ScanRootResult}, }, + error::{NotFoundReason, ProtocolError, ProtocolErrorKind}, + handshake::Plane, + manifest::{FsScan, ProcessRead, RequestMethod}, + registration::{RegisterParams, RegisterResponse}, }; -use serde::{Serialize, de::DeserializeOwned}; +use serde::de::DeserializeOwned; use serde_json::{Value, json}; -fn fixture(name: &str) -> Value { - serde_json::from_str(match name { - "data_initialize_params" => include_str!("../fixtures/data_initialize_params.json"), - "fs_read_file_response" => include_str!("../fixtures/fs_read_file_response.json"), - "process_start_params" => include_str!("../fixtures/process_start_params.json"), - "process_read_response" => include_str!("../fixtures/process_read_response.json"), - "job_start_params" => include_str!("../fixtures/job_start_params.json"), - "job_read_response" => include_str!("../fixtures/job_read_response.json"), - "controller_initialize_params" => { - include_str!("../fixtures/controller_initialize_params.json") - } - "controller_create_target_params" => { - include_str!("../fixtures/controller_create_target_params.json") - } - "controller_create_target_response" => { - include_str!("../fixtures/controller_create_target_response.json") - } - "controller_list_targets_response" => { - include_str!("../fixtures/controller_list_targets_response.json") - } - other => panic!("unknown fixture {other}"), - }) - .expect("fixture JSON") -} - -fn assert_round_trip(value: T, expected: Value) -where - T: Serialize + DeserializeOwned + PartialEq + std::fmt::Debug, -{ - let encoded = serde_json::to_value(&value).expect("serialize"); - assert_eq!(encoded, expected); - let decoded: T = serde_json::from_value(encoded).expect("deserialize"); - assert_eq!(decoded, value); -} - -#[test] -fn single_file_transfer_has_distinct_wire_semantics() { - use environment_protocol::{ - data::transfer_session::TransferSelection, shared::EnvironmentPath, - }; - assert_round_trip( - TransferSelection::WriteFile { - destination: EnvironmentPath::new("/workspace/output.bin").unwrap(), - }, - json!({"direction":"writeFile","destination":"/workspace/output.bin"}), - ); - let existing: TransferSelection = - serde_json::from_value(json!({"direction":"materialize","destination":"/workspace/tree"})) - .unwrap(); - assert_eq!( - serde_json::to_value(existing).unwrap(), - json!({"direction":"materialize","destination":"/workspace/tree","onExisting":"replace"}) - ); -} - -#[test] -fn method_names_match_data_plane_contract() { - assert_eq!(INITIALIZE_METHOD, "initialize"); - assert_eq!(FS_READ_FILE_METHOD, "fs/readFile"); - assert_eq!(PROCESS_START_METHOD, "process/start"); - assert_eq!(JOB_START_METHOD, "job/start"); - assert_eq!(JOB_LIST_METHOD, "job/list"); - assert_eq!(JOB_READ_METHOD, "job/read"); - assert_eq!(JOB_CANCEL_METHOD, "job/cancel"); - assert_eq!(PROCESS_OUTPUT_METHOD, "process/output"); +fn fixture(path: &str) -> Value { + let full = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join(path); + serde_json::from_str(&std::fs::read_to_string(&full).expect("fixture")).expect("JSON") } -#[test] -fn method_names_match_controller_plane_contract() { - assert_eq!(CONTROL_INITIALIZE_METHOD, "controller/initialize"); - assert_eq!(LIST_TARGETS_METHOD, "controller/listTargets"); - assert_eq!(CREATE_TARGET_METHOD, "controller/createTarget"); +fn decode(value: &Value) -> T { + serde_json::from_value(value.clone()).expect("decode") } #[test] -fn initialize_params_match_fixture() { - assert_round_trip( - InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-test".to_owned(), - scope: EnvironmentScope::Session { - session_id: "sandbox-123".to_owned(), - }, - resume_connection_id: Some(EnvironmentConnectionId::new("conn-prev")), - }, - fixture("data_initialize_params"), - ); +fn both_numbers_are_three() { + assert_eq!(ENVD_PROTOCOL_VERSION, 3); + assert_eq!(CONTROLLER_PROTOCOL_VERSION, 3); } #[test] -fn read_file_response_uses_base64_byte_chunk() { - assert_round_trip( - ReadFileResponse { - data: ByteChunk::from(b"hello\n".as_slice()), - file_size: None, - truncated: false, - }, - fixture("fs_read_file_response"), +fn handshakes_require_a_stated_protocol_version() { + let client_info = json!({"name": "x"}); + assert!( + serde_json::from_value::(json!({"clientInfo": client_info})).is_err() ); -} - -#[test] -fn process_start_params_match_fixture() { - assert_round_trip( - StartProcessParams { - process_id: ProcessId::new("proc-1"), - argv: vec!["sh".to_owned(), "-lc".to_owned(), "cat".to_owned()], - cwd: Some("/workspace".try_into().expect("cwd")), - env: BTreeMap::from([("RUST_LOG".to_owned(), "debug".to_owned())]), - secret_env: BTreeMap::new(), - stdin: Some(ByteChunk::from(b"input\n".as_slice())), - timeout_ms: Some(60_000), - tty: false, - }, - fixture("process_start_params"), + assert!( + serde_json::from_value::(json!({"clientInfo": client_info})) + .is_err() ); + let mut register = fixture("registration/registration.register.json")["params"].clone(); + register.as_object_mut().unwrap().remove("protocolVersion"); + assert!(serde_json::from_value::(register).is_err()); } #[test] -fn process_read_response_preserves_ordered_output_chunks() { - assert_round_trip( - ReadProcessResponse { - chunks: vec![ - ProcessOutputChunk { - seq: 1, - stream: ProcessOutputStream::Stdout, - chunk: ByteChunk::from(b"ok\n".as_slice()), - }, - ProcessOutputChunk { - seq: 2, - stream: ProcessOutputStream::Stderr, - chunk: ByteChunk::from(b"warn\n".as_slice()), - }, - ], - next_seq: 3, - exited: true, - exit_code: Some(0), - closed: true, - failure: None, - termination: None, - pid: Some(90), - omitted_bytes: 0, - leftover_processes: vec![LeftoverProcess { - pid: 91, - command: "python -m http.server 8080".to_owned(), - }], - }, - fixture("process_read_response"), - ); +fn initialize_names_the_plane_and_the_daemon_instance() { + let response: InitializeResponse = decode(&fixture("data/initialize.json")["response"]); + assert_eq!(response.plane, Plane::Data); + assert_eq!(response.daemon_instance_id.as_str(), "envd_01jb8x7k2m"); + assert!(response.capabilities.filesystem.unwrap().write); + assert!(response.limits.max_concurrent_requests >= 16); + let absent: DataCapabilities = decode(&json!({})); + assert!(absent.process.is_none(), "an absent family is unavailable"); } #[test] -fn job_start_params_match_fixture() { - assert_round_trip( - StartJobsParams { - namespace: "session_1".to_owned(), - request_id: "request-1".to_owned(), - jobs: vec![JobStartSpec { - job_id: JobId::new("checkout"), - name: Some("checkout".to_owned()), - argv: vec!["git".to_owned(), "status".to_owned()], - cwd: Some("/workspace".try_into().expect("cwd")), - env: BTreeMap::from([("RUST_LOG".to_owned(), "debug".to_owned())]), - secret_env: BTreeMap::new(), - stdin: Some(ByteChunk::from(b"hello\n".as_slice())), - timeout_ms: Some(60_000), - depends_on: vec![JobDependency::name("setup")], - dependency_policy: JobDependencyPolicy::AllSucceeded, - queue_key: Some("repo".to_owned()), - }], - }, - fixture("job_start_params"), - ); -} +fn process_reads_require_bounded_waits_and_report_byte_gaps() { + let mut params = fixture("data/process.read.json")["params"].clone(); + params.as_object_mut().unwrap().remove("waitMs"); + assert!(serde_json::from_value::(params).is_err()); -#[test] -fn job_read_response_matches_fixture() { - assert_round_trip( - ReadJobsResponse { - jobs: vec![JobReadResult { - summary: JobSummary { - namespace: "session_1".to_owned(), - job_id: JobId::new("tests"), - name: Some("tests".to_owned()), - status: JobStatus::Succeeded, - dependencies: vec![JobId::new("checkout")], - created_at_ms: 1, - queued_at_ms: Some(2), - started_at_ms: Some(3), - finished_at_ms: Some(4), - exit_code: Some(0), - orphaned_descendants: true, - failure: None, - queue_key: Some("repo".to_owned()), - }, - output_chunks: vec![JobOutputChunk { - seq: 1, - stream: JobOutputStream::Stdout, - chunk: ByteChunk::from(b"ok\n".as_slice()), - }], - output_next_seq: 2, - artifacts: vec![JobArtifact { - path: "/workspace/result.txt".try_into().expect("artifact path"), - kind: Some("file".to_owned()), - metadata: BTreeMap::from([("role".to_owned(), "report".to_owned())]), - }], - }], - }, - fixture("job_read_response"), - ); + let response: ReadProcessResponse = decode(&fixture("data/process.read.json")["response"]); + let OutputPage { + chunks, + gaps, + next, + end, + } = response.output; + let gap = gaps[0]; + assert_eq!(gap.reason, GapReason::Overflow); + assert_eq!(chunks[0].offset + chunks[0].data.len() as u64, gap.offset); + assert_eq!(gap.offset + gap.omitted_bytes, chunks[1].offset); + assert_eq!(next, end); + assert!(matches!( + response.exit, + Some(ExitStatus::Signal { + core_dumped: true, + .. + }) + )); } #[test] -fn job_list_params_use_session_namespace_and_limit() { - assert_round_trip( - ListJobsParams { - namespace: "session_1".to_owned(), - limit: Some(25), - }, - json!({ - "namespace": "session_1", - "limit": 25 - }), - ); -} - -#[test] -fn byte_chunk_rejects_invalid_base64() { - assert!(serde_json::from_value::(json!("not base64 !!!")).is_err()); -} - -#[test] -fn controller_initialize_params_match_fixture() { - assert_round_trip( - ControllerInitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-test".to_owned(), - }, - fixture("controller_initialize_params"), - ); -} - -#[test] -fn create_target_params_match_provider_fixture() { - assert_round_trip( - CreateTargetParams { - request_id: "request-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-1".to_owned(), - binding_id: "primary".to_owned(), - }, - template_id: "lightspeed-dev-v1".to_owned(), - }, - fixture("controller_create_target_params"), +fn job_entries_report_unknown_ids_as_errors() { + let response: ReadJobsResponse = decode(&fixture("data/job.read.json")["response"]); + let JobReadEntry::Error { job_id, error } = &response.jobs[2] else { + panic!("expected an error entry"); + }; + assert_eq!(job_id.as_str(), "gone"); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + let JobReadEntry::Job { summary, .. } = &response.jobs[0] else { + panic!("expected a job entry"); + }; + assert_eq!(summary.status, JobStatus::Failed); + assert!(summary.status.is_terminal()); + assert!(!JobStatus::Queued.is_terminal()); +} + +#[test] +fn job_dependencies_are_kind_tagged() { + let params: StartJobsParams = decode(&fixture("data/job.start.json")["params"]); + assert!(matches!( + ¶ms.jobs[1].depends_on[0], + JobDependency::Name { name } if name.as_str() == "setup" + )); + assert!( + serde_json::from_value::( + json!({"kind": "jobId", "jobId": "a", "name": "b"}) + ) + .is_err() + ); + let mut job = fixture("data/job.start.json")["params"].clone(); + job["jobs"][0]["namespace"] = json!("shared"); + assert!(serde_json::from_value::(job).is_err()); + let pipe = json!({"argv": ["cat"], "input": {"kind": "pipe"}}); + let spec: environment_protocol::data::exec::CommandSpec = decode(&pipe); + assert_eq!(spec.input, ExecInput::Pipe {}); +} + +#[test] +fn scan_roots_are_reported_independently() { + let response: ScanResponse = decode(&fixture("data/fs.scan.json")["response"]); + assert_eq!(response.roots.len(), 3); + assert!(matches!( + response.roots[1], + ScanRootResult::Unchanged { .. } + )); + let ScanRootResult::Complete { skipped, .. } = &response.roots[2] else { + panic!("expected a complete root"); + }; + assert!( + skipped[1].path.is_none(), + "non-UTF-8 names are counted, not returned" ); + assert_eq!(FsScan::PLANE, Plane::Data); } #[test] -fn create_target_response_carries_only_provider_lifecycle_state() { - assert_round_trip( - CreateTargetResponse { - target: ready_target_summary(), - }, - fixture("controller_create_target_response"), - ); +fn close_by_owner_answers_closed_without_a_target_id() { + let response: TargetResponse = decode(&fixture("controller/target.close.json")["response"]); + let TargetObservation { + target_id, state, .. + } = response.target; + assert!(target_id.is_none()); + assert_eq!(state.phase, TargetPhase::Closed); + assert!(state.power.is_none()); } #[test] -fn list_targets_response_matches_fixture() { - assert_round_trip( - ListTargetsResponse { - targets: vec![ready_target_summary()], - }, - fixture("controller_list_targets_response"), - ); -} - -fn ready_target_summary() -> ProviderTargetSummary { - ProviderTargetSummary { - target_id: ProviderTargetId::new("sandbox-123"), - display_name: Some("lightspeed dev".to_owned()), - status: ProviderTargetStatus::Ready, - scope: EnvironmentScope::Session { - session_id: "sandbox-123".to_owned(), - }, - capabilities: remote_environment_capabilities(), - default_cwd: Some("/workspace".try_into().expect("cwd")), - power_states: Vec::new(), - metadata: BTreeMap::from([("provider".to_owned(), "smolvm".to_owned())]), - } +fn the_receipt_is_nested_and_typed() { + let response: RegisterResponse = + decode(&fixture("registration/registration.register.json")["response"]); + assert_eq!(response.receipt.daemon_id.as_str(), "daemon_3b3b"); + let mut flattened = fixture("registration/registration.register.json")["response"].clone(); + flattened["environmentId"] = json!("env_1"); + assert!(serde_json::from_value::(flattened).is_err()); } #[test] -fn power_vocabulary_round_trips_and_maps_steady_states() { - assert_eq!(SET_TARGET_POWER_METHOD, "controller/setTargetPower"); - assert_eq!(ENV_IDLE_METHOD, "env/idle"); - let params = SetTargetPowerParams { - request_id: "power-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-1".to_owned(), - binding_id: "binding-1".to_owned(), - }, - target_id: ProviderTargetId::new("sandbox-123"), - power: PowerState::Suspended, - }; - assert_round_trip( - params, - json!({ - "requestId": "power-1", - "environmentId": "environment-1", - "incarnationId": "incarnation-1", - "binding": {"universeId": "universe-1", "bindingId": "binding-1"}, - "targetId": "sandbox-123", - "power": "suspended" - }), - ); - let mut summary = ready_target_summary(); - summary.status = ProviderTargetStatus::Paused; - summary.power_states = vec![PowerState::Running, PowerState::Paused]; - let encoded = serde_json::to_value(&summary).expect("serialize"); - assert_eq!(encoded["status"], json!("paused")); - assert_eq!(encoded["powerStates"], json!(["running", "paused"])); - // A summary from a provider that predates power control decodes with an - // empty state list. - let legacy: ProviderTargetSummary = - serde_json::from_value(serde_json::to_value(ready_target_summary()).unwrap()).unwrap(); - assert!(legacy.power_states.is_empty()); - assert_eq!( - ProviderTargetStatus::Suspended.power_state(), - Some(PowerState::Suspended) - ); - assert_eq!(ProviderTargetStatus::Starting.power_state(), None); - assert_round_trip( - IdleResponse { - idle_for_ms: 1_500, - running_processes: 0, - running_jobs: 2, - leftover_process_groups: 0, - }, - json!({"idleForMs": 1500, "runningProcesses": 0, "runningJobs": 2}), - ); -} - -fn remote_environment_capabilities() -> EnvironmentCapabilities { - EnvironmentCapabilities { - filesystem_read: true, - filesystem_write: true, - filesystem_search: true, - filesystem_glob: true, - filesystem_ranged_read: true, - filesystem_capture: false, - filesystem_transfer: false, - filesystem_scan: false, - filesystem_materialize: false, - process_start: true, - process_stdin: true, - process_terminate: true, - process_output_polling: true, - process_output_notifications: true, - process_pty: false, - job_start: true, - job_list: true, - job_read: true, - job_cancel: true, - job_wait_hint: false, - job_dependencies: true, - job_queue_keys: true, - network: true, - } +fn errors_carry_typed_data() { + let error: ProtocolError = decode(&fixture("data/process.read.json")["error"]); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + assert!(matches!( + error.data, + environment_protocol::error::ErrorData::NotFound { + reason: Some(NotFoundReason::Expired) + } + )); + assert!(!error.is_retryable()); + assert_eq!(ProcessRead::NAME, "process/read"); } diff --git a/crates/environment-provider-incus/CLUSTER.md b/crates/environment-provider-incus/CLUSTER.md index 99e2d752d..adcc873fb 100644 --- a/crates/environment-provider-incus/CLUSTER.md +++ b/crates/environment-provider-incus/CLUSTER.md @@ -49,7 +49,10 @@ incus cluster unset member-a scheduler.instance ## Verification -The non-destructive live topology/reconciliation test is opt-in: +The live tests are opt-in. The topology and reconciliation test is +non-destructive; `the_controller_conforms_on_live_incus` runs the controller +conformance suite and creates and deletes one VM in the universe +`lightspeed-conformance`, binding `incus-live`: ```bash LIGHTSPEED_INCUS_PROVIDER_CONFIG=/absolute/path/provider.json \ diff --git a/crates/environment-provider-incus/Cargo.toml b/crates/environment-provider-incus/Cargo.toml index 6ac582bff..db0ce284f 100644 --- a/crates/environment-provider-incus/Cargo.toml +++ b/crates/environment-provider-incus/Cargo.toml @@ -19,10 +19,15 @@ sha2 = "0.11" thiserror = "2" tokio = { version = "1", features = ["fs", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] } -uuid = { version = "1", features = ["v4"] } [dev-dependencies] +# The controller conformance suite runs against this server in unit tests +# (with an in-memory backend) and against live Incus. +environment-client = { path = "../environment-client", features = ["conformance"] } tempfile = "3" +# The data conformance suite runs against a guest envd on live Incus. +tools = { path = "../tools" } +uuid = { version = "1", features = ["v4"] } [[bin]] name = "lightspeed-provider-incus" diff --git a/crates/environment-provider-incus/README.md b/crates/environment-provider-incus/README.md index 1f48afeea..144fe2c1f 100644 --- a/crates/environment-provider-incus/README.md +++ b/crates/environment-provider-incus/README.md @@ -1,16 +1,22 @@ # Incus environment provider -`lightspeed-provider-incus` is a stateless controller, passive data endpoint, -optional application edge, and single-node/cluster Incus adapter. It depends -on the public host +`lightspeed-provider-incus` is a stateless controller, data-route relay, +optional application edge, and single-node/cluster Incus adapter. It serves +the controller plane of the environment protocol and depends on the public protocol crate, not Lightspeed's database, API implementation, harness, or Temporal runtime. -Start from `config.example.json`. Application-level authentication for both -the provider endpoint and private envd endpoint is intentionally deferred. -Protect those listeners with the deployment network/transport boundary for -now. The Incus client certificate and key remain deployment files mounted into -the provider process. +Start from `config.example.json`. The listener serves the controller on +`/control` and the data routes under `/routes/...`. Both require the provider +credential: `credentialFile` names a file holding a secret of at least 32 +bytes without whitespace (a trailing newline is ignored), which Lightspeed +presents as `Authorization: Bearer` and the provider compares in constant +time. A missing or wrong credential is refused with HTTP 401 before the +WebSocket upgrade. `GET /health` needs no credential. Register the same +credential with the provider in Lightspeed. The guest envd listener has no +authentication of its own; keep guest networks private. The Incus client +certificate and key remain deployment files mounted into the provider +process. ## Incus topology @@ -74,9 +80,15 @@ cargo run -p environment-provider-incus -- \ --config crates/environment-provider-incus/config.example.json ``` -Register its controller as `ws://127.0.0.1:19090/control`. Lightspeed derives -the sibling `/routes/...` endpoint and connects only while an environment is -being used; the provider then dials private envd and reaps idle connections. +Register its controller as `ws://127.0.0.1:19090/control` with the credential. +`dataRouteBase` is advertised as `routes.data` in the controller's +`initialize`: it must reach this listener's `/routes` from the environment +gateway, and it must use `wss://` when Lightspeed reaches the controller over +TLS. Lightspeed connects to a data route only while an environment is being +used; the provider then dials private envd and relays frames unchanged. The +relay forwards WebSocket pings, counts every frame as activity, and closes a +connection only after `relayIdleSeconds` without frames, which cannot be set +below 60. Registering the provider and creating an enabled binding in Lightspeed is sufficient to use it. Provider configuration contains no universe or binding allowlist. Every request carries a binding context, and the provider lazily @@ -108,3 +120,57 @@ bodies, and WebSockets to the approved private guest port. The provider records the realized hostname and port in `user.lightspeed.*` Incus target metadata so route operations remain ownership-fenced and reconstructible without a provider database or configured universe list. + +## Controller behavior + +Every controller method answers within about five seconds with the target's +current observation. Slow Incus work (an image pull, a boot, a graceful stop) +runs as one background job per target, serialized per target; later calls +report `provisioning` or `closing` while it runs, and a failed create reports +`failed` with the Incus error as the target's `message`. Job state is in +memory; Incus itself, read through the deterministic owner name, is the record +of every target, so a provider restart loses nothing. + +- `target/create` looks up the owner's instance before applying any creation + policy, so deprecating a template never affects existing targets. +- `target/get` only reads. A missing instance is `notFound`. +- `target/close` resolves the instance by owner when no target id is given, + stops it gracefully for up to 30 seconds, then forces the stop and deletes + the instance, its disk, and its ingress metadata. +- Templates report typed `resources`. `memory` and `disk` must be whole Incus + sizes such as `4GiB`; the provider converts them to bytes at startup. +- Cloud-init sets `LIGHTSPEED_ENVD_STATE_DIR=/var/lib/lightspeed-envd`, so the + daemon key and job records live outside the agent's `/workspace`. +- Incus API failures answer `unavailable`, which the runtime retries. +- The provider advertises `adoptTarget`, and `ingress` when an `ingress` block + is configured. + +## Adoption + +`target/adopt` takes over an existing Incus VM as a managed environment. The +`sourceTarget` is `instance` or `project/instance`; the project defaults to +`default`. Project and instance names are 1 to 63 ASCII letters, digits, and +hyphens, without a leading or trailing hyphen. Only persistent virtual +machines that are not already Lightspeed-managed can be adopted; containers +and ephemeral instances are refused with `notAdoptable`. + +Adoption moves the VM into the binding's project under the owner's +deterministic name and records ownership metadata. It replaces the VM's +profiles with the binding baseline, rewrites `eth0` onto the binding's managed +network, and disables every other local NIC. A running source is stopped +first. A later close destroys the adopted VM like any other target, so adopt +only machines whose disk Lightspeed may delete. + +## Conformance + +The ignored live test `the_controller_conforms_on_live_incus` runs the +environment client's controller conformance suite against this provider on a +live Incus server or cluster. It creates and deletes one VM from the first +non-deprecated template, using the fixed universe `lightspeed-conformance` and +binding `incus-live`: + +```bash +LIGHTSPEED_INCUS_PROVIDER_CONFIG=/absolute/path/provider.json \ + cargo test -p environment-provider-incus --test incus_live \ + the_controller_conforms_on_live_incus -- --ignored +``` diff --git a/crates/environment-provider-incus/config.cluster.example.json b/crates/environment-provider-incus/config.cluster.example.json index e0755c1ce..779c8b2d1 100644 --- a/crates/environment-provider-incus/config.cluster.example.json +++ b/crates/environment-provider-incus/config.cluster.example.json @@ -1,5 +1,7 @@ { "controllerListen": "127.0.0.1:19090", + "credentialFile": "/run/secrets/lightspeed-provider-credential", + "dataRouteBase": "ws://127.0.0.1:19090/routes", "incus": { "mode": "cluster", "endpoints": [ diff --git a/crates/environment-provider-incus/config.example.json b/crates/environment-provider-incus/config.example.json index 5fd78e2d0..8bfdedbb6 100644 --- a/crates/environment-provider-incus/config.example.json +++ b/crates/environment-provider-incus/config.example.json @@ -1,5 +1,7 @@ { "controllerListen": "127.0.0.1:19090", + "credentialFile": "/run/secrets/lightspeed-provider-credential", + "dataRouteBase": "ws://127.0.0.1:19090/routes", "incus": { "mode": "single", "endpoints": ["https://hz02.example.ts.net:8443"], diff --git a/crates/environment-provider-incus/src/config.rs b/crates/environment-provider-incus/src/config.rs index b07e40ef2..d15ac86e4 100644 --- a/crates/environment-provider-incus/src/config.rs +++ b/crates/environment-provider-incus/src/config.rs @@ -2,7 +2,19 @@ use std::{collections::BTreeMap, net::SocketAddr, path::PathBuf, sync::Arc}; use anyhow::{Context as _, bail}; use clap::Parser; +use environment_protocol::{ + bytes::SecretString, + control::template::TemplateResources, + ids::TemplateId, + limits::MIN_INTERMEDIARY_IDLE_MS, + metadata::{DescriptiveText, DisplayName}, + routes::DataRouteBase, +}; use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; + +/// Shortest provider credential accepted, in bytes. +const MIN_CREDENTIAL_BYTES: usize = 32; #[derive(Parser, Debug)] #[command(name = "lightspeed-provider-incus", version = release_info::LONG_VERSION)] @@ -15,6 +27,12 @@ pub struct ProviderArgs { #[serde(rename_all = "camelCase", deny_unknown_fields)] struct RawConfig { controller_listen: SocketAddr, + /// File holding the provider credential Lightspeed sends as + /// `Authorization: Bearer` on `/control` and every data route. + credential_file: PathBuf, + /// `routes.data` advertised in `initialize`: how Lightspeed reaches this + /// listener's `/routes`. + data_route_base: String, incus: IncusConfig, templates: Vec, #[serde(default)] @@ -26,13 +44,15 @@ struct RawConfig { } #[derive(Clone, Debug)] -pub struct Config(Arc); +pub struct Config(pub(crate) Arc); #[derive(Debug)] pub struct ConfigInner { pub controller_listen: SocketAddr, + pub credential: ProviderCredential, + pub data_route_base: DataRouteBase, pub incus: IncusConfig, - pub templates: BTreeMap, + pub templates: BTreeMap, pub network: NetworkPolicy, pub relay_idle_seconds: u64, pub dial_timeout_seconds: u64, @@ -74,12 +94,14 @@ pub struct NetworkPolicy { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct TemplatePolicy { - pub template_id: String, - pub display_name: String, - pub description: Option, + pub template_id: TemplateId, + pub display_name: DisplayName, + pub description: Option, pub image_fingerprint: String, pub cpu: u32, + /// Incus size string, such as `4GiB`. pub memory: String, + /// Incus size string, such as `40GiB`. pub disk: String, /// Optional native Incus cluster group used as the hard placement set. #[serde(default)] @@ -90,6 +112,9 @@ pub struct TemplatePolicy { pub ingress_port: Option, #[serde(default)] pub deprecated: bool, + /// `cpu`, `memory` and `disk` in provider-neutral units, derived at load. + #[serde(skip)] + pub resources: TemplateResources, } #[derive(Clone, Debug, Deserialize)] @@ -114,8 +139,21 @@ impl ProviderArgs { if raw.templates.is_empty() { bail!("provider must configure at least one template") } + let envd_port = raw.envd_port.unwrap_or(19091); + let relay_idle_seconds = raw + .relay_idle_seconds + .unwrap_or(MIN_INTERMEDIARY_IDLE_MS / 1000); + if relay_idle_seconds * 1000 < MIN_INTERMEDIARY_IDLE_MS { + bail!( + "relayIdleSeconds must be at least {}: relays never idle-close a data connection sooner", + MIN_INTERMEDIARY_IDLE_MS / 1000 + ) + } + let data_route_base = DataRouteBase::new(raw.data_route_base.clone()) + .with_context(|| format!("dataRouteBase {}", raw.data_route_base))?; + let credential = ProviderCredential::load(&raw.credential_file).await?; let mut templates = BTreeMap::new(); - for template in raw.templates { + for mut template in raw.templates { if template.cluster_group.is_some() && raw.incus.mode != IncusMode::Cluster { bail!( "template {} selects a clusterGroup in single mode", @@ -133,10 +171,7 @@ impl ProviderArgs { ) } if let Some(port) = template.ingress_port { - if port == 0 - || port == raw.envd_port.unwrap_or(19091) - || matches!(port, 22 | 2375 | 2376) - { + if port == 0 || port == envd_port || matches!(port, 22 | 2375 | 2376) { bail!( "template {} uses a reserved management port for ingress", template.template_id @@ -149,6 +184,17 @@ impl ProviderArgs { ) } } + template.resources = TemplateResources { + vcpus: Some(template.cpu), + memory_bytes: Some( + parse_incus_size(&template.memory) + .with_context(|| format!("template {} memory", template.template_id))?, + ), + disk_bytes: Some( + parse_incus_size(&template.disk) + .with_context(|| format!("template {} disk", template.template_id))?, + ), + }; let template_id = template.template_id.clone(); if templates.insert(template_id.clone(), template).is_some() { bail!("duplicate provider template: {template_id}") @@ -170,12 +216,14 @@ impl ProviderArgs { } Ok(Config(Arc::new(ConfigInner { controller_listen: raw.controller_listen, + credential, + data_route_base, incus: raw.incus, templates, network: raw.network, - relay_idle_seconds: raw.relay_idle_seconds.unwrap_or(60).max(1), + relay_idle_seconds, dial_timeout_seconds: raw.dial_timeout_seconds.unwrap_or(10).max(1), - envd_port: raw.envd_port.unwrap_or(19091), + envd_port, ingress: raw.ingress, }))) } @@ -184,6 +232,92 @@ impl ProviderArgs { } } +/// The provider credential: the bearer token Lightspeed presents on +/// `/control` and on every data route. `Debug` is redacted. +#[derive(Clone)] +pub struct ProviderCredential(SecretString); + +impl ProviderCredential { + pub fn new(value: impl Into) -> anyhow::Result { + let value = value.into(); + if value.len() < MIN_CREDENTIAL_BYTES + || value + .chars() + .any(|character| character.is_whitespace() || character.is_control()) + { + bail!( + "the provider credential must be at least {MIN_CREDENTIAL_BYTES} bytes without whitespace" + ) + } + Ok(Self(SecretString::new(value))) + } + + async fn load(path: &std::path::Path) -> anyhow::Result { + let contents = tokio::fs::read_to_string(path) + .await + .with_context(|| format!("read credentialFile {}", path.display()))?; + Self::new(contents.trim_end_matches(['\r', '\n'])) + .with_context(|| format!("credentialFile {}", path.display())) + } + + pub fn expose(&self) -> &str { + self.0.expose() + } + + /// True when `presented` is this credential. Compares fixed-size digests + /// so the time taken reveals neither the credential's length nor how + /// much of it matched. + pub fn matches(&self, presented: &str) -> bool { + let expected = Sha256::digest(self.0.expose().as_bytes()); + let presented = Sha256::digest(presented.as_bytes()); + expected + .iter() + .zip(presented.iter()) + .fold(0u8, |difference, (left, right)| difference | (left ^ right)) + == 0 + } +} + +impl std::fmt::Debug for ProviderCredential { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("ProviderCredential()") + } +} + +/// Bytes of an Incus size string: a whole number with an optional unit +/// (`B`, decimal `kB` to `EB`, binary `KiB` to `EiB`), as Incus accepts +/// for `limits.memory` and disk `size`. +fn parse_incus_size(value: &str) -> anyhow::Result { + const UNITS: [(&str, u64); 13] = [ + ("EiB", 1 << 60), + ("PiB", 1 << 50), + ("TiB", 1 << 40), + ("GiB", 1 << 30), + ("MiB", 1 << 20), + ("KiB", 1 << 10), + ("EB", 1_000_000_000_000_000_000), + ("PB", 1_000_000_000_000_000), + ("TB", 1_000_000_000_000), + ("GB", 1_000_000_000), + ("MB", 1_000_000), + ("kB", 1_000), + ("B", 1), + ]; + let (number, multiplier) = UNITS + .iter() + .find_map(|(unit, multiplier)| value.strip_suffix(unit).map(|number| (number, *multiplier))) + .unwrap_or((value, 1)); + if number.is_empty() || !number.bytes().all(|byte| byte.is_ascii_digit()) { + bail!("must be a whole number of bytes with an optional Incus unit such as GiB: {value}") + } + number + .parse::() + .ok() + .and_then(|number| number.checked_mul(multiplier)) + .filter(|bytes| *bytes > 0 && *bytes < (1 << 53)) + .ok_or_else(|| anyhow::anyhow!("size is out of range: {value}")) +} + fn validate_incus(incus: &IncusConfig) -> anyhow::Result<()> { if incus.endpoints.is_empty() { bail!("incus.endpoints must contain at least one HTTPS endpoint") @@ -256,13 +390,6 @@ impl std::ops::Deref for Config { } impl Config { - pub fn template(&self, template_id: &str) -> anyhow::Result<&TemplatePolicy> { - self.templates - .get(template_id) - .filter(|template| !template.deprecated) - .ok_or_else(|| anyhow::anyhow!("template is not offered by this provider")) - } - pub fn incus_mode_name(&self) -> &'static str { match self.incus.mode { IncusMode::Single => "single", @@ -275,88 +402,169 @@ impl Config { mod tests { use super::*; - #[tokio::test(flavor = "current_thread")] - async fn config_loads_provider_wide_templates_without_universe_bindings() { - let directory = tempfile::tempdir().expect("tempdir"); - let path = directory.path().join("provider.json"); - let document = serde_json::json!({ + const CREDENTIAL: &str = "0123456789abcdef0123456789abcdef"; + + fn base() -> serde_json::Value { + serde_json::json!({ "controllerListen":"127.0.0.1:0", + "dataRouteBase":"ws://127.0.0.1:19090/routes", "incus":{"mode":"single","endpoints":["https://incus.test"],"clientCertificatePem":"cert","clientPrivateKeyPem":"key","serverCaPem":"ca","storagePool":"default"}, "network":{"deniedEgressCidrs":["100.64.0.0/10"]}, "templates":[{"templateId":"dev-small-v1","displayName":"Small","imageFingerprint":"abc","cpu":2,"memory":"4GiB","disk":"40GiB"}] - }); + }) + } + + /// Write `document` (with `credentialFile` pointing at a fresh file + /// holding `credential`) and load it. + async fn load( + directory: &tempfile::TempDir, + mut document: serde_json::Value, + credential: &str, + ) -> anyhow::Result { + let credential_path = directory.path().join("credential"); + tokio::fs::write(&credential_path, format!("{credential}\n")) + .await + .unwrap(); + document["credentialFile"] = serde_json::json!(credential_path); + let path = directory.path().join("provider.json"); tokio::fs::write(&path, serde_json::to_vec(&document).unwrap()) .await .unwrap(); - let config = ProviderArgs { - config: path.clone(), - } - .load() - .await - .expect("config"); - assert!(config.template("dev-small-v1").is_ok()); + ProviderArgs { config: path }.load().await + } + + #[tokio::test(flavor = "current_thread")] + async fn config_loads_provider_wide_templates_without_universe_bindings() { + let directory = tempfile::tempdir().expect("tempdir"); + let config = load(&directory, base(), CREDENTIAL).await.expect("config"); + let template = &config.templates[&TemplateId::new("dev-small-v1").unwrap()]; + assert_eq!( + template.resources, + TemplateResources { + vcpus: Some(2), + memory_bytes: Some(4 << 30), + disk_bytes: Some(40 << 30), + } + ); assert_eq!(config.network.denied_egress_cidrs, ["100.64.0.0/10"]); - let mut duplicate = document; + assert_eq!(config.relay_idle_seconds, 60); + assert_eq!( + config.data_route_base.as_str(), + "ws://127.0.0.1:19090/routes" + ); + + let mut duplicate = base(); let first_template = duplicate["templates"][0].clone(); duplicate["templates"] .as_array_mut() .unwrap() .push(first_template); - tokio::fs::write(&path, serde_json::to_vec(&duplicate).unwrap()) - .await - .unwrap(); - assert!(ProviderArgs { config: path }.load().await.is_err()); + assert!(load(&directory, duplicate, CREDENTIAL).await.is_err()); } #[tokio::test(flavor = "current_thread")] async fn config_rejects_invalid_provider_network_cidr() { let directory = tempfile::tempdir().expect("tempdir"); - let path = directory.path().join("provider.json"); - let document = serde_json::json!({ - "controllerListen":"127.0.0.1:0", - "incus":{"mode":"single","endpoints":["https://incus.test"],"clientCertificatePem":"cert","clientPrivateKeyPem":"key","serverCaPem":"ca","storagePool":"default"}, - "network":{"deniedEgressCidrs":["not-a-cidr"]}, - "templates":[{"templateId":"dev-small-v1","displayName":"Small","imageFingerprint":"abc","cpu":2,"memory":"4GiB","disk":"40GiB"}] - }); - tokio::fs::write(&path, serde_json::to_vec(&document).unwrap()) - .await - .unwrap(); - assert!(ProviderArgs { config: path }.load().await.is_err()); + let mut document = base(); + document["network"]["deniedEgressCidrs"] = serde_json::json!(["not-a-cidr"]); + assert!(load(&directory, document, CREDENTIAL).await.is_err()); } #[tokio::test(flavor = "current_thread")] async fn config_requires_explicit_valid_mode_topology() { let directory = tempfile::tempdir().expect("tempdir"); - let path = directory.path().join("provider.json"); - let base = serde_json::json!({ - "controllerListen":"127.0.0.1:0", - "incus":{"mode":"single","endpoints":["https://one.test","https://two.test"],"clientCertificatePem":"cert","clientPrivateKeyPem":"key","serverCaPem":"ca","storagePool":"default"}, - "network":{"deniedEgressCidrs":[]}, - "templates":[{"templateId":"t","displayName":"T","imageFingerprint":"abc","cpu":1,"memory":"1GiB","disk":"10GiB"}] - }); - tokio::fs::write(&path, serde_json::to_vec(&base).unwrap()) - .await - .unwrap(); + let mut document = base(); + document["incus"]["endpoints"] = + serde_json::json!(["https://one.test", "https://two.test"]); assert!( - ProviderArgs { - config: path.clone() - } - .load() - .await - .is_err() + load(&directory, document.clone(), CREDENTIAL) + .await + .is_err() ); - let mut clustered = base; - clustered["incus"]["mode"] = serde_json::json!("cluster"); - clustered["incus"]["clusterNetworkUplink"] = serde_json::json!("UPLINK"); - clustered["templates"][0]["clusterGroup"] = serde_json::json!("x86-production"); - tokio::fs::write(&path, serde_json::to_vec(&clustered).unwrap()) - .await - .unwrap(); - let config = ProviderArgs { config: path } - .load() + document["incus"]["mode"] = serde_json::json!("cluster"); + document["incus"]["clusterNetworkUplink"] = serde_json::json!("UPLINK"); + document["templates"][0]["clusterGroup"] = serde_json::json!("x86-production"); + let config = load(&directory, document, CREDENTIAL) .await .expect("cluster config"); assert_eq!(config.incus_mode_name(), "cluster"); } + + #[tokio::test(flavor = "current_thread")] + async fn config_requires_a_strong_credential_and_a_valid_route_base() { + let directory = tempfile::tempdir().expect("tempdir"); + assert!(load(&directory, base(), "short").await.is_err()); + assert!( + load(&directory, base(), "0123456789abcdef 0123456789abcdef") + .await + .is_err() + ); + let mut document = base(); + document["dataRouteBase"] = serde_json::json!("http://127.0.0.1:19090/routes/"); + assert!(load(&directory, document, CREDENTIAL).await.is_err()); + let mut document = base(); + document.as_object_mut().unwrap().remove("credentialFile"); + document.as_object_mut().unwrap().remove("dataRouteBase"); + let path = directory.path().join("missing.json"); + tokio::fs::write(&path, serde_json::to_vec(&document).unwrap()) + .await + .unwrap(); + assert!(ProviderArgs { config: path }.load().await.is_err()); + } + + #[tokio::test(flavor = "current_thread")] + async fn relays_never_idle_close_below_the_protocol_minimum() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut document = base(); + document["relayIdleSeconds"] = serde_json::json!(59); + assert!( + load(&directory, document.clone(), CREDENTIAL) + .await + .is_err() + ); + document["relayIdleSeconds"] = serde_json::json!(1900); + let config = load(&directory, document, CREDENTIAL) + .await + .expect("config"); + assert_eq!(config.relay_idle_seconds, 1900); + } + + #[tokio::test(flavor = "current_thread")] + async fn template_sizes_and_descriptions_are_validated_at_load() { + let directory = tempfile::tempdir().expect("tempdir"); + for (field, value) in [ + ("memory", serde_json::json!("50%")), + ("disk", serde_json::json!("lots")), + ("displayName", serde_json::json!("")), + ("description", serde_json::json!("a\u{0}b")), + ("templateId", serde_json::json!("")), + ] { + let mut document = base(); + document["templates"][0][field] = value; + assert!( + load(&directory, document, CREDENTIAL).await.is_err(), + "{field}" + ); + } + } + + #[test] + fn incus_sizes_parse_decimal_and_binary_units() { + assert_eq!(parse_incus_size("4GiB").unwrap(), 4 << 30); + assert_eq!(parse_incus_size("512MB").unwrap(), 512_000_000); + assert_eq!(parse_incus_size("1024").unwrap(), 1024); + assert!(parse_incus_size("GiB").is_err()); + assert!(parse_incus_size("1.5GiB").is_err()); + assert!(parse_incus_size("0").is_err()); + } + + #[test] + fn credential_comparison_needs_the_exact_value() { + let credential = ProviderCredential::new(CREDENTIAL).unwrap(); + assert!(credential.matches(CREDENTIAL)); + assert!(!credential.matches(&CREDENTIAL[1..])); + assert!(!credential.matches("")); + assert!(!format!("{credential:?}").contains(CREDENTIAL)); + } } diff --git a/crates/environment-provider-incus/src/edge.rs b/crates/environment-provider-incus/src/edge.rs index e3812d70d..7a9bcb9e1 100644 --- a/crates/environment-provider-incus/src/edge.rs +++ b/crates/environment-provider-incus/src/edge.rs @@ -134,7 +134,7 @@ async fn resolve( fn resolve_targets(targets: Vec, hostname: &str) -> Option { for target in targets { if target.ingress_hostname.as_deref() == Some(hostname) - && target.status == environment_protocol::control::targets::ProviderTargetStatus::Ready + && target.status == crate::incus::InstanceStatus::Running && let (Some(address), Some(port)) = (target.ipv4_address, target.ingress_port) { return Some(Route { address, port }); @@ -188,32 +188,20 @@ async fn proxy_websocket(mut client: WebSocket, endpoint: String) { #[cfg(test)] mod tests { use super::*; - use crate::incus::OwnedTarget; + use crate::incus::{InstanceStatus, OwnedTarget}; use axum::{Router, extract::WebSocketUpgrade, routing::get}; - use environment_protocol::{control::targets::ProviderTargetStatus, shared::ProviderTargetId}; use std::time::Duration; - fn target(status: ProviderTargetStatus, hostname: Option<&str>) -> OwnedTarget { + fn target(status: InstanceStatus, hostname: Option<&str>) -> OwnedTarget { OwnedTarget { - target_id: ProviderTargetId::new("target-a"), - name: "target-a".to_owned(), - universe_id: "universe-a".to_owned(), - binding_id: "binding-a".to_owned(), - environment_id: "environment-a".to_owned(), - incarnation_id: "incarnation-a".to_owned(), - request_id: "request-a".to_owned(), - template_id: "template-a".to_owned(), - image_fingerprint: "fingerprint-a".to_owned(), - adoption_source: None, status, - ipv4_address: Some("10.0.0.2".to_owned()), ingress_hostname: hostname.map(ToOwned::to_owned), ingress_port: Some(8080), - location: None, + ..crate::ingress::tests::target() } } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn ingress_proxy_closes_upstream_when_client_disappears() { let upstream_listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await @@ -268,7 +256,7 @@ mod tests { fn authorization_resolves_only_the_exact_ready_ingress_hostname() { let hostname = "route.env.example.com"; let route = resolve_targets( - vec![target(ProviderTargetStatus::Ready, Some(hostname))], + vec![target(InstanceStatus::Running, Some(hostname))], hostname, ) .expect("ready route"); @@ -277,14 +265,14 @@ mod tests { assert!( resolve_targets( - vec![target(ProviderTargetStatus::Starting, Some(hostname))], + vec![target(InstanceStatus::Starting, Some(hostname))], hostname, ) .is_none() ); assert!( resolve_targets( - vec![target(ProviderTargetStatus::Ready, Some(hostname))], + vec![target(InstanceStatus::Running, Some(hostname))], "other.env.example.com", ) .is_none() diff --git a/crates/environment-provider-incus/src/incus.rs b/crates/environment-provider-incus/src/incus.rs index 53a94f72f..d256a7091 100644 --- a/crates/environment-provider-incus/src/incus.rs +++ b/crates/environment-provider-incus/src/incus.rs @@ -15,11 +15,10 @@ use serde_json::{Value, json}; use thiserror::Error; use environment_protocol::{ - control::targets::{ - AdoptTargetParams, CreateTargetParams, PowerState, ProviderBindingContext, - ProviderTargetStatus, + control::target::{BindingRef, PowerState, TargetOwner}, + ids::{ + BindingId, EnvironmentId, IncarnationId, ProviderNativeRef, ProviderTargetId, UniverseId, }, - shared::ProviderTargetId, }; use crate::{ @@ -27,19 +26,16 @@ use crate::{ policy, }; +/// A Lightspeed-managed Incus instance, read from its ownership metadata. #[derive(Clone, Debug)] pub struct OwnedTarget { + /// The instance name, which is the deterministic name of `owner`. pub target_id: ProviderTargetId, - pub name: String, - pub universe_id: String, - pub binding_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub request_id: String, + pub owner: TargetOwner, pub template_id: String, pub image_fingerprint: String, pub adoption_source: Option, - pub status: ProviderTargetStatus, + pub status: InstanceStatus, pub ipv4_address: Option, pub ingress_hostname: Option, pub ingress_port: Option, @@ -48,54 +44,94 @@ pub struct OwnedTarget { pub location: Option, } +impl OwnedTarget { + pub fn name(&self) -> &str { + self.target_id.as_str() + } + + pub fn binding(&self) -> BindingRef { + policy::binding_of(&self.owner) + } +} + +/// Incus instance status, reduced to what the provider acts on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InstanceStatus { + Running, + Starting, + Stopping, + Stopped, + /// `Frozen` or `Freezing`: a paused VM. + Frozen, + Error, + /// Any other status Incus reports. + Other, +} + +impl InstanceStatus { + fn from_incus(status: &str) -> Self { + match status { + "Running" | "Thawed" => Self::Running, + "Starting" => Self::Starting, + "Stopping" => Self::Stopping, + "Stopped" => Self::Stopped, + "Frozen" | "Freezing" => Self::Frozen, + "Error" => Self::Error, + _ => Self::Other, + } + } +} + +/// What an adoption source is, checked before any change is made. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AdoptionSource { + Missing, + /// A stopped or running, persistent, unmanaged virtual machine. + Adoptable, + /// Already managed by Lightspeed, so it belongs to some owner. + Managed, + /// It exists but cannot be taken over; the text says why. + NotAdoptable(&'static str), +} + +/// The Incus operations the controller needs. Mutations are slow (image +/// pulls, boots, graceful shutdowns); the controller runs them in the +/// background and observes the result with [`IncusBackend::get_owned`]. #[async_trait] pub trait IncusBackend: Clone + Send + Sync + 'static { async fn topology(&self) -> anyhow::Result; - async fn reconcile_binding(&self, binding: &ProviderBindingContext) -> anyhow::Result<()>; + async fn reconcile_binding(&self, binding: &BindingRef) -> anyhow::Result<()>; async fn ensure_image(&self, fingerprint: &str) -> anyhow::Result<()>; - async fn list_owned( - &self, - binding: &ProviderBindingContext, - ) -> anyhow::Result>; async fn list_all_owned(&self) -> anyhow::Result>; + /// The managed instance `name` in the binding's project. `None` when the + /// project or instance does not exist, when `name` is not an Incus + /// instance name, or when the instance carries no valid ownership + /// metadata. async fn get_owned( &self, - binding: &ProviderBindingContext, - target_id: &ProviderTargetId, + binding: &BindingRef, + name: &str, ) -> anyhow::Result>; - async fn create_vm( - &self, - binding: &ProviderBindingContext, - template: &TemplatePolicy, - params: &CreateTargetParams, - ) -> anyhow::Result; - async fn adopt_vm( - &self, - binding: &ProviderBindingContext, - params: &AdoptTargetParams, - ) -> anyhow::Result; - async fn delete_vm( - &self, - binding: &ProviderBindingContext, - target: &OwnedTarget, - force: bool, - ) -> anyhow::Result<()>; + /// Create the owner's instance from the template and start it. + async fn create_vm(&self, owner: &TargetOwner, template: &TemplatePolicy) + -> anyhow::Result<()>; + async fn adoption_source(&self, source: &ProviderNativeRef) -> anyhow::Result; + /// Move the source into the owner's project under the owner's name and + /// start it. + async fn adopt_vm(&self, owner: &TargetOwner, source: &ProviderNativeRef) + -> anyhow::Result<()>; + /// Destroy the instance: a graceful stop within 30 s, then a forced stop, + /// then delete. Its ingress metadata goes with it. + async fn delete_vm(&self, target: &OwnedTarget) -> anyhow::Result<()>; + /// Record (or with `None` clear) the realized ingress hostname and port. async fn set_ingress( &self, - binding: &ProviderBindingContext, target: &OwnedTarget, - hostname: Option<&str>, - port: Option, - ) -> anyhow::Result; - /// Converge one owned instance toward `power`. Idempotent by inventory; - /// transitional instances are reported as observed and converged on a - /// later call. - async fn set_power( - &self, - binding: &ProviderBindingContext, - target: &OwnedTarget, - power: PowerState, - ) -> anyhow::Result; + route: Option<(&str, u16)>, + ) -> anyhow::Result<()>; + /// Converge the instance toward `power`. A stopped VM asked to pause is + /// started; a later call freezes it. + async fn set_power(&self, target: &OwnedTarget, power: PowerState) -> anyhow::Result<()>; } /// Power states the Incus provider can move a VM to. Stateful stop @@ -103,6 +139,16 @@ pub trait IncusBackend: Clone + Send + Sync + 'static { pub const SUPPORTED_POWER_STATES: [PowerState; 3] = [PowerState::Running, PowerState::Paused, PowerState::Stopped]; +/// envd's state directory inside every VM: outside the agent's working tree +/// and writable by the service (`ReadWritePaths` in the image's unit). +pub const ENVD_STATE_DIR: &str = "/var/lib/lightspeed-envd"; + +/// Bound on one graceful Incus state action, in seconds. +const STATE_ACTION_TIMEOUT_SECONDS: i64 = 30; + +/// `template` and `image` metadata of an adopted target. +pub const ADOPTED: &str = "adopted"; + #[derive(Clone, Debug, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct IncusTopology { @@ -444,6 +490,9 @@ impl IncusClient { } async fn instance(&self, project: &str, name: &str) -> anyhow::Result> { + if !policy::is_instance_name(name) { + return Ok(None); + } if !self .exists(&format!("/instances/{name}"), Some(project)) .await? @@ -467,13 +516,7 @@ impl IncusClient { ) .await .unwrap_or_default(); - owned_from_instance(instance, state).map(Some) - } - - async fn observed_instance(&self, project: &str, name: &str) -> anyhow::Result { - self.instance(project, name) - .await? - .ok_or_else(|| anyhow::anyhow!("Incus target disappeared during a power change")) + Ok(owned_from_instance(instance, state)) } async fn instance_state_action( @@ -481,7 +524,7 @@ impl IncusClient { project: &str, name: &str, action: &str, - timeout: u64, + timeout: i64, force: bool, ) -> anyhow::Result<()> { self.request_unit( @@ -493,6 +536,44 @@ impl IncusClient { .await } + /// Run a state action; when it fails, accept it anyway if the instance + /// already reached a status in `reached` (a stale read, a transport + /// failure after Incus accepted the action, or a concurrent change). + async fn converge_state( + &self, + project: &str, + name: &str, + action: &str, + force: bool, + reached: &[InstanceStatus], + ) -> anyhow::Result<()> { + let Err(error) = self + .instance_state_action(project, name, action, STATE_ACTION_TIMEOUT_SECONDS, force) + .await + else { + return Ok(()); + }; + match self.instance(project, name).await? { + Some(target) if reached.contains(&target.status) => Ok(()), + None if action == "stop" => Ok(()), + _ => Err(error), + } + } + + /// Incus can briefly report the pre-operation `Stopped` status right + /// after a start operation succeeds. Wait a few seconds for the first + /// post-start status so the next observation is not stale. + async fn settle_after_start(&self, project: &str, name: &str) -> anyhow::Result<()> { + for _ in 0..10 { + match self.instance(project, name).await? { + Some(target) if target.status != InstanceStatus::Stopped => return Ok(()), + None => anyhow::bail!("the Incus instance disappeared while starting"), + Some(_) => tokio::time::sleep(Duration::from_millis(500)).await, + } + } + Ok(()) + } + async fn raw_instance(&self, project: &str, name: &str) -> anyhow::Result> { if !self .exists(&format!("/instances/{name}"), Some(project)) @@ -516,7 +597,7 @@ impl IncusClient { .await?; let mut networks = Vec::new(); for project in projects { - let Some(binding) = project.binding_context() else { + let Some(binding) = project.binding() else { continue; }; let network_name = policy::network_name(&binding); @@ -651,12 +732,12 @@ impl IncusBackend for IncusClient { }) } - async fn reconcile_binding(&self, binding: &ProviderBindingContext) -> anyhow::Result<()> { + async fn reconcile_binding(&self, binding: &BindingRef) -> anyhow::Result<()> { let _reconcile_guard = self.reconcile_lock.lock().await; let project = policy::project_name(binding); let isolated_project_networks = self.config.incus.mode == IncusMode::Cluster; let network_project = network_resource_project(self.config.incus.mode, &project); - self.ensure_resource(&format!("/projects/{project}"), "/projects", None, json!({"name":project,"description":"Lightspeed managed binding","config":{"features.images":"false","features.networks":isolated_project_networks.to_string(),"features.profiles":"true","restricted":"true","restricted.devices.nic":"managed","restricted.devices.disk":"managed","restricted.networks.access":policy::network_name(binding),"user.lightspeed.managed":"true","user.lightspeed.universe":binding.universe_id,"user.lightspeed.binding":binding.binding_id}})).await?; + self.ensure_resource(&format!("/projects/{project}"), "/projects", None, json!({"name":project,"description":"Lightspeed managed binding","config":{"features.images":"false","features.networks":isolated_project_networks.to_string(),"features.profiles":"true","restricted":"true","restricted.devices.nic":"managed","restricted.devices.disk":"managed","restricted.networks.access":policy::network_name(binding),"user.lightspeed.managed":"true","user.lightspeed.universe":binding.universe_id.as_str(),"user.lightspeed.binding":binding.binding_id.as_str()}})).await?; self.ensure_resource(&format!("/network-acls/{}", policy::acl_name(binding)), "/network-acls", network_project, json!({"name":policy::acl_name(binding),"description":"Lightspeed binding baseline","ingress":[],"egress":[],"config":{}})).await?; let network = match self.config.incus.mode { IncusMode::Single => { @@ -693,35 +774,6 @@ impl IncusBackend for IncusClient { Ok(()) } - async fn list_owned( - &self, - binding: &ProviderBindingContext, - ) -> anyhow::Result> { - let project = policy::project_name(binding); - if !self.exists(&format!("/projects/{project}"), None).await? { - return Ok(Vec::new()); - } - let instances: Vec = self - .request(Method::GET, "/instances?recursion=1", Some(&project), None) - .await?; - let mut targets = Vec::new(); - for instance in instances { - let state = self - .request( - Method::GET, - &format!("/instances/{}/state", instance.name), - Some(&project), - None, - ) - .await - .unwrap_or_default(); - if let Ok(target) = owned_from_instance(instance, state) { - targets.push(target); - } - } - Ok(targets) - } - async fn list_all_owned(&self) -> anyhow::Result> { let projects: Vec = self .request(Method::GET, "/projects?recursion=1", None, None) @@ -746,14 +798,10 @@ impl IncusBackend for IncusClient { ) .await .unwrap_or_default(); - if let Ok(target) = owned_from_instance(instance, state) { - let binding = ProviderBindingContext { - universe_id: target.universe_id.clone(), - binding_id: target.binding_id.clone(), - }; - if project.name == policy::project_name(&binding) { - targets.push(target); - } + if let Some(target) = owned_from_instance(instance, state) + && project.name == policy::project_name(&target.binding()) + { + targets.push(target); } } } @@ -762,68 +810,52 @@ impl IncusBackend for IncusClient { async fn get_owned( &self, - binding: &ProviderBindingContext, - target_id: &ProviderTargetId, + binding: &BindingRef, + name: &str, ) -> anyhow::Result> { - self.instance(&policy::project_name(binding), target_id.as_str()) - .await + if !policy::is_instance_name(name) { + return Ok(None); + } + let project = policy::project_name(binding); + if !self.exists(&format!("/projects/{project}"), None).await? { + return Ok(None); + } + Ok(self + .instance(&project, name) + .await? + .filter(|target| target.binding() == *binding)) } async fn create_vm( &self, - binding: &ProviderBindingContext, + owner: &TargetOwner, template: &TemplatePolicy, - params: &CreateTargetParams, - ) -> anyhow::Result { - let project = policy::project_name(binding); - let name = policy::instance_name( - &binding.universe_id, - &binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); + ) -> anyhow::Result<()> { + let binding = policy::binding_of(owner); + let project = policy::project_name(&binding); + let name = policy::instance_name(owner); let created_at = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH)? .as_millis() .to_string(); let cloud_init = format!( - "#cloud-config\nwrite_files:\n - path: /etc/lightspeed-envd/provider.env\n permissions: '0600'\n content: |\n LIGHTSPEED_ENVD_LISTEN=0.0.0.0:{}\n LIGHTSPEED_ENVD_CWD=/workspace\n LIGHTSPEED_ENVD_FS_ROOT=/\nruncmd:\n - [mkdir, -p, /workspace]\n - [systemctl, enable, --now, lightspeed-envd]\n", - self.config.envd_port + "#cloud-config\nwrite_files:\n - path: /etc/lightspeed-envd/provider.env\n permissions: '0600'\n content: |\n LIGHTSPEED_ENVD_LISTEN=0.0.0.0:{}\n LIGHTSPEED_ENVD_CWD=/workspace\n LIGHTSPEED_ENVD_FS_ROOT=/\n LIGHTSPEED_ENVD_STATE_DIR={}\nruncmd:\n - [mkdir, -p, /workspace]\n - [systemctl, enable, --now, lightspeed-envd]\n", + self.config.envd_port, ENVD_STATE_DIR, ); - let config = BTreeMap::from([ + let mut config = ownership_config(owner); + config.extend([ ("limits.cpu".to_owned(), template.cpu.to_string()), ("limits.memory".to_owned(), template.memory.clone()), ("cloud-init.user-data".to_owned(), cloud_init), - ("user.lightspeed.managed".to_owned(), "true".to_owned()), - ( - "user.lightspeed.universe".to_owned(), - params.binding.universe_id.clone(), - ), ( - "user.lightspeed.binding".to_owned(), - params.binding.binding_id.clone(), - ), - ( - "user.lightspeed.environment".to_owned(), - params.environment_id.clone(), - ), - ( - "user.lightspeed.incarnation".to_owned(), - params.incarnation_id.clone(), - ), - ( - "user.lightspeed.request".to_owned(), - params.request_id.clone(), - ), - ( - "user.lightspeed.template".to_owned(), - template.template_id.clone(), + format!("{}template", policy::META_PREFIX), + template.template_id.to_string(), ), ( - "user.lightspeed.image".to_owned(), + format!("{}image", policy::META_PREFIX), template.image_fingerprint.clone(), ), - ("user.lightspeed.created_at_ms".to_owned(), created_at), + (format!("{}created_at_ms", policy::META_PREFIX), created_at), ]); let path = if let Some(group) = &template.cluster_group { if !self @@ -836,112 +868,65 @@ impl IncusBackend for IncusClient { } else { "/instances".to_owned() }; - if let Err(create_error) = self.request_unit(Method::POST, &path, Some(&project), Some(json!({"name":name,"type":"virtual-machine","start":false,"profiles":[policy::profile_name(binding)],"source":{"type":"image","fingerprint":template.image_fingerprint},"config":config,"devices":{"root":{"type":"disk","pool":self.config.incus.storage_pool,"path":"/","size":template.disk}}}))).await { + if let Err(create_error) = self.request_unit(Method::POST, &path, Some(&project), Some(json!({"name":name,"type":"virtual-machine","start":false,"profiles":[policy::profile_name(&binding)],"source":{"type":"image","fingerprint":template.image_fingerprint},"config":config,"devices":{"root":{"type":"disk","pool":self.config.incus.storage_pool,"path":"/","size":template.disk}}}))).await // A transport can fail after Incus accepted the deterministic // create. Resolve that uncertainty before returning an error. - if let Some(target) = self.instance(&project, &name).await? { - return Ok(target); - } - return Err(create_error); - } - if let Err(error) = self - .request_unit( - Method::PUT, - &format!("/instances/{name}/state"), - Some(&project), - Some(json!({"action":"start","timeout":30,"force":false})), - ) - .await + && self.instance(&project, &name).await?.is_none() { - let observed = self.instance(&project, &name).await?; - if !observed.as_ref().is_some_and(|target| { - matches!( - target.status, - ProviderTargetStatus::Starting | ProviderTargetStatus::Ready - ) - }) { - return Err(error); - } + return Err(create_error); } - let target = self - .instance(&project, &name) - .await? - .ok_or_else(|| anyhow::anyhow!("created Incus instance disappeared"))?; - Ok(target_after_accepted_start(target)) + start_instance(self, &project, &name).await + } + + async fn adoption_source(&self, source: &ProviderNativeRef) -> anyhow::Result { + let (source_project, source_name) = parse_source_target(source.as_str())?; + let Some(source) = self.raw_instance(source_project, source_name).await? else { + return Ok(AdoptionSource::Missing); + }; + Ok(classify_adoption_source(&source)) } async fn adopt_vm( &self, - binding: &ProviderBindingContext, - params: &AdoptTargetParams, - ) -> anyhow::Result { - let project = policy::project_name(binding); - let destination = policy::instance_name( - &binding.universe_id, - &binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); - if let Some(target) = self.instance(&project, &destination).await? { - verify_adopted_target(&target, params)?; - return ensure_instance_started(self, &project, &destination, target).await; + owner: &TargetOwner, + source_target: &ProviderNativeRef, + ) -> anyhow::Result<()> { + let binding = policy::binding_of(owner); + let project = policy::project_name(&binding); + let destination = policy::instance_name(owner); + if self.instance(&project, &destination).await?.is_some() { + return start_instance(self, &project, &destination).await; } - let (source_project, source_name) = parse_source_target(¶ms.source_target)?; + let (source_project, source_name) = parse_source_target(source_target.as_str())?; let source = self .raw_instance(source_project, source_name) .await? .ok_or_else(|| anyhow::anyhow!("Incus adoption source target not found"))?; - if source.kind != "virtual-machine" { - anyhow::bail!("only Incus virtual-machine instances can be adopted") - } - if source.ephemeral { - anyhow::bail!("ephemeral Incus instances cannot be adopted") - } - if source - .config - .get(&format!("{}managed", policy::META_PREFIX)) - .is_some_and(|value| value == "true") - { - anyhow::bail!("Incus source target is already Lightspeed-managed") + match classify_adoption_source(&source) { + AdoptionSource::Adoptable => {} + AdoptionSource::Managed => { + anyhow::bail!("Incus source target is already Lightspeed-managed") + } + AdoptionSource::NotAdoptable(reason) => { + anyhow::bail!("Incus source target cannot be adopted: {reason}") + } + AdoptionSource::Missing => anyhow::bail!("Incus adoption source target not found"), } // Incus merges these overrides with the instance's existing local // configuration during a cross-project move. User configuration is // therefore preserved while ownership metadata is added. - let config = BTreeMap::from([ - (format!("{}managed", policy::META_PREFIX), "true".to_owned()), - ( - format!("{}universe", policy::META_PREFIX), - binding.universe_id.clone(), - ), - ( - format!("{}binding", policy::META_PREFIX), - binding.binding_id.clone(), - ), - ( - format!("{}environment", policy::META_PREFIX), - params.environment_id.clone(), - ), - ( - format!("{}incarnation", policy::META_PREFIX), - params.incarnation_id.clone(), - ), - ( - format!("{}request", policy::META_PREFIX), - params.request_id.clone(), - ), + let mut config = ownership_config(owner); + config.extend([ ( format!("{}template", policy::META_PREFIX), - "adopted".to_owned(), - ), - ( - format!("{}image", policy::META_PREFIX), - "adopted".to_owned(), + ADOPTED.to_owned(), ), + (format!("{}image", policy::META_PREFIX), ADOPTED.to_owned()), ( format!("{}adoption_source", policy::META_PREFIX), - params.source_target.clone(), + source_target.to_string(), ), ]); @@ -957,7 +942,7 @@ impl IncusBackend for IncusClient { let replacement = if name == "eth0" { json!({ "type":"nic", - "network":policy::network_name(binding), + "network":policy::network_name(&binding), "security.port_isolation":"true", "security.ipv4_filtering":"true" }) @@ -972,7 +957,7 @@ impl IncusBackend for IncusClient { Method::PUT, &format!("/instances/{source_name}/state"), Some(source_project), - Some(json!({"action":"stop","timeout":30,"force":true})), + Some(json!({"action":"stop","timeout":STATE_ACTION_TIMEOUT_SECONDS,"force":true})), ) .await .context("stop Incus source target before adoption")?; @@ -992,7 +977,7 @@ impl IncusBackend for IncusClient { "instance_only":false, "config":config, "devices":devices, - "profiles":[policy::profile_name(binding)] + "profiles":[policy::profile_name(&binding)] })), ) .await @@ -1004,158 +989,138 @@ impl IncusBackend for IncusClient { .instance(&project, &destination) .await? .ok_or_else(|| anyhow::anyhow!("adopted Incus target is not visible"))?; - verify_adopted_target(&target, params)?; - ensure_instance_started(self, &project, &destination, target).await + if target.owner != *owner + || target.adoption_source.as_deref() != Some(source_target.as_str()) + { + anyhow::bail!("adopted Incus target carries unexpected ownership metadata") + } + start_instance(self, &project, &destination).await } - async fn delete_vm( - &self, - binding: &ProviderBindingContext, - target: &OwnedTarget, - force: bool, - ) -> anyhow::Result<()> { - let project = policy::project_name(binding); - if target.status != ProviderTargetStatus::Stopped { - let stop_result = self - .request_unit( - Method::PUT, - &format!("/instances/{}/state", target.name), - Some(&project), - Some(json!({ - "action":"stop", - "timeout": if force { 0 } else { 30 }, - "force": force - })), - ) + async fn delete_vm(&self, target: &OwnedTarget) -> anyhow::Result<()> { + let project = policy::project_name(&target.binding()); + let name = target.name(); + if target.status != InstanceStatus::Stopped { + // Graceful first; a guest that ignores ACPI power-down (a hung + // kernel, an emergency shell) is then stopped by force, since its + // disk is destroyed either way. + let graceful = self + .converge_state(&project, name, "stop", false, &[InstanceStatus::Stopped]) .await; - if let Err(error) = stop_result { - let observed = self.instance(&project, &target.name).await?; - if observed - .as_ref() - .is_some_and(|target| target.status != ProviderTargetStatus::Stopped) - { - return Err(error); - } + if graceful.is_err() { + self.converge_state(&project, name, "stop", true, &[InstanceStatus::Stopped]) + .await + .context("force-stop Incus instance before delete")?; } } if let Err(error) = self .request_unit( Method::DELETE, - &format!("/instances/{}", target.name), + &format!("/instances/{name}"), Some(&project), None, ) .await - && self.instance(&project, &target.name).await?.is_some() + && self.instance(&project, name).await?.is_some() { return Err(error); } Ok(()) } - async fn set_power( - &self, - binding: &ProviderBindingContext, - target: &OwnedTarget, - power: PowerState, - ) -> anyhow::Result { + async fn set_power(&self, target: &OwnedTarget, power: PowerState) -> anyhow::Result<()> { if !SUPPORTED_POWER_STATES.contains(&power) { anyhow::bail!("Incus provider does not support power state {power}") } - let project = policy::project_name(binding); - let name = target.name.clone(); + let project = policy::project_name(&target.binding()); + let name = target.name(); match (target.status, power) { - // Already there or already on the way: report the observation. - (ProviderTargetStatus::Ready | ProviderTargetStatus::Starting, PowerState::Running) - | (ProviderTargetStatus::Paused, PowerState::Paused) - | (ProviderTargetStatus::Stopped, PowerState::Stopped) => Ok(target.clone()), - (ProviderTargetStatus::Paused, PowerState::Running) => { - self.instance_state_action(&project, &name, "unfreeze", 30, false) - .await?; - let observed = self.observed_instance(&project, &name).await?; - Ok(target_after_accepted_resume(observed)) + // Already there or already on the way. + (InstanceStatus::Running | InstanceStatus::Starting, PowerState::Running) + | (InstanceStatus::Frozen, PowerState::Paused) + | (InstanceStatus::Stopped | InstanceStatus::Stopping, PowerState::Stopped) => Ok(()), + (InstanceStatus::Frozen, PowerState::Running) => { + self.converge_state( + &project, + name, + "unfreeze", + false, + &[InstanceStatus::Running], + ) + .await } - (ProviderTargetStatus::Stopped, PowerState::Running | PowerState::Paused) => { + (InstanceStatus::Stopped, PowerState::Running | PowerState::Paused) => { // A stopped VM cannot be frozen directly; converge through // running and let a later call freeze it. - ensure_instance_started(self, &project, &name, target.clone()).await + start_instance(self, &project, name).await } - (ProviderTargetStatus::Ready, PowerState::Paused) => { - self.instance_state_action(&project, &name, "freeze", 30, false) - .await?; - let observed = self.observed_instance(&project, &name).await?; - Ok(target_after_accepted_freeze(observed)) + (InstanceStatus::Running, PowerState::Paused) => { + self.converge_state(&project, name, "freeze", false, &[InstanceStatus::Frozen]) + .await } - (ProviderTargetStatus::Ready | ProviderTargetStatus::Paused, PowerState::Stopped) => { - if target.status == ProviderTargetStatus::Paused { - self.instance_state_action(&project, &name, "unfreeze", 30, false) - .await?; + (InstanceStatus::Running | InstanceStatus::Frozen, PowerState::Stopped) => { + if target.status == InstanceStatus::Frozen { + self.converge_state( + &project, + name, + "unfreeze", + false, + &[InstanceStatus::Running], + ) + .await?; } - if let Err(error) = self - .instance_state_action(&project, &name, "stop", 30, false) + self.converge_state(&project, name, "stop", false, &[InstanceStatus::Stopped]) .await - { - let observed = self.instance(&project, &name).await?; - if !observed - .as_ref() - .is_some_and(|target| target.status == ProviderTargetStatus::Stopped) - { - return Err(error); - } - } - let observed = self.observed_instance(&project, &name).await?; - Ok(target_after_accepted_stop(observed)) } - // Starting/creating/closing/failed/unknown: nothing safe to do - // until the instance settles; the caller re-observes later. - (_, _) => Ok(target.clone()), + // Transitional, failed or unknown: nothing safe to do until the + // instance settles; the caller re-observes later. + _ => Ok(()), } } async fn set_ingress( &self, - binding: &ProviderBindingContext, target: &OwnedTarget, - hostname: Option<&str>, - port: Option, - ) -> anyhow::Result { - if hostname.is_some() != port.is_some() { - anyhow::bail!("ingress hostname and port must be set together") - } - let project = policy::project_name(binding); + route: Option<(&str, u16)>, + ) -> anyhow::Result<()> { + let project = policy::project_name(&target.binding()); + let name = target.name(); let instance: Instance = self .request( Method::GET, - &format!("/instances/{}", target.name), + &format!("/instances/{name}"), Some(&project), None, ) .await?; let mut config = instance.config; - match (hostname, port) { - (Some(hostname), Some(port)) => { - config.insert( - "user.lightspeed.ingress_hostname".to_owned(), - hostname.to_owned(), - ); - config.insert("user.lightspeed.ingress_port".to_owned(), port.to_string()); + let hostname_key = format!("{}ingress_hostname", policy::META_PREFIX); + let port_key = format!("{}ingress_port", policy::META_PREFIX); + match route { + Some((hostname, port)) => { + config.insert(hostname_key, hostname.to_owned()); + config.insert(port_key, port.to_string()); } - _ => { - config.remove("user.lightspeed.ingress_hostname"); - config.remove("user.lightspeed.ingress_port"); + None => { + config.remove(&hostname_key); + config.remove(&port_key); } } - let update = self.request_unit(Method::PUT, &format!("/instances/{}", target.name), Some(&project), Some(json!({"config":config,"description":instance.description,"profiles":instance.profiles,"devices":instance.devices,"ephemeral":instance.ephemeral}))).await; + let update = self.request_unit(Method::PUT, &format!("/instances/{name}"), Some(&project), Some(json!({"config":config,"description":instance.description,"profiles":instance.profiles,"devices":instance.devices,"ephemeral":instance.ephemeral}))).await; let observed = self - .instance(&project, &target.name) + .instance(&project, name) .await? .ok_or_else(|| anyhow::anyhow!("target disappeared while updating ingress"))?; - if observed.ingress_hostname.as_deref() != hostname || observed.ingress_port != port { + let realized = observed + .ingress_hostname + .as_deref() + .zip(observed.ingress_port); + if realized != route { return Err(update.err().unwrap_or_else(|| { anyhow::anyhow!("Incus did not realize the requested ingress metadata") })); } - Ok(observed) + Ok(()) } } @@ -1212,7 +1177,7 @@ struct Project { } impl Project { - fn binding_context(&self) -> Option { + fn binding(&self) -> Option { if self .config .get("user.lightspeed.managed") @@ -1220,15 +1185,17 @@ impl Project { { return None; } - let binding = ProviderBindingContext { - universe_id: self.config.get("user.lightspeed.universe")?.clone(), - binding_id: self.config.get("user.lightspeed.binding")?.clone(), + let binding = BindingRef { + universe_id: UniverseId::new(self.config.get("user.lightspeed.universe")?.as_str()) + .ok()?, + binding_id: BindingId::new(self.config.get("user.lightspeed.binding")?.as_str()) + .ok()?, }; (self.name == policy::project_name(&binding)).then_some(binding) } fn is_lightspeed_managed(&self) -> bool { - self.binding_context().is_some() + self.binding().is_some() } } @@ -1338,7 +1305,7 @@ fn network_resource_project(mode: IncusMode, project: &str) -> Option<&str> { } } -fn parse_source_target(value: &str) -> anyhow::Result<(&str, &str)> { +pub(crate) fn parse_source_target(value: &str) -> anyhow::Result<(&str, &str)> { let (project, name) = value.split_once('/').unwrap_or(("default", value)); if project.is_empty() || name.is_empty() @@ -1357,165 +1324,106 @@ fn parse_source_target(value: &str) -> anyhow::Result<(&str, &str)> { Ok((project, name)) } -async fn ensure_instance_started( - client: &IncusClient, - project: &str, - name: &str, - target: OwnedTarget, -) -> anyhow::Result { - if matches!( - target.status, - ProviderTargetStatus::Ready | ProviderTargetStatus::Starting - ) { - return Ok(target); - } - if target.status == ProviderTargetStatus::Paused { - client - .instance_state_action(project, name, "unfreeze", 30, false) - .await?; - let observed = client.observed_instance(project, name).await?; - return Ok(target_after_accepted_resume(observed)); - } - if let Err(error) = client - .request_unit( - Method::PUT, - &format!("/instances/{name}/state"), - Some(project), - Some(json!({"action":"start","timeout":30,"force":false})), - ) - .await - { - let observed = client.instance(project, name).await?; - if !observed.as_ref().is_some_and(|target| { - matches!( - target.status, - ProviderTargetStatus::Starting | ProviderTargetStatus::Ready - ) - }) { - return Err(error); - } - } +/// Start a stopped instance or unfreeze a frozen one; a running or starting +/// one needs nothing. +async fn start_instance(client: &IncusClient, project: &str, name: &str) -> anyhow::Result<()> { let target = client .instance(project, name) .await? - .ok_or_else(|| anyhow::anyhow!("Incus target disappeared while starting"))?; - Ok(target_after_accepted_start(target)) -} - -/// Incus can briefly return the pre-operation `Stopped` state immediately -/// after its asynchronous start operation succeeds. Keep the provisioning -/// lifecycle pending until a later observation sees the actual running state. -fn target_after_accepted_start(mut target: OwnedTarget) -> OwnedTarget { - target.status = status_after_accepted_start(target.status); - target -} - -fn status_after_accepted_start(status: ProviderTargetStatus) -> ProviderTargetStatus { - match status { - ProviderTargetStatus::Stopped => ProviderTargetStatus::Starting, - status => status, + .ok_or_else(|| anyhow::anyhow!("the Incus instance disappeared before it started"))?; + match target.status { + InstanceStatus::Running | InstanceStatus::Starting => Ok(()), + InstanceStatus::Frozen => { + client + .converge_state(project, name, "unfreeze", false, &[InstanceStatus::Running]) + .await + } + _ => { + client + .converge_state( + project, + name, + "start", + false, + &[InstanceStatus::Running, InstanceStatus::Starting], + ) + .await?; + client.settle_after_start(project, name).await + } } } -/// After an accepted unfreeze, a stale `Frozen` read means "resuming", not -/// "still paused". -fn target_after_accepted_resume(mut target: OwnedTarget) -> OwnedTarget { - if target.status == ProviderTargetStatus::Paused { - target.status = ProviderTargetStatus::Starting; - } - target +/// The `user.lightspeed.*` keys that make an instance the owner's. +fn ownership_config(owner: &TargetOwner) -> BTreeMap { + BTreeMap::from([ + (format!("{}managed", policy::META_PREFIX), "true".to_owned()), + ( + format!("{}universe", policy::META_PREFIX), + owner.universe_id.to_string(), + ), + ( + format!("{}binding", policy::META_PREFIX), + owner.binding_id.to_string(), + ), + ( + format!("{}environment", policy::META_PREFIX), + owner.environment_id.to_string(), + ), + ( + format!("{}incarnation", policy::META_PREFIX), + owner.incarnation_id.to_string(), + ), + ]) } -/// After an accepted freeze, a stale `Running` read must not make the caller -/// freeze again; report the requested steady state. -fn target_after_accepted_freeze(mut target: OwnedTarget) -> OwnedTarget { - if target.status == ProviderTargetStatus::Ready { - target.status = ProviderTargetStatus::Paused; +fn classify_adoption_source(source: &Instance) -> AdoptionSource { + if source + .config + .get(&format!("{}managed", policy::META_PREFIX)) + .is_some_and(|value| value == "true") + { + return AdoptionSource::Managed; } - target -} - -/// After an accepted stop, a stale `Running`/`Frozen` read is reported as -/// stopped; a later observation corrects it if the stop did not stick. -fn target_after_accepted_stop(mut target: OwnedTarget) -> OwnedTarget { - if matches!( - target.status, - ProviderTargetStatus::Ready | ProviderTargetStatus::Paused - ) { - target.status = ProviderTargetStatus::Stopped; + if source.kind != "virtual-machine" { + return AdoptionSource::NotAdoptable("only Incus virtual machines can be adopted"); } - target -} - -fn verify_adopted_target(target: &OwnedTarget, params: &AdoptTargetParams) -> anyhow::Result<()> { - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - || target.request_id != params.request_id - || target.template_id != "adopted" - || target.image_fingerprint != "adopted" - || target.adoption_source.as_deref() != Some(params.source_target.as_str()) - { - anyhow::bail!("existing target metadata conflicts with adoption request") + if source.ephemeral { + return AdoptionSource::NotAdoptable("ephemeral Incus instances cannot be adopted"); } - Ok(()) + AdoptionSource::Adoptable } -fn owned_from_instance(instance: Instance, state: InstanceState) -> anyhow::Result { - let name = instance.name.clone(); +/// The owned target an instance describes, or `None` when its ownership +/// metadata is missing, invalid or does not match its deterministic name. +fn owned_from_instance(instance: Instance, state: InstanceState) -> Option { let get = |key: &str| { instance .config .get(&format!("{}{}", policy::META_PREFIX, key)) .cloned() - .ok_or_else(|| anyhow::anyhow!("instance {name} is not owned: missing {key}")) }; if get("managed")? != "true" { - anyhow::bail!("instance is not Lightspeed-managed") + return None; } - let universe_id = get("universe")?; - let binding_id = get("binding")?; - let environment_id = get("environment")?; - let incarnation_id = get("incarnation")?; - if instance.name - != policy::instance_name(&universe_id, &binding_id, &environment_id, &incarnation_id) - { - anyhow::bail!("managed instance name does not match immutable ownership metadata") + let owner = TargetOwner { + universe_id: UniverseId::new(get("universe")?).ok()?, + binding_id: BindingId::new(get("binding")?).ok()?, + environment_id: EnvironmentId::new(get("environment")?).ok()?, + incarnation_id: IncarnationId::new(get("incarnation")?).ok()?, + }; + if instance.name != policy::instance_name(&owner) { + return None; } - let ipv4_address = managed_ipv4_address(&state); - Ok(OwnedTarget { - target_id: ProviderTargetId::new(instance.name.clone()), - name: instance.name, - universe_id, - binding_id, - environment_id, - incarnation_id, - request_id: get("request")?, + Some(OwnedTarget { + target_id: ProviderTargetId::new(instance.name.clone()).ok()?, + owner, template_id: get("template")?, image_fingerprint: get("image")?, - adoption_source: instance - .config - .get(&format!("{}adoption_source", policy::META_PREFIX)) - .cloned(), - status: match instance.status.as_str() { - "Running" => ProviderTargetStatus::Ready, - "Starting" => ProviderTargetStatus::Starting, - "Stopped" => ProviderTargetStatus::Stopped, - // Incus reports a frozen container or paused VM as `Frozen`. - "Frozen" | "Freezing" => ProviderTargetStatus::Paused, - "Error" => ProviderTargetStatus::Failed, - _ => ProviderTargetStatus::Unknown, - }, - ipv4_address, - ingress_hostname: instance - .config - .get(&format!("{}ingress_hostname", policy::META_PREFIX)) - .cloned(), - ingress_port: instance - .config - .get(&format!("{}ingress_port", policy::META_PREFIX)) - .and_then(|value| value.parse().ok()), + adoption_source: get("adoption_source"), + status: InstanceStatus::from_incus(&instance.status), + ipv4_address: managed_ipv4_address(&state), + ingress_hostname: get("ingress_hostname"), + ingress_port: get("ingress_port").and_then(|value| value.parse().ok()), location: (!instance.location.is_empty()).then_some(instance.location), }) } @@ -1543,10 +1451,7 @@ mod tests { #[test] fn managed_projects_require_matching_binding_metadata() { - let binding = ProviderBindingContext { - universe_id: "u".to_owned(), - binding_id: "b".to_owned(), - }; + let binding = policy::binding_of(&policy::tests::owner("u", "b", "e")); let project = Project { name: policy::project_name(&binding), config: BTreeMap::from([ @@ -1563,6 +1468,63 @@ mod tests { assert!(!spoofed.is_lightspeed_managed()); } + fn instance(name: &str, config: BTreeMap) -> Instance { + Instance { + name: name.to_owned(), + kind: "virtual-machine".to_owned(), + description: String::new(), + profiles: Vec::new(), + devices: BTreeMap::new(), + ephemeral: false, + status: "Running".to_owned(), + config, + location: String::new(), + } + } + + #[test] + fn ownership_metadata_must_match_the_deterministic_name() { + let owner = policy::tests::owner("u", "b", "e"); + let mut config = ownership_config(&owner); + config.insert("user.lightspeed.template".to_owned(), "t".to_owned()); + config.insert("user.lightspeed.image".to_owned(), "f".to_owned()); + let name = policy::instance_name(&owner); + let owned = owned_from_instance(instance(&name, config.clone()), InstanceState::default()) + .expect("owned"); + assert_eq!(owned.owner, owner); + assert_eq!(owned.status, InstanceStatus::Running); + assert!( + owned_from_instance( + instance("ls-other", config.clone()), + InstanceState::default() + ) + .is_none() + ); + config.insert("user.lightspeed.universe".to_owned(), "bad id!".to_owned()); + assert!(owned_from_instance(instance(&name, config), InstanceState::default()).is_none()); + } + + #[test] + fn adoption_sources_are_classified_before_any_change() { + let mut source = instance("manual-vm", BTreeMap::new()); + assert_eq!(classify_adoption_source(&source), AdoptionSource::Adoptable); + source.ephemeral = true; + assert!(matches!( + classify_adoption_source(&source), + AdoptionSource::NotAdoptable(_) + )); + source.ephemeral = false; + source.kind = "container".to_owned(); + assert!(matches!( + classify_adoption_source(&source), + AdoptionSource::NotAdoptable(_) + )); + source + .config + .insert("user.lightspeed.managed".to_owned(), "true".to_owned()); + assert_eq!(classify_adoption_source(&source), AdoptionSource::Managed); + } + #[test] fn adoption_source_defaults_to_the_default_project() { assert_eq!( @@ -1620,53 +1582,18 @@ mod tests { } #[test] - fn accepted_power_changes_do_not_report_stale_pre_operation_states() { - let target = |status| OwnedTarget { - target_id: "t".into(), - name: "t".to_owned(), - universe_id: "u".to_owned(), - binding_id: "b".to_owned(), - environment_id: "e".to_owned(), - incarnation_id: "i".to_owned(), - request_id: "r".to_owned(), - template_id: "tpl".to_owned(), - image_fingerprint: "img".to_owned(), - adoption_source: None, - status, - ipv4_address: None, - ingress_hostname: None, - ingress_port: None, - location: None, - }; - assert_eq!( - target_after_accepted_freeze(target(ProviderTargetStatus::Ready)).status, - ProviderTargetStatus::Paused - ); - assert_eq!( - target_after_accepted_resume(target(ProviderTargetStatus::Paused)).status, - ProviderTargetStatus::Starting - ); + fn incus_statuses_reduce_to_provider_statuses() { assert_eq!( - target_after_accepted_resume(target(ProviderTargetStatus::Ready)).status, - ProviderTargetStatus::Ready + InstanceStatus::from_incus("Running"), + InstanceStatus::Running ); assert_eq!( - target_after_accepted_stop(target(ProviderTargetStatus::Paused)).status, - ProviderTargetStatus::Stopped + InstanceStatus::from_incus("Freezing"), + InstanceStatus::Frozen ); + assert_eq!(InstanceStatus::from_incus("Error"), InstanceStatus::Error); + assert_eq!(InstanceStatus::from_incus("Pending"), InstanceStatus::Other); assert!(SUPPORTED_POWER_STATES.contains(&PowerState::Paused)); assert!(!SUPPORTED_POWER_STATES.contains(&PowerState::Suspended)); } - - #[test] - fn accepted_start_does_not_report_a_stale_stopped_state() { - assert_eq!( - status_after_accepted_start(ProviderTargetStatus::Stopped), - ProviderTargetStatus::Starting - ); - assert_eq!( - status_after_accepted_start(ProviderTargetStatus::Ready), - ProviderTargetStatus::Ready - ); - } } diff --git a/crates/environment-provider-incus/src/ingress.rs b/crates/environment-provider-incus/src/ingress.rs index 5373c1328..541e1e3d8 100644 --- a/crates/environment-provider-incus/src/ingress.rs +++ b/crates/environment-provider-incus/src/ingress.rs @@ -1,5 +1,7 @@ //! Provider-owned public hostname allocation. +use environment_protocol::routes::PublicUrl; + use crate::{Config, incus::OwnedTarget, policy}; pub fn hostname(target: &OwnedTarget, public_base_url: &str) -> anyhow::Result { @@ -10,15 +12,15 @@ pub fn hostname(target: &OwnedTarget, public_base_url: &str) -> anyhow::Result anyhow::Result { +pub fn public_endpoint(config: &Config, hostname: &str) -> anyhow::Result { let ingress = config .ingress .as_ref() @@ -26,36 +28,32 @@ pub fn public_endpoint(config: &Config, hostname: &str) -> anyhow::Result anyhow::Result<(String, String)> { +/// The hostname the provider allocates for the target. +pub fn allocated_hostname(config: &Config, target: &OwnedTarget) -> anyhow::Result { let ingress = config .ingress .as_ref() .ok_or_else(|| anyhow::anyhow!("provider ingress is not configured"))?; - let hostname = hostname(target, &ingress.public_base_url)?; - Ok((hostname.clone(), public_endpoint(config, &hostname)?)) + hostname(target, &ingress.public_base_url) } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - use environment_protocol::{control::targets::ProviderTargetStatus, shared::ProviderTargetId}; + use crate::incus::InstanceStatus; - fn target() -> OwnedTarget { + pub(crate) fn target() -> OwnedTarget { + let owner = policy::tests::owner("universe-a", "binding-a", "environment-a"); OwnedTarget { - target_id: ProviderTargetId::new("target-a"), - name: "target-a".to_owned(), - universe_id: "universe-a".to_owned(), - binding_id: "binding-a".to_owned(), - environment_id: "environment-a".to_owned(), - incarnation_id: "incarnation-a".to_owned(), - request_id: "request-a".to_owned(), + target_id: policy::instance_name(&owner).try_into().unwrap(), + owner, template_id: "template-a".to_owned(), image_fingerprint: "fingerprint-a".to_owned(), adoption_source: None, - status: ProviderTargetStatus::Ready, + status: InstanceStatus::Running, ipv4_address: Some("10.0.0.2".to_owned()), ingress_hostname: None, ingress_port: None, diff --git a/crates/environment-provider-incus/src/policy.rs b/crates/environment-provider-incus/src/policy.rs index e0bdfa7c8..6a7a8973a 100644 --- a/crates/environment-provider-incus/src/policy.rs +++ b/crates/environment-provider-incus/src/policy.rs @@ -1,9 +1,13 @@ use sha2::{Digest as _, Sha256}; -use environment_protocol::control::targets::ProviderBindingContext; +use environment_protocol::control::target::{BindingRef, TargetOwner}; pub const META_PREFIX: &str = "user.lightspeed."; +/// Longest Incus instance, project or network name component the provider +/// addresses in an Incus API path. +const MAX_INCUS_NAME_BYTES: usize = 63; + pub fn stable_component(kind: &str, parts: &[&str]) -> String { let mut hash = Sha256::new(); hash.update(kind.as_bytes()); @@ -14,42 +18,72 @@ pub fn stable_component(kind: &str, parts: &[&str]) -> String { hex(&hash.finalize()[..10]) } -pub fn project_name(binding: &ProviderBindingContext) -> String { - format!( - "ls-{}", - stable_component("binding", &[&binding.universe_id, &binding.binding_id]) +pub fn binding_of(owner: &TargetOwner) -> BindingRef { + BindingRef { + universe_id: owner.universe_id.clone(), + binding_id: owner.binding_id.clone(), + } +} + +fn binding_component(binding: &BindingRef) -> String { + stable_component( + "binding", + &[binding.universe_id.as_str(), binding.binding_id.as_str()], ) } -pub fn network_name(binding: &ProviderBindingContext) -> String { - let component = stable_component("binding", &[&binding.universe_id, &binding.binding_id]); +pub fn project_name(binding: &BindingRef) -> String { + format!("ls-{}", binding_component(binding)) +} + +pub fn network_name(binding: &BindingRef) -> String { // A managed bridge's Incus network name also becomes its Linux interface // name, whose hard limit is 15 bytes. Keep 48 bits of the scoped digest // while making the resource kind visible within that limit. - format!("ls{}n", &component[..12]) + format!("ls{}n", &binding_component(binding)[..12]) } -pub fn profile_name(binding: &ProviderBindingContext) -> String { + +pub fn profile_name(binding: &BindingRef) -> String { format!("{}-vm", project_name(binding)) } -pub fn acl_name(binding: &ProviderBindingContext) -> String { + +pub fn acl_name(binding: &BindingRef) -> String { format!("{}-acl", project_name(binding)) } -pub fn instance_name( - universe_id: &str, - binding_id: &str, - environment_id: &str, - incarnation_id: &str, -) -> String { +/// The one instance name, and therefore `targetId`, an owner can hold. The +/// provider's owner index is this deterministic name inside the binding's +/// project, so create and adopt are idempotent per owner without any +/// provider-side state. +pub fn instance_name(owner: &TargetOwner) -> String { format!( "ls-{}", stable_component( "instance", - &[universe_id, binding_id, environment_id, incarnation_id] + &[ + owner.universe_id.as_str(), + owner.binding_id.as_str(), + owner.environment_id.as_str(), + owner.incarnation_id.as_str(), + ] ) ) } +/// True for a string the provider may place in an Incus API path as an +/// instance name: 1 to 63 ASCII letters, digits or hyphens, not starting or +/// ending with a hyphen. Every `targetId` this provider assigns qualifies; +/// any other opaque id names nothing here. +pub fn is_instance_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_INCUS_NAME_BYTES + && !value.starts_with('-') + && !value.ends_with('-') + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') +} + fn hex(bytes: &[u8]) -> String { const TABLE: &[u8; 16] = b"0123456789abcdef"; let mut output = String::with_capacity(bytes.len() * 2); @@ -61,27 +95,50 @@ fn hex(bytes: &[u8]) -> String { } #[cfg(test)] -mod tests { +pub(crate) mod tests { + use environment_protocol::ids::{BindingId, EnvironmentId, IncarnationId, UniverseId}; + use super::*; + + pub(crate) fn owner(universe: &str, binding: &str, environment: &str) -> TargetOwner { + TargetOwner { + universe_id: UniverseId::new(universe).unwrap(), + binding_id: BindingId::new(binding).unwrap(), + environment_id: EnvironmentId::new(environment).unwrap(), + incarnation_id: IncarnationId::new("i").unwrap(), + } + } + #[test] fn names_are_stable_and_scoped() { - let a = instance_name("u", "b", "e", "i"); - assert_eq!(a, instance_name("u", "b", "e", "i")); - assert_ne!(a, instance_name("u2", "b", "e", "i")); + let a = instance_name(&owner("u", "b", "e")); + assert_eq!(a, instance_name(&owner("u", "b", "e"))); + assert_ne!(a, instance_name(&owner("u2", "b", "e"))); + assert!(is_instance_name(&a)); } #[test] fn binding_network_names_fit_the_linux_interface_limit() { - let binding = ProviderBindingContext { - universe_id: "0d3d9e5e-2428-4e60-b66e-8d4520f64e5d".to_owned(), - binding_id: "hz02-incus".to_owned(), - }; - let other = ProviderBindingContext { - universe_id: "61092591-cd39-4504-b845-a817e3d9cb71".to_owned(), - binding_id: "hz02-incus".to_owned(), - }; + let binding = binding_of(&owner( + "0d3d9e5e-2428-4e60-b66e-8d4520f64e5d", + "hz02-incus", + "e", + )); + let other = binding_of(&owner( + "61092591-cd39-4504-b845-a817e3d9cb71", + "hz02-incus", + "e", + )); assert_eq!(network_name(&binding).len(), 15); assert_ne!(network_name(&binding), network_name(&other)); } + + #[test] + fn only_incus_names_reach_an_api_path() { + assert!(is_instance_name("ls-0123456789abcdef0123")); + for value in ["", "-a", "a-", "a/b", "a?b", "default/pod", &"a".repeat(64)] { + assert!(!is_instance_name(value), "{value}"); + } + } } diff --git a/crates/environment-provider-incus/src/relay.rs b/crates/environment-provider-incus/src/relay.rs index 6ac992bd1..ff177fb54 100644 --- a/crates/environment-provider-incus/src/relay.rs +++ b/crates/environment-provider-incus/src/relay.rs @@ -1,41 +1,83 @@ //! On-demand provider data connections. //! //! Lightspeed opens one provider WebSocket for a specific current target. The -//! provider validates ownership, dials the target's private envd, and proxies -//! the connection until it closes or becomes idle. There is no provider-initiated -//! connection and no provider route registry in Lightspeed. +//! provider validates the credential and ownership, dials the target's private +//! envd, and proxies the connection until it closes or becomes idle. There is +//! no provider-initiated connection and no provider route registry in +//! Lightspeed. +//! +//! The relay is an intermediary under the protocol's liveness rule: it +//! forwards pings and pongs, counts every frame in either direction as +//! activity, and never idle-closes sooner than `relayIdleSeconds`, which +//! config loading keeps at or above the protocol minimum. Both legs carry the +//! protocol's message and frame limit. use std::time::Duration; use anyhow::Context as _; use axum::extract::ws::{Message as AxumMessage, WebSocket}; +use environment_protocol::limits::MAX_MESSAGE_BYTES; use futures_util::{SinkExt as _, StreamExt as _}; use tokio::net::TcpStream; use tokio_tungstenite::{ - MaybeTlsStream, WebSocketStream, connect_async, tungstenite::Message as TungsteniteMessage, + MaybeTlsStream, WebSocketStream, connect_async_with_config, + tungstenite::{Message as TungsteniteMessage, protocol::WebSocketConfig}, }; use crate::{Config, incus::OwnedTarget}; pub type GuestSocket = WebSocketStream>; +/// Bound on the readiness probe's dial, so an observation stays prompt while +/// a guest boots. +const PROBE_TIMEOUT: Duration = Duration::from_secs(3); + +/// The protocol's message limit, applied to both relay legs. +pub fn websocket_config() -> WebSocketConfig { + WebSocketConfig::default() + .max_message_size(Some(MAX_MESSAGE_BYTES as usize)) + .max_frame_size(Some(MAX_MESSAGE_BYTES as usize)) +} + pub async fn dial_guest(config: &Config, target: &OwnedTarget) -> anyhow::Result { + dial_guest_within( + config, + target, + Duration::from_secs(config.dial_timeout_seconds), + ) + .await +} + +async fn dial_guest_within( + config: &Config, + target: &OwnedTarget, + timeout: Duration, +) -> anyhow::Result { let address = target .ipv4_address .as_deref() .ok_or_else(|| anyhow::anyhow!("target has no private IPv4 address"))?; let endpoint = format!("ws://{address}:{}/", config.envd_port); let (socket, _) = tokio::time::timeout( - Duration::from_secs(config.dial_timeout_seconds), - connect_async(endpoint), + timeout, + connect_async_with_config(endpoint, Some(websocket_config()), true), ) .await .context("envd dial timeout")??; Ok(socket) } +/// The provider's own liveness probe of envd: a WebSocket upgrade on the +/// guest's data listener, without `initialize`, so the provider never +/// depends on envd's protocol version. pub async fn probe_guest(config: &Config, target: &OwnedTarget) -> bool { - match dial_guest(config, target).await { + match dial_guest_within( + config, + target, + PROBE_TIMEOUT.min(Duration::from_secs(config.dial_timeout_seconds)), + ) + .await + { Ok(mut socket) => { let _ = socket.close(None).await; true @@ -124,7 +166,7 @@ mod tests { use super::*; - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn relay_closes_guest_when_lightspeed_disappears_without_a_close_frame() { let guest_listener = TcpListener::bind("127.0.0.1:0") .await @@ -144,7 +186,7 @@ mod tests { } false }); - let (guest, _) = connect_async(format!("ws://{guest_address}")) + let (guest, _) = tokio_tungstenite::connect_async(format!("ws://{guest_address}")) .await .expect("connect guest"); let guest = Arc::new(Mutex::new(Some(guest))); @@ -178,7 +220,7 @@ mod tests { .expect("relay server") }); - let (lightspeed, _) = connect_async(format!("ws://{relay_address}")) + let (lightspeed, _) = tokio_tungstenite::connect_async(format!("ws://{relay_address}")) .await .expect("connect lightspeed"); drop(lightspeed); @@ -190,4 +232,82 @@ mod tests { relay_server.abort(); assert!(saw_close, "relay dropped guest without a close frame"); } + + #[tokio::test(flavor = "current_thread")] + async fn relay_forwards_pings_and_counts_them_as_activity() { + let guest_listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("guest listener"); + let guest_address = guest_listener.local_addr().expect("guest address"); + let guest_observer = tokio::spawn(async move { + let (stream, _) = guest_listener.accept().await.expect("guest accept"); + let mut socket = tokio_tungstenite::accept_async(stream) + .await + .expect("guest websocket"); + let mut pings = 0; + while let Some(message) = socket.next().await { + match message { + Ok(TungsteniteMessage::Ping(_)) => pings += 1, + Ok(TungsteniteMessage::Text(text)) => return Some((pings, text.to_string())), + Ok(_) => {} + Err(_) => return None, + } + } + None + }); + let (guest, _) = tokio_tungstenite::connect_async(format!("ws://{guest_address}")) + .await + .expect("connect guest"); + let guest = Arc::new(Mutex::new(Some(guest))); + let relay_listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("relay listener"); + let relay_address = relay_listener.local_addr().expect("relay address"); + let app = Router::new().route( + "/", + get(move |upgrade: WebSocketUpgrade| { + let guest = guest.clone(); + async move { + let mut guest = guest.lock().await.take().expect("one relay connection"); + upgrade + .on_upgrade(move |mut socket| async move { + proxy_with_idle(Duration::from_millis(300), &mut socket, &mut guest) + .await; + }) + .into_response() + } + }), + ); + let relay_server = tokio::spawn(async move { + axum::serve(relay_listener, app) + .await + .expect("relay server") + }); + + let (mut lightspeed, _) = tokio_tungstenite::connect_async(format!("ws://{relay_address}")) + .await + .expect("connect lightspeed"); + // Eight pings 100 ms apart keep the 300 ms idle timer from firing + // across 800 ms with no data frame. + for _ in 0..8 { + lightspeed + .send(TungsteniteMessage::Ping(vec![1].into())) + .await + .expect("ping"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + lightspeed + .send(TungsteniteMessage::Text("after pings".into())) + .await + .expect("text"); + + let observed = tokio::time::timeout(Duration::from_secs(2), guest_observer) + .await + .expect("guest timeout") + .expect("guest observer"); + relay_server.abort(); + let (pings, text) = observed.expect("the relay closed the guest leg while pings flowed"); + assert_eq!(pings, 8); + assert_eq!(text, "after pings"); + } } diff --git a/crates/environment-provider-incus/src/server.rs b/crates/environment-provider-incus/src/server.rs index f6090c2e3..7877d2e39 100644 --- a/crates/environment-provider-incus/src/server.rs +++ b/crates/environment-provider-incus/src/server.rs @@ -1,64 +1,153 @@ -use std::{collections::BTreeMap, sync::Arc}; +//! The provider's listener: the controller plane on `/control` and the data +//! routes under `/routes`, both behind the provider credential. +//! +//! Every target method answers promptly with the current observation. Slow +//! Incus work (an image pull, a boot, a graceful shutdown) runs as one +//! background job per target, serialized per target, and later calls observe +//! its progress: a running create reports `provisioning`, a running close +//! `closing`, and a failed create `failed` with its error as the `message`. +//! Job state is in memory; Incus itself, read through the deterministic owner +//! name, is the record of every target. + +use std::{ + collections::{BTreeMap, HashMap}, + panic::AssertUnwindSafe, + sync::{ + Arc, Mutex as StdMutex, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; use axum::{ Json, Router, extract::{ Path, State, WebSocketUpgrade, - ws::{Message, WebSocket}, + ws::{Message, WebSocket, rejection::WebSocketUpgradeRejection}, }, - http::StatusCode, + http::{HeaderMap, StatusCode, header}, response::{IntoResponse, Response}, routing::get, }; use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, control::{ handshake::{ ControllerCapabilities, ControllerInitializeParams, ControllerInitializeResponse, + ControllerRoutes, }, - ingress::{ - EnsureIngressParams, IngressResponse, ProviderIngressStatus, RemoveIngressParams, + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + AdoptTargetParams, CloseTargetParams, CreateTargetParams, GetTargetParams, + IngressObservation, PowerState, SetTargetPowerParams, TargetObservation, TargetOwner, + TargetPhase, TargetRef, TargetResponse, TargetState, }, - methods::*, - targets::*, + template::{EnvironmentTemplate, ListTemplatesParams, ListTemplatesResponse}, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::{ - CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentScope, ImplementationInfo, + error::{BusyReason, Capability, PreconditionReason, ProtocolError}, + handshake::{ImplementationInfo, Plane}, + ids::{BindingId, EnvironmentId, IncarnationId, ProviderTargetId, TemplateId, UniverseId}, + jsonrpc::{self, Request, Response as RpcResponse}, + limits::MAX_MESSAGE_BYTES, + manifest::{ + self, ControllerInitialize, IngressEnsure, IngressRemove, MessageKind, TargetAdopt, + TargetClose, TargetCreate, TargetGet, TargetSetPower, TemplateList, }, + metadata::{DescriptiveText, Metadata, validate_metadata_entry}, }; -use futures_util::{StreamExt as _, stream}; -use serde::de::DeserializeOwned; -use serde_json::{Value, json}; +use futures_util::{FutureExt as _, SinkExt as _, StreamExt as _, future::BoxFuture}; +use serde::Serialize; +use serde_json::Value; +use tokio::sync::{Mutex as AsyncMutex, Semaphore, mpsc, watch}; + +use crate::{ + Config, IncusBackend, + incus::{AdoptionSource, InstanceStatus, OwnedTarget, SUPPORTED_POWER_STATES}, + ingress, policy, relay, +}; + +/// How long a mutation waits for its background job before it answers with +/// the current observation. +const PROMPT_WAIT: Duration = Duration::from_secs(5); -use crate::{Config, IncusBackend, incus::OwnedTarget, ingress, policy, relay}; +/// Requests one controller connection may have in flight before `busy`. +const MAX_CONCURRENT_REQUESTS: usize = 32; -type BindingKey = (String, String); -type BindingLock = Arc>; -type BindingLocks = Arc>>; +const _: () = assert!( + MAX_CONCURRENT_REQUESTS >= environment_protocol::limits::MIN_CONCURRENT_REQUESTS as usize +); #[derive(Clone)] struct App { config: Config, backend: B, - binding_locks: BindingLocks, + work: Arc, + initialize: Arc, +} + +impl App { + fn new(config: Config, backend: B) -> Self { + let initialize = ControllerInitializeResponse { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + plane: Plane::Controller, + server_info: server_info(), + capabilities: ControllerCapabilities { + adopt_target: true, + ingress: config.ingress.is_some(), + }, + routes: ControllerRoutes { + data: config.data_route_base.clone(), + }, + }; + Self { + config, + backend, + work: Arc::default(), + initialize: Arc::new(initialize), + } + } +} + +fn server_info() -> ImplementationInfo { + let info = ImplementationInfo::new("lightspeed-provider-incus") + .expect("the provider name is a valid implementation name"); + let info = info + .clone() + .with_version(release_info::VERSION) + .unwrap_or(info); + let info = info + .clone() + .with_git_sha(release_info::GIT_SHA) + .unwrap_or(info); + info.clone() + .with_target(release_info::TARGET) + .unwrap_or(info) } pub async fn serve(config: Config, backend: B) -> anyhow::Result<()> { let listener = tokio::net::TcpListener::bind(config.controller_listen).await?; - let app = Router::new() + serve_on(listener, config, backend).await +} + +/// Serve on an already bound listener. +pub async fn serve_on( + listener: tokio::net::TcpListener, + config: Config, + backend: B, +) -> anyhow::Result<()> { + axum::serve(listener, router(config, backend)).await?; + Ok(()) +} + +fn router(config: Config, backend: B) -> Router { + Router::new() .route("/health", get(health::)) - .route("/control", get(upgrade::)) + .route("/control", get(control_upgrade::)) .route( "/routes/{universe}/{binding}/{environment}/{incarnation}/{target}", get(data_upgrade::), ) - .with_state(App { - config, - backend, - binding_locks: Arc::default(), - }); - axum::serve(listener, app).await?; - Ok(()) + .with_state(App::new(config, backend)) } async fn health(State(app): State>) -> Response { @@ -68,151 +157,268 @@ async fn health(State(app): State>) -> Response { } } -async fn upgrade( +/// True when the request carries the provider credential as a bearer token. +fn authorized(config: &Config, headers: &HeaderMap) -> bool { + headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .is_some_and(|token| config.credential.matches(token)) +} + +fn unauthorized() -> Response { + ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, "Bearer")], + ) + .into_response() +} + +fn with_limits(upgrade: WebSocketUpgrade) -> WebSocketUpgrade { + upgrade + .max_message_size(MAX_MESSAGE_BYTES as usize) + .max_frame_size(MAX_MESSAGE_BYTES as usize) +} + +async fn control_upgrade( State(app): State>, - upgrade: WebSocketUpgrade, + headers: HeaderMap, + upgrade: Result, ) -> Response { - upgrade - .on_upgrade(move |socket| connection(app, socket)) - .into_response() + if !authorized(&app.config, &headers) { + return unauthorized(); + } + match upgrade { + Ok(upgrade) => with_limits(upgrade) + .on_upgrade(move |socket| connection(app, socket)) + .into_response(), + Err(rejection) => rejection.into_response(), + } } async fn data_upgrade( State(app): State>, - Path((universe, binding_id, environment, incarnation, target_id)): Path<( + Path((universe, binding, environment, incarnation, target_id)): Path<( String, String, String, String, String, )>, - upgrade: WebSocketUpgrade, + headers: HeaderMap, + upgrade: Result, ) -> Response { - let binding = ProviderBindingContext { - universe_id: universe.clone(), - binding_id: binding_id.clone(), + if !authorized(&app.config, &headers) { + return unauthorized(); + } + let owner = match ( + UniverseId::new(universe), + BindingId::new(binding), + EnvironmentId::new(environment), + IncarnationId::new(incarnation), + ) { + (Ok(universe_id), Ok(binding_id), Ok(environment_id), Ok(incarnation_id)) => TargetOwner { + universe_id, + binding_id, + environment_id, + incarnation_id, + }, + _ => return StatusCode::NOT_FOUND.into_response(), }; - let expected = policy::instance_name(&universe, &binding_id, &environment, &incarnation); - if expected != target_id { - return StatusCode::CONFLICT.into_response(); + if target_id != policy::instance_name(&owner) { + return StatusCode::NOT_FOUND.into_response(); } - let target_id = environment_protocol::shared::ProviderTargetId::new(target_id); - let target = match app.backend.get_owned(&binding, &target_id).await { - Ok(Some(target)) => target, + let target = match app + .backend + .get_owned(&policy::binding_of(&owner), &target_id) + .await + { + Ok(Some(target)) if target.owner == owner => target, + Ok(Some(_)) => return StatusCode::CONFLICT.into_response(), Ok(None) => return StatusCode::NOT_FOUND.into_response(), Err(_) => return StatusCode::BAD_GATEWAY.into_response(), }; - if verify_binding(&target, &binding).is_err() - || target.environment_id != environment - || target.incarnation_id != incarnation - { - return StatusCode::CONFLICT.into_response(); - } + let upgrade = match upgrade { + Ok(upgrade) => upgrade, + Err(rejection) => return rejection.into_response(), + }; let guest = match relay::dial_guest(&app.config, &target).await { Ok(guest) => guest, Err(_) => return StatusCode::SERVICE_UNAVAILABLE.into_response(), }; let config = app.config.clone(); - upgrade - .max_message_size(64 * 1024 * 1024) + with_limits(upgrade) .on_upgrade(move |socket| relay::proxy(config, socket, guest)) .into_response() } -async fn connection(app: App, mut socket: WebSocket) { - while let Some(Ok(message)) = socket.recv().await { - match message { - Message::Text(text) => { - let response = dispatch(&app, &text).await; - if socket - .send(Message::Text(response.to_string().into())) - .await - .is_err() - { +/// One controller connection: the JSON-RPC envelope, the `initialize`-first +/// handshake, and pipelined requests answered in any order by one writer. +async fn connection(app: App, socket: WebSocket) { + let (mut sink, mut stream) = socket.split(); + let (outgoing, mut queue) = mpsc::channel::(MAX_CONCURRENT_REQUESTS * 2); + let writer = tokio::spawn(async move { + while let Some(message) = queue.recv().await { + if sink.send(message).await.is_err() { + break; + } + } + let _ = sink.close().await; + }); + let permits = Arc::new(Semaphore::new(MAX_CONCURRENT_REQUESTS)); + let mut initialized = false; + while let Some(Ok(frame)) = stream.next().await { + let reply = match frame { + Message::Text(text) => match jsonrpc::parse(text.as_str()) { + Err(rejection) => rejection.into_response(), + Ok(jsonrpc::Message::Request(request)) => { + if request.method == ControllerInitialize::NAME { + Some(initialize(&app, &mut initialized, &request)) + } else if !initialized { + Some(RpcResponse::failure( + Some(request.id), + ProtocolError::not_initialized(), + )) + } else { + match permits.clone().try_acquire_owned() { + Ok(permit) => { + let app = app.clone(); + let outgoing = outgoing.clone(); + tokio::spawn(async move { + let response = handle(&app, request).await; + let _ = outgoing + .send(Message::Text(response.to_text().into())) + .await; + drop(permit); + }); + None + } + Err(_) => Some(RpcResponse::failure( + Some(request.id), + ProtocolError::busy( + Some(BusyReason::RequestLimit), + None, + format!( + "more than {MAX_CONCURRENT_REQUESTS} requests in flight" + ), + ), + )), + } + } + } + // The controller plane defines no notifications, and this + // server sends no requests, so no response answers anything. + Ok(jsonrpc::Message::Notification(_) | jsonrpc::Message::Response(_)) => None, + }, + Message::Binary(_) => Some(RpcResponse::failure( + None, + ProtocolError::invalid_request("binary frames are not part of any plane"), + )), + Message::Ping(payload) => { + if outgoing.send(Message::Pong(payload)).await.is_err() { break; } + None } - Message::Ping(bytes) => { - let _ = socket.send(Message::Pong(bytes)).await; - } + Message::Pong(_) => None, Message::Close(_) => break, - _ => {} + }; + if let Some(reply) = reply + && outgoing + .send(Message::Text(reply.to_text().into())) + .await + .is_err() + { + break; } } - let _ = socket.send(Message::Close(None)).await; + drop(outgoing); + let _ = writer.await; } -async fn dispatch(app: &App, text: &str) -> Value { - let request: Value = match serde_json::from_str(text) { - Ok(value) => value, - Err(decode_error) => { - return error( - Value::Null, - EnvironmentProtocolErrorCode::InvalidRequest, - decode_error.to_string(), - ); +fn initialize(app: &App, initialized: &mut bool, request: &Request) -> RpcResponse { + let id = Some(request.id.clone()); + if *initialized { + return RpcResponse::failure( + id, + ProtocolError::invalid_request("initialize was already called on this connection"), + ); + } + // The version decides before the rest of the params: a peer of another + // version may send another shape. + if let Some(version) = request + .params + .get("protocolVersion") + .and_then(Value::as_u64) + && version != u64::from(CONTROLLER_PROTOCOL_VERSION) + { + return RpcResponse::failure( + id, + ProtocolError::version_mismatch( + CONTROLLER_PROTOCOL_VERSION, + u32::try_from(version).unwrap_or(u32::MAX), + ), + ); + } + match request.params::() { + Ok(_) => { + *initialized = true; + RpcResponse::success(request.id.clone(), app.initialize.as_ref()) } + Err(error) => RpcResponse::failure(id, error), + } +} + +async fn handle(app: &App, request: Request) -> RpcResponse { + let result = route(app, &request).await; + RpcResponse { + id: Some(request.id), + result, + } +} + +async fn route(app: &App, request: &Request) -> Result { + let Some(spec) = manifest::find(Plane::Controller, &request.method) + .filter(|spec| spec.kind == MessageKind::Request) + else { + return Err(ProtocolError::method_not_found(&request.method)); }; - let id = request.get("id").cloned().unwrap_or(Value::Null); - let method = request - .get("method") - .and_then(Value::as_str) - .unwrap_or_default(); - let params = request.get("params").cloned().unwrap_or_else(|| json!({})); - let result = match method { - INITIALIZE_METHOD => decode::(params) - .and_then(|params| { - if params.protocol_version != CURRENT_PROTOCOL_VERSION { - anyhow::bail!("unsupported protocol version") - } - Ok(ControllerInitializeResponse { - protocol_version: CURRENT_PROTOCOL_VERSION, - capabilities: ControllerCapabilities { - list_templates: true, - list_targets: true, - create_target: true, - adopt_target: true, - get_target: true, - close_target: true, - set_target_power: true, - ingress: app.config.ingress.is_some(), - }, - implementation: ImplementationInfo { - name: "lightspeed-provider-incus".to_owned(), - version: Some(release_info::VERSION.to_owned()), - git_sha: Some(release_info::GIT_SHA.to_owned()), - target: Some(release_info::TARGET.to_owned()), - }, - }) - }) - .and_then(encode), - LIST_TEMPLATES_METHOD => list_templates(app, decode(params)).await.and_then(encode), - LIST_TARGETS_METHOD => list_targets(app, decode(params)).await.and_then(encode), - CREATE_TARGET_METHOD => create_target(app, decode(params)).await.and_then(encode), - ADOPT_TARGET_METHOD => adopt_target(app, decode(params)).await.and_then(encode), - GET_TARGET_METHOD => get_target(app, decode(params)).await.and_then(encode), - CLOSE_TARGET_METHOD => close_target(app, decode(params)).await.and_then(encode), - SET_TARGET_POWER_METHOD => set_target_power(app, decode(params)).await.and_then(encode), - ENSURE_INGRESS_METHOD => ensure_ingress(app, decode(params)).await.and_then(encode), - REMOVE_INGRESS_METHOD => remove_ingress(app, decode(params)).await.and_then(encode), - _ => Err(anyhow::anyhow!("unknown controller method: {method}")), - }; - match result { - Ok(result) => json!({"jsonrpc":"2.0","id":id,"result":result}), - Err(error_value) => error( - id, - EnvironmentProtocolErrorCode::Internal, - error_value.to_string(), - ), + let capabilities = &app.initialize.capabilities; + match spec.capability { + Some(Capability::AdoptTarget) if !capabilities.adopt_target => { + return Err(ProtocolError::capability_unavailable( + Capability::AdoptTarget, + "this provider does not adopt targets", + )); + } + Some(Capability::Ingress) if !capabilities.ingress => { + return Err(ProtocolError::capability_unavailable( + Capability::Ingress, + "this provider has no public ingress configured", + )); + } + _ => {} + } + match request.method.as_str() { + TemplateList::NAME => encode(list_templates(app, request.params()?)), + TargetCreate::NAME => encode(create_target(app, request.params()?).await?), + TargetAdopt::NAME => encode(adopt_target(app, request.params()?).await?), + TargetGet::NAME => encode(get_target(app, request.params()?).await?), + TargetSetPower::NAME => encode(set_target_power(app, request.params()?).await?), + TargetClose::NAME => encode(close_target(app, request.params()?).await?), + IngressEnsure::NAME => encode(ensure_ingress(app, request.params()?).await?), + IngressRemove::NAME => encode(remove_ingress(app, request.params()?).await?), + _ => Err(ProtocolError::method_not_found(&request.method)), } } -async fn list_templates( - app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let _params = params?; - Ok(ListTemplatesResponse { +fn encode(value: T) -> Result { + serde_json::to_value(value) + .map_err(|error| ProtocolError::internal(format!("encode result: {error}"))) +} + +fn list_templates(app: &App, _params: ListTemplatesParams) -> ListTemplatesResponse { + ListTemplatesResponse { templates: app .config .templates @@ -223,463 +429,1368 @@ async fn list_templates( description: template.description.clone(), public_ingress: template.public_ingress, deprecated: template.deprecated, - metadata: BTreeMap::from([ - ( - "imageFingerprint".to_owned(), - template.image_fingerprint.clone(), - ), - ("cpu".to_owned(), template.cpu.to_string()), - ("memory".to_owned(), template.memory.clone()), - ("disk".to_owned(), template.disk.clone()), - ]), + resources: Some(template.resources.clone()), + metadata: display_metadata([( + "imageFingerprint", + template.image_fingerprint.as_str(), + )]), }) .collect(), - }) + } } -async fn list_targets( +async fn get_target( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let config = app.config.clone(); - let mut targets = stream::iter(app.backend.list_owned(binding).await?) - .map(move |mut target| { - let config = config.clone(); - async move { - observe_daemon_readiness(&config, &mut target).await; - target - } - }) - .buffer_unordered(16) - .collect::>() - .await; - if let Some(status) = params.status { - targets.retain(|target| target.status == status); - } - Ok(ListTargetsResponse { - targets: targets.into_iter().map(summary).collect(), - }) + params: GetTargetParams, +) -> Result { + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + observation(app, owner, &name, &observed, true) + .await + .map(|target| TargetResponse { target }) + .ok_or_else(|| ProtocolError::not_found(format!("no target exists for owner {name}"))) } -async fn get_target( +async fn create_target( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let mut target = app - .backend - .get_owned(binding, ¶ms.target_id) + params: CreateTargetParams, +) -> Result { + let owner = params.owner; + let name = policy::instance_name(&owner); + let observed = observe(app, &owner, &name).await?; + if observed.holds_target() { + // Idempotent per owner: an owner that holds a target gets it back, + // and no creation policy runs again. + return observed_response(app, &owner, &name, &observed).await; + } + let template = app + .config + .templates + .get(¶ms.template_id) + .ok_or_else(|| { + ProtocolError::not_found(format!( + "template {} is not offered by this provider", + params.template_id + )) + })?; + if template.deprecated { + return Err(ProtocolError::failed_precondition( + PreconditionReason::TemplateDeprecated, + format!( + "template {} is deprecated and creates no new targets", + template.template_id + ), + )); + } + let template = template.clone(); + let backend = app.backend.clone(); + let job_owner = owner.clone(); + let job_name = name.clone(); + let job = async move { + let binding = policy::binding_of(&job_owner); + if backend.get_owned(&binding, &job_name).await?.is_some() { + return Ok(()); + } + backend.reconcile_binding(&binding).await?; + backend.ensure_image(&template.image_fingerprint).await?; + backend.create_vm(&job_owner, &template).await + }; + let target = mutate(app, &owner, &name, WorkKind::Create, job.boxed()) .await? - .ok_or_else(|| anyhow::anyhow!("target not found"))?; - verify_binding(&target, binding)?; - observe_daemon_readiness(&app.config, &mut target).await; - Ok(GetTargetResponse { - target: summary(target), - }) + .ok_or_else(|| ProtocolError::unavailable("the created Incus instance is not visible"))?; + Ok(TargetResponse { target }) } -async fn create_target( +async fn adopt_target( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = params.binding.clone(); - let binding_lock = { - let mut locks = app.binding_locks.lock().await; - locks - .entry(( - params.binding.universe_id.clone(), - params.binding.binding_id.clone(), - )) - .or_default() - .clone() + params: AdoptTargetParams, +) -> Result { + let owner = params.owner; + let source = params.source_target; + let name = policy::instance_name(&owner); + let observed = observe(app, &owner, &name).await?; + let adopting_this = match (&observed.instance, &observed.slot) { + (_, Some(slot)) if slot.running() && slot.kind == WorkKind::Close => true, + (Some(instance), _) => instance.adoption_source.as_deref() == Some(source.as_str()), + (None, Some(slot)) if slot.running() => slot.kind == WorkKind::Adopt, + (None, _) => { + crate::incus::parse_source_target(source.as_str()) + .map_err(|error| ProtocolError::invalid_field("sourceTarget", error.to_string()))?; + match app + .backend + .adoption_source(&source) + .await + .map_err(unavailable)? + { + AdoptionSource::Adoptable => {} + AdoptionSource::Missing => { + return Err(ProtocolError::not_found(format!( + "adoption source {source} does not exist" + ))); + } + AdoptionSource::Managed => { + return Err(ProtocolError::failed_precondition( + PreconditionReason::OwnerMismatch, + format!("adoption source {source} already belongs to an owner"), + )); + } + AdoptionSource::NotAdoptable(reason) => { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotAdoptable, + format!("adoption source {source}: {reason}"), + )); + } + } + let backend = app.backend.clone(); + let job_owner = owner.clone(); + let job_name = name.clone(); + let job = async move { + let binding = policy::binding_of(&job_owner); + if backend.get_owned(&binding, &job_name).await?.is_some() { + return Ok(()); + } + backend.reconcile_binding(&binding).await?; + backend.adopt_vm(&job_owner, &source).await + }; + let target = mutate(app, &owner, &name, WorkKind::Adopt, job.boxed()) + .await? + .ok_or_else(|| { + ProtocolError::unavailable("the adopted Incus instance is not visible") + })?; + return Ok(TargetResponse { target }); + } }; - let _binding_guard = binding_lock.lock().await; - let template = app.config.template(¶ms.template_id)?; - app.backend.reconcile_binding(&binding).await?; - app.backend - .ensure_image(&template.image_fingerprint) - .await?; - let expected_target_id = policy::instance_name( - ¶ms.binding.universe_id, - ¶ms.binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); - if let Some(mut target) = app - .backend - .get_owned(&binding, &expected_target_id.clone().into()) + if !adopting_this { + return Err(ProtocolError::failed_precondition( + PreconditionReason::OwnerMismatch, + "the owner already holds a different target", + )); + } + observed_response(app, &owner, &name, &observed).await +} + +async fn set_target_power( + app: &App, + params: SetTargetPowerParams, +) -> Result { + let power = params.power; + if !SUPPORTED_POWER_STATES.contains(&power) { + return Err(ProtocolError::invalid_field( + "power", + format!("this provider cannot move a target to {power}"), + )); + } + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + let current = observation(app, owner, &name, &observed, false) + .await + .ok_or_else(|| ProtocolError::not_found(format!("target {name} does not exist")))?; + if current.state.phase != TargetPhase::Active { + return Err(ProtocolError::failed_precondition( + PreconditionReason::TargetNotActive, + format!("target {name} is {:?}, not active", current.state.phase), + )); + } + if current.state.power == Some(power) && !observed.slot.as_ref().is_some_and(Slot::running) { + return observed_response(app, owner, &name, &observed).await; + } + let backend = app.backend.clone(); + let binding = policy::binding_of(owner); + let job_name = name.clone(); + let job = async move { + let target = backend + .get_owned(&binding, &job_name) + .await? + .ok_or_else(|| anyhow::anyhow!("the Incus instance disappeared"))?; + backend.set_power(&target, power).await + }; + let target = mutate(app, owner, &name, WorkKind::Power(power), job.boxed()) .await? - { - verify_create(&target, ¶ms, template)?; - observe_daemon_readiness(&app.config, &mut target).await; - return Ok(CreateTargetResponse { - target: summary(target), + .ok_or_else(|| ProtocolError::not_found(format!("target {name} disappeared")))?; + Ok(TargetResponse { target }) +} + +async fn close_target( + app: &App, + params: CloseTargetParams, +) -> Result { + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + if !observed.holds_target() { + app.work.forget(&name); + return Ok(TargetResponse { + target: closed(owner), }); } - let existing = app.backend.list_owned(&binding).await?; - if let Some(target) = existing - .iter() - .find(|target| target.request_id == params.request_id) - { - verify_create(target, ¶ms, template)?; - let mut target = target.clone(); - observe_daemon_readiness(&app.config, &mut target).await; - return Ok(CreateTargetResponse { - target: summary(target), + let backend = app.backend.clone(); + let binding = policy::binding_of(owner); + let job_name = name.clone(); + let job = async move { + match backend.get_owned(&binding, &job_name).await? { + Some(target) => backend.delete_vm(&target).await, + None => Ok(()), + } + }; + app.work.start(&name, WorkKind::Close, job.boxed()); + wait_for_work(app, &name).await; + let observed = observe(app, owner, &name).await?; + if !observed.holds_target() { + return Ok(TargetResponse { + target: closed(owner), }); } - let mut target = app.backend.create_vm(&binding, template, ¶ms).await?; - verify_create(&target, ¶ms, template)?; - observe_daemon_readiness(&app.config, &mut target).await; - Ok(CreateTargetResponse { - target: summary(target), - }) + let mut target = observation(app, owner, &name, &observed, false) + .await + .unwrap_or_else(|| closed(owner)); + // Until the instance is gone, a close answers closing, with the error of + // a failed attempt as its message; the next close tries again. + target.state = TargetState { + phase: TargetPhase::Closing, + power: None, + daemon_ready: false, + message: target.state.message, + }; + target.supported_power_states.clear(); + Ok(TargetResponse { target }) } -async fn adopt_target( +async fn ensure_ingress( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = params.binding.clone(); - let binding_lock = { - let mut locks = app.binding_locks.lock().await; - locks - .entry((binding.universe_id.clone(), binding.binding_id.clone())) - .or_default() - .clone() + params: EnsureIngressParams, +) -> Result { + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + let Some(instance) = ingress_subject(app, owner, &name, &observed).await? else { + return observed_response(app, owner, &name, &observed).await; }; - let _binding_guard = binding_lock.lock().await; - app.backend.reconcile_binding(&binding).await?; - let mut target = app.backend.adopt_vm(&binding, ¶ms).await?; - verify_adoption(&target, ¶ms)?; - observe_daemon_readiness(&app.config, &mut target).await; - Ok(AdoptTargetResponse { - target: summary(target), - }) + let port = TemplateId::new(instance.template_id.as_str()) + .ok() + .and_then(|template_id| app.config.templates.get(&template_id)) + .and_then(|template| template.ingress_port) + .ok_or_else(|| { + ProtocolError::failed_precondition( + PreconditionReason::IngressUnsupported, + format!("the template of target {name} has no public ingress"), + ) + })?; + let hostname = ingress::allocated_hostname(&app.config, &instance).map_err(|error| { + ProtocolError::internal(format!("allocate ingress hostname: {error:#}")) + })?; + if instance.ingress_hostname.as_deref() == Some(hostname.as_str()) + && instance.ingress_port == Some(port) + { + return observed_response(app, owner, &name, &observed).await; + } + let backend = app.backend.clone(); + let binding = policy::binding_of(owner); + let job_name = name.clone(); + let job = async move { + let target = backend + .get_owned(&binding, &job_name) + .await? + .ok_or_else(|| anyhow::anyhow!("the Incus instance disappeared"))?; + backend.set_ingress(&target, Some((&hostname, port))).await + }; + let target = mutate( + app, + owner, + &name, + WorkKind::Ingress { enabled: true }, + job.boxed(), + ) + .await? + .ok_or_else(|| ProtocolError::not_found(format!("target {name} disappeared")))?; + Ok(TargetResponse { target }) } -async fn close_target( +async fn remove_ingress( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let expected_target_id = policy::instance_name( - ¶ms.binding.universe_id, - ¶ms.binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); - if params.target_id.as_str() != expected_target_id { - anyhow::bail!("target handle does not match deterministic ownership identity") - } - let Some(target) = app.backend.get_owned(binding, ¶ms.target_id).await? else { - return Ok(CloseTargetResponse { - target_id: params.target_id, - status: ProviderTargetStatus::Closed, - }); + params: RemoveIngressParams, +) -> Result { + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + let Some(instance) = ingress_subject(app, owner, &name, &observed).await? else { + return observed_response(app, owner, &name, &observed).await; }; - verify_binding(&target, binding)?; - if target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - { - anyhow::bail!("target ownership does not match close request") - } - app.backend - .delete_vm(binding, &target, params.force) - .await?; - Ok(CloseTargetResponse { - target_id: params.target_id, - status: ProviderTargetStatus::Closed, + if instance.ingress_hostname.is_none() && instance.ingress_port.is_none() { + return observed_response(app, owner, &name, &observed).await; + } + let backend = app.backend.clone(); + let binding = policy::binding_of(owner); + let job_name = name.clone(); + let job = async move { + let target = backend + .get_owned(&binding, &job_name) + .await? + .ok_or_else(|| anyhow::anyhow!("the Incus instance disappeared"))?; + backend.set_ingress(&target, None).await + }; + let target = mutate( + app, + owner, + &name, + WorkKind::Ingress { enabled: false }, + job.boxed(), + ) + .await? + .ok_or_else(|| ProtocolError::not_found(format!("target {name} disappeared")))?; + Ok(TargetResponse { target }) +} + +/// The instance whose ingress may change, `None` when the target is closing +/// (the caller answers with the observation), or an error when there is no +/// instance yet or at all. +async fn ingress_subject( + app: &App, + owner: &TargetOwner, + name: &str, + observed: &Observed, +) -> Result, ProtocolError> { + let current = observation(app, owner, name, observed, false) + .await + .ok_or_else(|| ProtocolError::not_found(format!("target {name} does not exist")))?; + if current.state.phase == TargetPhase::Closing { + return Ok(None); + } + observed.instance.clone().map(Some).ok_or_else(|| { + ProtocolError::failed_precondition( + PreconditionReason::TargetNotActive, + format!("target {name} has no instance yet"), + ) }) } -async fn set_target_power( +/// The instance name a reference resolves to. The provider assigns exactly +/// one name per owner; a `targetId` naming any other instance is another +/// owner's target or none at all. +async fn resolve( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let expected_target_id = policy::instance_name( - ¶ms.binding.universe_id, - ¶ms.binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); - if params.target_id.as_str() != expected_target_id { - anyhow::bail!("target handle does not match deterministic ownership identity") - } - let target = app - .backend - .get_owned(binding, ¶ms.target_id) - .await? - .ok_or_else(|| anyhow::anyhow!("target not found"))?; - verify_binding(&target, binding)?; - if target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id + target: &TargetRef, +) -> Result { + let name = policy::instance_name(&target.owner); + if let Some(target_id) = &target.target_id + && target_id.as_str() != name { - anyhow::bail!("target ownership does not match power request") + // A target being created is known by its job before its instance + // exists. + let foreign = app.work.view(target_id.as_str()).is_some() + || app + .backend + .get_owned(&policy::binding_of(&target.owner), target_id.as_str()) + .await + .map_err(unavailable)? + .is_some(); + return Err(if foreign { + owner_mismatch(target_id.as_str()) + } else { + ProtocolError::not_found(format!("target {target_id} does not exist")) + }); } - let mut target = app - .backend - .set_power(binding, &target, params.power) - .await?; - observe_daemon_readiness(&app.config, &mut target).await; - Ok(SetTargetPowerResponse { - target: summary(target), - }) + Ok(name) } -async fn ensure_ingress( +/// What the provider knows about one owner's target. +struct Observed { + instance: Option, + slot: Option, +} + +impl Observed { + /// The owner holds something: an instance or work in progress. + fn holds_target(&self) -> bool { + self.instance.is_some() || self.slot.as_ref().is_some_and(Slot::running) + } +} + +async fn observe( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let mut target = app - .backend - .get_owned(binding, ¶ms.target_id) - .await? - .ok_or_else(|| anyhow::anyhow!("target not found"))?; - verify_ingress_target( - &target, - binding, - ¶ms.environment_id, - ¶ms.incarnation_id, - )?; - let template = app - .config - .templates - .get(&target.template_id) - .ok_or_else(|| anyhow::anyhow!("target template is no longer configured"))?; - let port = template - .ingress_port - .ok_or_else(|| anyhow::anyhow!("template does not permit public ingress"))?; - let (hostname, endpoint) = ingress::endpoint(&app.config, &target)?; - target = app + owner: &TargetOwner, + name: &str, +) -> Result { + let instance = app .backend - .set_ingress(binding, &target, Some(&hostname), Some(port)) - .await?; - verify_ingress_target( - &target, - binding, - ¶ms.environment_id, - ¶ms.incarnation_id, - )?; - Ok(IngressResponse { - status: ProviderIngressStatus::Ready, - public_endpoint: Some(endpoint), + .get_owned(&policy::binding_of(owner), name) + .await + .map_err(unavailable)?; + if let Some(instance) = &instance + && instance.owner != *owner + { + return Err(owner_mismatch(name)); + } + Ok(Observed { + instance, + slot: app.work.view(name), }) } -async fn remove_ingress( +async fn observed_response( + app: &App, + owner: &TargetOwner, + name: &str, + observed: &Observed, +) -> Result { + observation(app, owner, name, observed, true) + .await + .map(|target| TargetResponse { target }) + .ok_or_else(|| ProtocolError::not_found(format!("target {name} does not exist"))) +} + +/// The observation of a target, or `None` when the owner holds nothing. +/// `probe` runs the envd liveness probe for a running instance. +/// +/// `targetId` is reported only once the Incus instance exists. Until then +/// the runtime keeps sending the idempotent `target/create` (or +/// `target/adopt`), which starts the job again when it failed or was lost +/// with a provider restart; a failed attempt is reported as `provisioning` +/// with its error as `message`, so the runtime retries it until its +/// provisioning deadline instead of failing at the first Incus hiccup. +async fn observation( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let Some(target) = app.backend.get_owned(binding, ¶ms.target_id).await? else { - return Ok(IngressResponse { - status: ProviderIngressStatus::Disabled, - public_endpoint: None, + owner: &TargetOwner, + name: &str, + observed: &Observed, + probe: bool, +) -> Option { + let failure = observed.slot.as_ref().and_then(Slot::failure); + let running = observed + .slot + .as_ref() + .filter(|slot| slot.running()) + .map(|slot| slot.kind); + let failed_creation = observed + .slot + .as_ref() + .filter(|slot| !slot.running()) + .and_then(|slot| match slot.kind { + WorkKind::Create => Some("target/create"), + WorkKind::Adopt => Some("target/adopt"), + _ => None, }); + let state = match (running, &observed.instance) { + (Some(WorkKind::Close), _) => { + phase_state(TargetPhase::Closing, None, "closing the Incus instance") + } + (Some(WorkKind::Create), _) => phase_state( + TargetPhase::Provisioning, + None, + "creating and starting the Incus instance", + ), + (Some(WorkKind::Adopt), _) => phase_state( + TargetPhase::Provisioning, + None, + "adopting the Incus instance", + ), + (_, Some(instance)) => { + let mut state = instance_state(instance); + if probe && instance.status == InstanceStatus::Running { + state.daemon_ready = relay::probe_guest(&app.config, instance).await; + } + if state.message.is_none() { + state.message = failure.and_then(DescriptiveText::lossy); + } + state + } + (_, None) => { + let retried_by = failed_creation?; + phase_state( + TargetPhase::Provisioning, + None, + &format!( + "the last attempt failed and the next {retried_by} retries it: {}", + failure.unwrap_or("unknown error") + ), + ) + } + }; + let target_id = match &observed.instance { + Some(_) => Some(ProviderTargetId::new(name).ok()?), + None => None, }; - verify_ingress_target( - &target, - binding, - ¶ms.environment_id, - ¶ms.incarnation_id, - )?; - app.backend - .set_ingress(binding, &target, None, None) - .await?; - Ok(IngressResponse { - status: ProviderIngressStatus::Disabled, - public_endpoint: None, + let instance = observed.instance.as_ref(); + let ingress = instance + .and_then(|instance| instance.ingress_hostname.as_deref()) + .and_then(|hostname| ingress::public_endpoint(&app.config, hostname).ok()) + .map(|endpoint| IngressObservation { endpoint }); + let metadata = instance.map_or_else(Metadata::default, |instance| { + display_metadata([ + ("templateId", instance.template_id.as_str()), + ("imageFingerprint", instance.image_fingerprint.as_str()), + ( + "incusMember", + instance.location.as_deref().unwrap_or_default(), + ), + ]) + }); + let supported_power_states = if state.phase == TargetPhase::Closing { + Vec::new() + } else { + SUPPORTED_POWER_STATES.to_vec() + }; + Some(TargetObservation { + target_id, + owner: owner.clone(), + display_name: None, + state, + supported_power_states, + ingress, + metadata, }) } -fn verify_ingress_target( - target: &OwnedTarget, - binding: &ProviderBindingContext, - environment: &str, - incarnation: &str, -) -> anyhow::Result<()> { - verify_binding(target, binding)?; - if target.environment_id != environment || target.incarnation_id != incarnation { - anyhow::bail!("target ownership does not match ingress request") +fn phase_state(phase: TargetPhase, power: Option, message: &str) -> TargetState { + TargetState { + phase, + power, + daemon_ready: false, + message: DescriptiveText::lossy(message), } - Ok(()) } -fn verify_binding(target: &OwnedTarget, binding: &ProviderBindingContext) -> anyhow::Result<()> { - if target.universe_id != binding.universe_id || target.binding_id != binding.binding_id { - anyhow::bail!("target belongs to another binding") +/// The two-axis state an Incus instance status maps to. A starting VM is +/// reported running and a stopping one stopped: `power` names where the +/// target is going, and `daemonReady` says whether envd answers yet. +fn instance_state(instance: &OwnedTarget) -> TargetState { + let active = |power| TargetState { + phase: TargetPhase::Active, + power: Some(power), + daemon_ready: false, + message: None, + }; + match instance.status { + InstanceStatus::Running | InstanceStatus::Starting => active(PowerState::Running), + InstanceStatus::Frozen => active(PowerState::Paused), + InstanceStatus::Stopped | InstanceStatus::Stopping => active(PowerState::Stopped), + InstanceStatus::Error => phase_state( + TargetPhase::Failed, + None, + "Incus reports the instance in the Error state", + ), + InstanceStatus::Other => phase_state( + TargetPhase::Unknown, + None, + "Incus reports an instance status this provider does not recognize", + ), } - Ok(()) } -fn verify_create( - target: &OwnedTarget, - params: &CreateTargetParams, - template: &crate::config::TemplatePolicy, -) -> anyhow::Result<()> { - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - || target.request_id != params.request_id - || target.template_id != template.template_id - || target.image_fingerprint != template.image_fingerprint - { - anyhow::bail!("existing target metadata conflicts with provision request") + +fn closed(owner: &TargetOwner) -> TargetObservation { + TargetObservation { + target_id: None, + owner: owner.clone(), + display_name: None, + state: TargetState { + phase: TargetPhase::Closed, + power: None, + daemon_ready: false, + message: None, + }, + supported_power_states: Vec::new(), + ingress: None, + metadata: Metadata::default(), } - Ok(()) } -fn verify_adoption(target: &OwnedTarget, params: &AdoptTargetParams) -> anyhow::Result<()> { - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - || target.request_id != params.request_id - || target.template_id != "adopted" - || target.image_fingerprint != "adopted" - || target.adoption_source.as_deref() != Some(params.source_target.as_str()) - { - anyhow::bail!("existing target metadata conflicts with adoption request") +/// Provider display facts; entries outside the metadata bounds are left out. +fn display_metadata<'a>(entries: impl IntoIterator) -> Metadata { + let entries: BTreeMap = entries + .into_iter() + .filter(|(key, value)| validate_metadata_entry(key, value).is_ok()) + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect(); + Metadata::new(entries).unwrap_or_default() +} + +fn unavailable(error: anyhow::Error) -> ProtocolError { + ProtocolError::unavailable(format!("Incus: {error:#}")) +} + +fn owner_mismatch(target_id: &str) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::OwnerMismatch, + format!("target {target_id} belongs to another owner"), + ) +} + +/// Start `job` for the target and wait up to the prompt bound, then +/// observe. +async fn mutate( + app: &App, + owner: &TargetOwner, + name: &str, + kind: WorkKind, + job: BoxFuture<'static, anyhow::Result<()>>, +) -> Result, ProtocolError> { + app.work.start(name, kind, job); + wait_for_work(app, name).await; + let observed = observe(app, owner, name).await?; + Ok(observation(app, owner, name, &observed, true).await) +} + +async fn wait_for_work(app: &App, name: &str) { + if let Some(mut done) = app.work.done(name) { + let _ = tokio::time::timeout(PROMPT_WAIT, done.wait_for(|finished| *finished)).await; } - Ok(()) } -async fn observe_daemon_readiness(config: &Config, target: &mut OwnedTarget) { - if target.status == ProviderTargetStatus::Ready && !relay::probe_guest(config, target).await { - target.status = ProviderTargetStatus::Starting; - } -} -fn summary(target: OwnedTarget) -> ProviderTargetSummary { - let mut metadata = BTreeMap::from([ - ("templateId".to_owned(), target.template_id), - ("imageFingerprint".to_owned(), target.image_fingerprint), - ]); - if let Some(location) = target.location { - metadata.insert("incusMember".to_owned(), location); - } - let mut capabilities = EnvironmentCapabilities::filesystem(true, true) - .with_filesystem_search() - .with_process() - .with_jobs(); - // Keep the controller-plane summary aligned with the envd handshake. The - // runtime consumes this summary before opening the data-plane connection. - capabilities.network = true; - capabilities.filesystem_glob = true; - capabilities.filesystem_ranged_read = true; - capabilities.process_output_notifications = false; - capabilities.process_pty = true; - ProviderTargetSummary { - target_id: target.target_id, - display_name: Some(target.environment_id), - status: target.status, - scope: EnvironmentScope::Default, - capabilities, - default_cwd: None, - power_states: crate::incus::SUPPORTED_POWER_STATES.to_vec(), - metadata, + +/// The background job kinds; at most one job per target is current. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum WorkKind { + Create, + Adopt, + Power(PowerState), + Ingress { enabled: bool }, + Close, +} + +#[derive(Clone, Debug)] +enum SlotState { + Running, + Failed(String), +} + +/// The current job of one target: running, or failed with its error. A job +/// that succeeds leaves no slot, because Incus then holds the truth. +#[derive(Clone, Debug)] +struct Slot { + id: u64, + kind: WorkKind, + state: SlotState, + done: watch::Receiver, +} + +impl Slot { + fn running(&self) -> bool { + matches!(self.state, SlotState::Running) + } + + fn failure(&self) -> Option<&str> { + match &self.state { + SlotState::Failed(message) => Some(message), + SlotState::Running => None, + } } } -fn decode(value: Value) -> anyhow::Result { - Ok(serde_json::from_value(value)?) + +/// Background jobs by instance name. Jobs of one target run one at a time +/// under that target's lock; a newer request of another kind replaces the +/// current slot (a close supersedes a running create) and its job runs once +/// the earlier one finished. +#[derive(Default)] +struct Work { + next_id: AtomicU64, + inner: StdMutex, } -fn encode(value: T) -> anyhow::Result { - Ok(serde_json::to_value(value)?) + +#[derive(Default)] +struct WorkInner { + slots: HashMap, + locks: HashMap>>, } -fn error(id: Value, code: EnvironmentProtocolErrorCode, message: String) -> Value { - json!({"jsonrpc":"2.0","id":id,"error":EnvironmentProtocolError::new(code,message)}) + +impl Work { + fn lock(&self) -> std::sync::MutexGuard<'_, WorkInner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// Run `job` as the target's current job unless a job of the same kind + /// is already running. + fn start( + self: &Arc, + name: &str, + kind: WorkKind, + job: BoxFuture<'static, anyhow::Result<()>>, + ) { + let mut inner = self.lock(); + if inner + .slots + .get(name) + .is_some_and(|slot| slot.kind == kind && slot.running()) + { + return; + } + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let (finished, done) = watch::channel(false); + let lock = inner.locks.entry(name.to_owned()).or_default().clone(); + inner.slots.insert( + name.to_owned(), + Slot { + id, + kind, + state: SlotState::Running, + done, + }, + ); + drop(inner); + let work = Arc::clone(self); + let name = name.to_owned(); + tokio::spawn(async move { + let outcome = { + let _guard = lock.lock().await; + AssertUnwindSafe(job) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(anyhow::anyhow!("the background job panicked"))) + }; + drop(lock); + work.finish(&name, id, outcome); + let _ = finished.send(true); + }); + } + + fn finish(&self, name: &str, id: u64, outcome: anyhow::Result<()>) { + let mut inner = self.lock(); + if inner.slots.get(name).is_some_and(|slot| slot.id == id) { + match outcome { + Ok(()) => { + inner.slots.remove(name); + } + Err(error) => { + if let Some(slot) = inner.slots.get_mut(name) { + slot.state = SlotState::Failed(format!("{error:#}")); + } + } + } + } + Self::release_lock(&mut inner, name); + } + + /// Drop the target's remembered failure once it holds nothing. + fn forget(&self, name: &str) { + let mut inner = self.lock(); + if inner.slots.get(name).is_some_and(|slot| !slot.running()) { + inner.slots.remove(name); + } + Self::release_lock(&mut inner, name); + } + + fn release_lock(inner: &mut WorkInner, name: &str) { + if !inner.slots.contains_key(name) + && inner + .locks + .get(name) + .is_some_and(|lock| Arc::strong_count(lock) == 1) + { + inner.locks.remove(name); + } + } + + fn view(&self, name: &str) -> Option { + self.lock().slots.get(name).cloned() + } + + fn done(&self, name: &str) -> Option> { + self.lock() + .slots + .get(name) + .filter(|slot| slot.running()) + .map(|slot| slot.done.clone()) + } } #[cfg(test)] mod tests { + use std::sync::atomic::AtomicUsize; + + use async_trait::async_trait; + use environment_protocol::{ + control::target::BindingRef, + error::{ErrorData, ProtocolErrorKind}, + ids::ProviderNativeRef, + jsonrpc::Id, + routes::DataRouteBase, + }; + use super::*; + use crate::{ + IncusTopology, + config::{ + ConfigInner, IncusConfig, IncusMode, NetworkPolicy, ProviderCredential, TemplatePolicy, + }, + incus::IncusMemberStatus, + policy::tests::owner, + }; - fn target() -> OwnedTarget { - OwnedTarget { - target_id: "target-a".into(), - name: "target-a".to_owned(), - universe_id: "u-a".to_owned(), - binding_id: "b-a".to_owned(), - environment_id: "e-a".to_owned(), - incarnation_id: "i-a".to_owned(), - request_id: "r-a".to_owned(), - template_id: "dev-small-v1".to_owned(), - image_fingerprint: "fingerprint-a".to_owned(), - adoption_source: None, - status: ProviderTargetStatus::Ready, - ipv4_address: Some("10.0.0.2".to_owned()), - ingress_hostname: None, - ingress_port: None, - location: Some("member-a".to_owned()), + const CREDENTIAL: &str = "0123456789abcdef0123456789abcdef"; + + /// An in-memory Incus: instances by name, every mutation instant. + #[derive(Clone, Default)] + struct FakeBackend { + instances: Arc>>, + creates: Arc, + deletes: Arc, + } + + impl FakeBackend { + fn insert(&self, owner: &TargetOwner, template_id: &str, status: InstanceStatus) { + let name = policy::instance_name(owner); + self.instances.lock().unwrap().insert( + name.clone(), + OwnedTarget { + target_id: ProviderTargetId::new(name).unwrap(), + owner: owner.clone(), + template_id: template_id.to_owned(), + image_fingerprint: "fingerprint".to_owned(), + adoption_source: None, + status, + ipv4_address: None, + ingress_hostname: None, + ingress_port: None, + location: None, + }, + ); } } - #[test] - fn ownership_checks_fence_cross_binding_and_changed_image_adoption() { - let binding = ProviderBindingContext { - universe_id: "u-a".to_owned(), - binding_id: "b-a".to_owned(), - }; - assert!(verify_binding(&target(), &binding).is_ok()); - let other = ProviderBindingContext { - universe_id: "u-b".to_owned(), - ..binding.clone() - }; - assert!(verify_binding(&target(), &other).is_err()); - let params = CreateTargetParams { - request_id: "r-a".to_owned(), - environment_id: "e-a".to_owned(), - incarnation_id: "i-a".to_owned(), - binding: ProviderBindingContext { - universe_id: "u-a".to_owned(), - binding_id: "b-a".to_owned(), - }, - template_id: "dev-small-v1".to_owned(), - }; - let mut template = crate::config::TemplatePolicy { - template_id: "dev-small-v1".to_owned(), - display_name: "Small".to_owned(), + #[async_trait] + impl IncusBackend for FakeBackend { + async fn topology(&self) -> anyhow::Result { + Ok(IncusTopology { + mode: "single", + members: vec![IncusMemberStatus { + name: "standalone".to_owned(), + status: "Online".to_owned(), + schedulable: true, + }], + }) + } + + async fn reconcile_binding(&self, _binding: &BindingRef) -> anyhow::Result<()> { + Ok(()) + } + + async fn ensure_image(&self, _fingerprint: &str) -> anyhow::Result<()> { + Ok(()) + } + + async fn list_all_owned(&self) -> anyhow::Result> { + Ok(self.instances.lock().unwrap().values().cloned().collect()) + } + + async fn get_owned( + &self, + binding: &BindingRef, + name: &str, + ) -> anyhow::Result> { + Ok(self + .instances + .lock() + .unwrap() + .get(name) + .filter(|target| target.binding() == *binding) + .cloned()) + } + + async fn create_vm( + &self, + owner: &TargetOwner, + template: &TemplatePolicy, + ) -> anyhow::Result<()> { + self.creates.fetch_add(1, Ordering::SeqCst); + self.insert( + owner, + template.template_id.as_str(), + InstanceStatus::Running, + ); + Ok(()) + } + + async fn adoption_source( + &self, + _source: &ProviderNativeRef, + ) -> anyhow::Result { + Ok(AdoptionSource::Missing) + } + + async fn adopt_vm( + &self, + _owner: &TargetOwner, + _source: &ProviderNativeRef, + ) -> anyhow::Result<()> { + anyhow::bail!("the fake backend adopts nothing") + } + + async fn delete_vm(&self, target: &OwnedTarget) -> anyhow::Result<()> { + self.deletes.fetch_add(1, Ordering::SeqCst); + self.instances.lock().unwrap().remove(target.name()); + Ok(()) + } + + async fn set_ingress( + &self, + target: &OwnedTarget, + route: Option<(&str, u16)>, + ) -> anyhow::Result<()> { + if let Some(instance) = self.instances.lock().unwrap().get_mut(target.name()) { + instance.ingress_hostname = route.map(|(hostname, _)| hostname.to_owned()); + instance.ingress_port = route.map(|(_, port)| port); + } + Ok(()) + } + + async fn set_power(&self, target: &OwnedTarget, power: PowerState) -> anyhow::Result<()> { + if let Some(instance) = self.instances.lock().unwrap().get_mut(target.name()) { + instance.status = match power { + PowerState::Running => InstanceStatus::Running, + PowerState::Paused => InstanceStatus::Frozen, + _ => InstanceStatus::Stopped, + }; + } + Ok(()) + } + } + + fn template(id: &str, deprecated: bool) -> TemplatePolicy { + TemplatePolicy { + template_id: TemplateId::new(id).unwrap(), + display_name: environment_protocol::metadata::DisplayName::new("Small").unwrap(), description: None, - image_fingerprint: "fingerprint-a".to_owned(), + image_fingerprint: "fingerprint".to_owned(), cpu: 2, memory: "4GiB".to_owned(), disk: "40GiB".to_owned(), cluster_group: None, public_ingress: false, ingress_port: None, - deprecated: false, - }; - assert!(verify_create(&target(), ¶ms, &template).is_ok()); - template.image_fingerprint = "fingerprint-b".to_owned(); - assert!(verify_create(&target(), ¶ms, &template).is_err()); + deprecated, + resources: Default::default(), + } + } + + fn config() -> Config { + let templates = [template("small", false), template("retired", true)] + .into_iter() + .map(|template| (template.template_id.clone(), template)) + .collect(); + Config(Arc::new(ConfigInner { + controller_listen: "127.0.0.1:0".parse().unwrap(), + credential: ProviderCredential::new(CREDENTIAL).unwrap(), + data_route_base: DataRouteBase::new("ws://127.0.0.1:19090/routes").unwrap(), + incus: IncusConfig { + mode: IncusMode::Single, + endpoints: vec!["https://incus.test".to_owned()], + client_certificate_pem: "cert".into(), + client_private_key_pem: "key".into(), + server_ca_pem: "ca".into(), + storage_pool: "default".to_owned(), + cluster_network_uplink: None, + image_server_url: None, + }, + templates, + network: NetworkPolicy::default(), + relay_idle_seconds: 60, + dial_timeout_seconds: 1, + envd_port: 19091, + ingress: None, + })) + } + + fn app() -> (App, FakeBackend) { + let backend = FakeBackend::default(); + (App::new(config(), backend.clone()), backend) + } + + async fn call( + app: &App, + method: &str, + params: &P, + ) -> Result { + let request = Request::new(Id::Number(1), method, params).unwrap(); + let value = route(app, &request).await?; + Ok(serde_json::from_value::(value) + .unwrap() + .target) + } + + fn by_owner(owner: &TargetOwner) -> TargetRef { + TargetRef { + owner: owner.clone(), + target_id: None, + } + } + + fn reason(error: &ProtocolError) -> Option { + match error.data { + ErrorData::FailedPrecondition { reason } => Some(reason), + _ => None, + } } #[test] - fn summary_reports_member_without_changing_target_identity() { - let summary = summary(target()); - assert_eq!(summary.target_id.as_str(), "target-a"); + fn initialize_enforces_version_and_runs_once() { + let (app, _) = app(); + let mut initialized = false; + let params = |version: u32| serde_json::json!({"protocolVersion": version, "clientInfo": {"name": "test"}}); + let request = Request::new(Id::Number(1), "initialize", ¶ms(4)).unwrap(); + let mismatch = initialize(&app, &mut initialized, &request) + .result + .unwrap_err(); + assert_eq!( + mismatch.data, + ErrorData::VersionMismatch { + expected: CONTROLLER_PROTOCOL_VERSION, + received: 4 + } + ); + assert!(!initialized); + + let request = Request::new(Id::Number(2), "initialize", ¶ms(3)).unwrap(); + let response: ControllerInitializeResponse = initialize(&app, &mut initialized, &request) + .decode() + .unwrap(); + assert!(initialized); + assert_eq!(response.plane, Plane::Controller); + assert!(response.capabilities.adopt_target); + assert!(!response.capabilities.ingress); + assert_eq!(response.routes.data.as_str(), "ws://127.0.0.1:19090/routes"); + + let repeat = initialize(&app, &mut initialized, &request) + .result + .unwrap_err(); + assert_eq!(repeat.kind(), ProtocolErrorKind::InvalidRequest); + } + + #[tokio::test(flavor = "current_thread")] + async fn unknown_and_unadvertised_methods_are_typed_errors() { + let (app, _) = app(); + let unknown = call(&app, "controller/listTargets", &serde_json::json!({})) + .await + .unwrap_err(); + assert_eq!(unknown.kind(), ProtocolErrorKind::MethodNotFound); + let ingress = call( + &app, + IngressEnsure::NAME, + &EnsureIngressParams { + target: by_owner(&owner("u", "b", "e")), + }, + ) + .await + .unwrap_err(); assert_eq!( - summary.metadata.get("incusMember").map(String::as_str), - Some("member-a") + ingress.data, + ErrorData::CapabilityUnavailable { + capability: Capability::Ingress + } ); - assert!(summary.capabilities.network); - assert!(summary.capabilities.filesystem_glob); - assert!(summary.capabilities.filesystem_ranged_read); - assert!(summary.capabilities.process_pty); - assert!(!summary.capabilities.process_output_notifications); + let undecodable = call(&app, TargetGet::NAME, &serde_json::json!({"target": 7})) + .await + .unwrap_err(); + assert_eq!(undecodable.kind(), ProtocolErrorKind::InvalidParams); + } + + #[tokio::test(flavor = "current_thread")] + async fn get_never_creates_and_answers_not_found() { + let (app, backend) = app(); + let owner = owner("u", "b", "e"); + for _ in 0..2 { + let error = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + } + let unknown = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: Some(ProviderTargetId::new("default/pod-abc").unwrap()), + }, + }, + ) + .await + .unwrap_err(); + assert_eq!(unknown.kind(), ProtocolErrorKind::NotFound); + assert_eq!(backend.creates.load(Ordering::SeqCst), 0); + assert!(backend.instances.lock().unwrap().is_empty()); + } + + #[tokio::test(flavor = "current_thread")] + async fn create_is_idempotent_per_owner_and_applies_policy_only_once() { + let (app, backend) = app(); + let owner = owner("u", "b", "e"); + let params = CreateTargetParams { + owner: owner.clone(), + template_id: TemplateId::new("small").unwrap(), + }; + let first = call(&app, TargetCreate::NAME, ¶ms).await.unwrap(); + let second = call(&app, TargetCreate::NAME, ¶ms).await.unwrap(); + assert_eq!(first.owner, owner); + assert_eq!(first.target_id, second.target_id); assert_eq!( - summary.power_states, - vec![PowerState::Running, PowerState::Paused, PowerState::Stopped] + first.target_id.as_ref().map(ProviderTargetId::as_str), + Some(policy::instance_name(&owner).as_str()) + ); + assert_eq!(second.state.phase, TargetPhase::Active); + assert_eq!(second.state.power, Some(PowerState::Running)); + assert_eq!(backend.creates.load(Ordering::SeqCst), 1); + + // A target that already exists comes back even when its template + // has been deprecated since: observation applies no creation policy. + let existing = crate::policy::tests::owner("u", "b", "old"); + backend.insert(&existing, "retired", InstanceStatus::Stopped); + let observed = call( + &app, + TargetCreate::NAME, + &CreateTargetParams { + owner: existing.clone(), + template_id: TemplateId::new("retired").unwrap(), + }, + ) + .await + .unwrap(); + assert_eq!(observed.state.power, Some(PowerState::Stopped)); + assert_eq!(backend.creates.load(Ordering::SeqCst), 1); + + let refused = call( + &app, + TargetCreate::NAME, + &CreateTargetParams { + owner: crate::policy::tests::owner("u", "b", "new"), + template_id: TemplateId::new("retired").unwrap(), + }, + ) + .await + .unwrap_err(); + assert_eq!( + reason(&refused), + Some(PreconditionReason::TemplateDeprecated) + ); + let missing = call( + &app, + TargetCreate::NAME, + &CreateTargetParams { + owner: crate::policy::tests::owner("u", "b", "new"), + template_id: TemplateId::new("absent").unwrap(), + }, + ) + .await + .unwrap_err(); + assert_eq!(missing.kind(), ProtocolErrorKind::NotFound); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_foreign_target_id_is_an_owner_mismatch() { + let (app, backend) = app(); + let holder = owner("u", "b", "holder"); + backend.insert(&holder, "small", InstanceStatus::Running); + let error = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: TargetRef { + owner: owner("u", "b", "other"), + target_id: Some(ProviderTargetId::new(policy::instance_name(&holder)).unwrap()), + }, + }, + ) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::OwnerMismatch)); + } + + #[tokio::test(flavor = "current_thread")] + async fn close_by_owner_destroys_and_is_idempotent() { + let (app, backend) = app(); + let owner = owner("u", "b", "e"); + backend.insert(&owner, "small", InstanceStatus::Running); + let closed = call( + &app, + TargetClose::NAME, + &CloseTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap(); + assert_eq!(closed.state.phase, TargetPhase::Closed); + assert_eq!(closed.target_id, None); + assert_eq!(closed.owner, owner); + let again = call( + &app, + TargetClose::NAME, + &CloseTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap(); + assert_eq!(again.state.phase, TargetPhase::Closed); + assert_eq!(backend.deletes.load(Ordering::SeqCst), 1); + let gone = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap_err(); + assert_eq!(gone.kind(), ProtocolErrorKind::NotFound); + } + + #[tokio::test(flavor = "current_thread")] + async fn power_changes_need_an_active_target_and_a_supported_state() { + let (app, backend) = app(); + let owner = owner("u", "b", "e"); + let set = |power| SetTargetPowerParams { + target: by_owner(&owner), + power, + }; + let missing = call(&app, TargetSetPower::NAME, &set(PowerState::Paused)) + .await + .unwrap_err(); + assert_eq!(missing.kind(), ProtocolErrorKind::NotFound); + + backend.insert(&owner, "small", InstanceStatus::Running); + let unsupported = call(&app, TargetSetPower::NAME, &set(PowerState::Suspended)) + .await + .unwrap_err(); + assert_eq!(unsupported.kind(), ProtocolErrorKind::InvalidParams); + let paused = call(&app, TargetSetPower::NAME, &set(PowerState::Paused)) + .await + .unwrap(); + assert_eq!(paused.state.power, Some(PowerState::Paused)); + + backend.insert(&owner, "small", InstanceStatus::Error); + let failed = call(&app, TargetSetPower::NAME, &set(PowerState::Running)) + .await + .unwrap_err(); + assert_eq!(reason(&failed), Some(PreconditionReason::TargetNotActive)); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_failed_create_is_provisioning_without_an_id_and_retried_by_create() { + let (app, backend) = app(); + let owner = owner("u", "b", "e"); + let name = policy::instance_name(&owner); + app.work.start( + &name, + WorkKind::Create, + async { Err(anyhow::anyhow!("image server unreachable")) }.boxed(), + ); + wait_for_work(&app, &name).await; + let observed = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap(); + // No instance exists, so no id is handed out and the runtime keeps + // sending target/create. + assert_eq!(observed.state.phase, TargetPhase::Provisioning); + assert_eq!(observed.target_id, None); + assert!( + observed + .state + .message + .as_ref() + .is_some_and(|message| message.as_str().contains("image server unreachable")) + ); + let created = call( + &app, + TargetCreate::NAME, + &CreateTargetParams { + owner: owner.clone(), + template_id: TemplateId::new("small").unwrap(), + }, + ) + .await + .unwrap(); + assert_eq!(created.state.phase, TargetPhase::Active); + assert_eq!( + created.target_id.as_ref().map(ProviderTargetId::as_str), + Some(name.as_str()) + ); + assert_eq!(backend.creates.load(Ordering::SeqCst), 1); + } + + #[tokio::test(flavor = "current_thread")] + async fn closing_an_owner_whose_create_failed_forgets_the_failure() { + let (app, _) = app(); + let owner = owner("u", "b", "e"); + let name = policy::instance_name(&owner); + app.work.start( + &name, + WorkKind::Create, + async { Err(anyhow::anyhow!("image pull failed")) }.boxed(), + ); + wait_for_work(&app, &name).await; + // Closing an owner whose create failed before any instance existed + // holds nothing, so it closes at once and forgets the failure. + let closed = call( + &app, + TargetClose::NAME, + &CloseTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap(); + assert_eq!(closed.state.phase, TargetPhase::Closed); + assert!(app.work.view(&name).is_none()); + } + + #[test] + fn only_the_exact_bearer_credential_is_authorized() { + let config = config(); + let mut headers = HeaderMap::new(); + assert!(!authorized(&config, &headers)); + headers.insert(header::AUTHORIZATION, "Bearer wrong".parse().unwrap()); + assert!(!authorized(&config, &headers)); + headers.insert(header::AUTHORIZATION, CREDENTIAL.parse().unwrap()); + assert!(!authorized(&config, &headers)); + headers.insert( + header::AUTHORIZATION, + format!("Bearer {CREDENTIAL}").parse().unwrap(), + ); + assert!(authorized(&config, &headers)); + } + + /// The client's controller conformance suite against this server over + /// a real WebSocket, with the in-memory backend standing in for Incus. + #[tokio::test(flavor = "current_thread")] + async fn the_controller_passes_the_conformance_suite() { + use environment_client::{ + ConnectOptions, + conformance::controller::{ControllerConformance, run}, + }; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(serve_on(listener, config(), FakeBackend::default())); + let options = ConnectOptions::new(ImplementationInfo::new("conformance").unwrap()) + .with_bearer_token(CREDENTIAL); + let counter = Arc::new(AtomicUsize::new(0)); + let owner_factory: Arc TargetOwner + Send + Sync> = Arc::new(move || { + let environment = format!("env-{}", counter.fetch_add(1, Ordering::SeqCst)); + owner("conformance", "fake", &environment) + }); + let mut suite = ControllerConformance::new( + format!("ws://{address}/control"), + options, + TemplateId::new("small").unwrap(), + owner_factory, ); + suite.poll_interval = Duration::from_millis(10); + suite.settle_timeout = Duration::from_secs(10); + let report = run(&suite).await; + server.abort(); + report.into_result().expect("the controller conforms"); } } diff --git a/crates/environment-provider-incus/tests/incus_live.rs b/crates/environment-provider-incus/tests/incus_live.rs index 72c46323d..f4a4b116b 100644 --- a/crates/environment-provider-incus/tests/incus_live.rs +++ b/crates/environment-provider-incus/tests/incus_live.rs @@ -1,19 +1,42 @@ -use std::path::PathBuf; +use std::{ + path::PathBuf, + sync::Arc, + time::{Duration, Instant}, +}; -use environment_provider_incus::{IncusBackend as _, IncusClient, ProviderArgs}; +use environment_client::{ + ConnectOptions, ControllerClient, DataClient, + conformance::controller::{ControllerConformance, run}, +}; +use environment_protocol::{ + control::target::{ + CloseTargetParams, CreateTargetParams, GetTargetParams, PowerState, TargetOwner, + TargetPhase, TargetRef, + }, + handshake::ImplementationInfo, + ids::{BindingId, EnvironmentId, IncarnationId, ProviderTargetId, TemplateId, UniverseId}, + paths::EnvironmentPath, + routes::provider_data_route, +}; +use environment_provider_incus::{Config, IncusBackend as _, IncusClient, ProviderArgs, server}; +use tools::environment_protocol::conformance::{self as data_conformance, DataConformance}; -#[tokio::test(flavor = "current_thread")] -#[ignore = "requires a reachable standalone Incus server or Incus cluster"] -async fn configured_topology_and_owned_inventory_are_live() { +async fn live_config() -> Config { let config_path = std::env::var("LIGHTSPEED_INCUS_PROVIDER_CONFIG") .map(PathBuf::from) .expect("LIGHTSPEED_INCUS_PROVIDER_CONFIG must name a live provider config"); - let config = ProviderArgs { + ProviderArgs { config: config_path, } .load() .await - .expect("load live provider config"); + .expect("load live provider config") +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a reachable standalone Incus server or Incus cluster"] +async fn configured_topology_and_owned_inventory_are_live() { + let config = live_config().await; let backend = IncusClient::new(config.clone()) .await .expect("connect and validate live Incus topology"); @@ -31,3 +54,190 @@ async fn configured_topology_and_owned_inventory_are_live() { } } } + +/// One owner in the fixed conformance universe and binding. +fn conformance_owner() -> TargetOwner { + TargetOwner { + universe_id: UniverseId::new("lightspeed-conformance").expect("universe id"), + binding_id: BindingId::new("incus-live").expect("binding id"), + environment_id: EnvironmentId::new(uuid::Uuid::new_v4().to_string()) + .expect("environment id"), + incarnation_id: IncarnationId::new(uuid::Uuid::new_v4().to_string()) + .expect("incarnation id"), + } +} + +fn first_template(config: &Config) -> TemplateId { + config + .templates + .values() + .find(|template| !template.deprecated) + .map(|template| template.template_id.clone()) + .expect("the live config offers a template") +} + +/// Create a target for `owner` the way the runtime does: `target/create` +/// until the observation carries a `targetId`, then `target/get` until the +/// provider's own probe reaches envd. +async fn create_until_daemon_ready( + controller: &ControllerClient, + owner: &TargetOwner, + template_id: &TemplateId, + timeout: Duration, +) -> anyhow::Result { + let deadline = Instant::now() + timeout; + let params = CreateTargetParams { + owner: owner.clone(), + template_id: template_id.clone(), + }; + let mut observation = controller.create_target(¶ms).await?.target; + loop { + if observation.state.phase == TargetPhase::Failed { + anyhow::bail!("the target failed: {:?}", observation.state.message); + } + if observation.state.phase == TargetPhase::Active + && observation.state.power == Some(PowerState::Running) + && observation.state.daemon_ready + && let Some(target_id) = observation.target_id.clone() + { + return Ok(target_id); + } + if Instant::now() >= deadline { + anyhow::bail!( + "envd did not come up within {timeout:?}: {:?}", + observation.state + ); + } + tokio::time::sleep(Duration::from_secs(2)).await; + observation = match &observation.target_id { + Some(target_id) => { + controller + .get_target(&GetTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: Some(target_id.clone()), + }, + }) + .await? + .target + } + None => controller.create_target(¶ms).await?.target, + }; + } +} + +/// Close by owner until the provider answers `closed`. +async fn close_until_closed( + controller: &ControllerClient, + owner: &TargetOwner, + timeout: Duration, +) -> anyhow::Result<()> { + let deadline = Instant::now() + timeout; + loop { + let observation = controller + .close_target(&CloseTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: None, + }, + }) + .await? + .target; + if observation.state.phase == TargetPhase::Closed { + return Ok(()); + } + if Instant::now() >= deadline { + anyhow::bail!("the target did not close within {timeout:?}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +/// The tools' data conformance suite against envd inside a live Incus VM, +/// reached through this provider's data route and relay with the provider +/// credential, exactly as the runtime reaches it. Creates one VM and +/// closes it. +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a reachable standalone Incus server or Incus cluster; creates and deletes one VM"] +async fn the_guest_envd_conforms_on_live_incus() { + let config = live_config().await; + let template_id = first_template(&config); + let credential = config.credential.expose().to_owned(); + let backend = IncusClient::new(config.clone()) + .await + .expect("connect and validate live Incus topology"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind the controller listener"); + let address = listener.local_addr().expect("listener address"); + let server = tokio::spawn(server::serve_on(listener, config, backend)); + + let options = ConnectOptions::new( + ImplementationInfo::new("environment-provider-incus-live").expect("client info"), + ) + .with_bearer_token(credential); + let (controller, initialized) = + ControllerClient::connect_initialized(&format!("ws://{address}/control"), options.clone()) + .await + .expect("controller handshake"); + let owner = conformance_owner(); + let outcome = async { + let target_id = + create_until_daemon_ready(&controller, &owner, &template_id, Duration::from_secs(900)) + .await?; + let route = provider_data_route(&initialized.routes.data, &owner, &target_id); + let (client, _) = DataClient::connect_initialized(&route, options.clone()).await?; + let suite = DataConformance::new( + client.clone(), + EnvironmentPath::new("/workspace").expect("work directory"), + ); + let report = data_conformance::run(&suite).await; + let _ = client.close().await; + report + .into_result() + .map(|_| ()) + .map_err(anyhow::Error::from) + } + .await; + let closed = close_until_closed(&controller, &owner, Duration::from_secs(300)).await; + let _ = controller.close().await; + server.abort(); + outcome.expect("the guest envd conforms"); + closed.expect("the conformance target closes"); +} + +/// The client's controller conformance suite against this provider on live +/// Incus. It creates one VM from the first offered template and closes it; +/// every owner lives in one fixed conformance universe and binding, so +/// repeated runs reuse one Incus project and network. +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a reachable standalone Incus server or Incus cluster; creates and deletes one VM"] +async fn the_controller_conforms_on_live_incus() { + let config = live_config().await; + let template_id = first_template(&config); + let credential = config.credential.expose().to_owned(); + let backend = IncusClient::new(config.clone()) + .await + .expect("connect and validate live Incus topology"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind the controller listener"); + let address = listener.local_addr().expect("listener address"); + let server = tokio::spawn(server::serve_on(listener, config, backend)); + + let options = ConnectOptions::new( + ImplementationInfo::new("environment-provider-incus-live").expect("client info"), + ) + .with_bearer_token(credential); + let owner = Arc::new(conformance_owner); + let mut suite = ControllerConformance::new( + format!("ws://{address}/control"), + options, + template_id, + owner, + ); + suite.settle_timeout = Duration::from_secs(900); + let report = run(&suite).await; + server.abort(); + report.into_result().expect("the Incus controller conforms"); +} diff --git a/crates/environments/src/endpoint.rs b/crates/environments/src/endpoint.rs new file mode 100644 index 000000000..ff711fa1f --- /dev/null +++ b/crates/environments/src/endpoint.rs @@ -0,0 +1,270 @@ +//! Environment endpoints: where Lightspeed dials a provider controller or an +//! environment's data plane. +//! +//! An endpoint is one validated URL whose scheme selects the dialer from a +//! closed set; there is no separate transport enum. Every scheme carries +//! WebSocket text frames running the environment protocol's JSON-RPC +//! envelope: +//! +//! - `ws://host[:port][/path]` and `wss://host[:port][/path]`: TCP, plain or +//! TLS. +//! - `unix:///absolute/socket/path`: a local Unix socket (envd's Unix +//! listener). The dialer connects to the socket and performs the +//! WebSocket upgrade with request target `/` and host `localhost`. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use thiserror::Error; + +/// Longest endpoint URL in bytes. +pub const MAX_ENDPOINT_BYTES: usize = 2048; + +/// The dialer an endpoint selects. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum EndpointScheme { + Ws, + Wss, + Unix, +} + +impl EndpointScheme { + pub fn as_str(self) -> &'static str { + match self { + Self::Ws => "ws", + Self::Wss => "wss", + Self::Unix => "unix", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum EndpointError { + #[error("endpoint is empty")] + Empty, + + #[error("endpoint is longer than {MAX_ENDPOINT_BYTES} bytes")] + TooLong, + + #[error("endpoint must not contain whitespace or control characters: {0}")] + InvalidCharacter(String), + + #[error("endpoint scheme must be ws://, wss:// or unix://: {0}")] + Scheme(String), + + #[error("endpoint has no host: {0}")] + NoHost(String), + + #[error("endpoint must not carry user information: {0}")] + UserInfo(String), + + #[error("endpoint must not carry a fragment: {0}")] + Fragment(String), + + #[error("endpoint has an invalid port: {0}")] + Port(String), + + #[error("unix endpoint must name an absolute socket path without query: {0}")] + UnixPath(String), +} + +/// A validated endpoint URL. Serializes as the URL string. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EnvironmentEndpoint { + url: String, + scheme: EndpointScheme, +} + +impl EnvironmentEndpoint { + pub fn parse(value: &str) -> Result { + if value.is_empty() { + return Err(EndpointError::Empty); + } + if value.len() > MAX_ENDPOINT_BYTES { + return Err(EndpointError::TooLong); + } + if value + .chars() + .any(|character| character.is_whitespace() || character.is_control()) + { + return Err(EndpointError::InvalidCharacter(value.to_owned())); + } + let (scheme, rest) = value + .split_once("://") + .ok_or_else(|| EndpointError::Scheme(value.to_owned()))?; + let scheme = match scheme { + "ws" => EndpointScheme::Ws, + "wss" => EndpointScheme::Wss, + "unix" => EndpointScheme::Unix, + _ => return Err(EndpointError::Scheme(value.to_owned())), + }; + if rest.contains('#') { + return Err(EndpointError::Fragment(value.to_owned())); + } + match scheme { + EndpointScheme::Unix => { + if !rest.starts_with('/') || rest.len() == 1 || rest.contains('?') { + return Err(EndpointError::UnixPath(value.to_owned())); + } + } + EndpointScheme::Ws | EndpointScheme::Wss => { + let authority = rest.split(['/', '?']).next().unwrap_or_default(); + if authority.contains('@') { + return Err(EndpointError::UserInfo(value.to_owned())); + } + let host = + host_of(authority).ok_or_else(|| EndpointError::Port(value.to_owned()))?; + if host.is_empty() || host == "[]" { + return Err(EndpointError::NoHost(value.to_owned())); + } + } + } + Ok(Self { + url: value.to_owned(), + scheme, + }) + } + + pub fn as_str(&self) -> &str { + &self.url + } + + pub fn scheme(&self) -> EndpointScheme { + self.scheme + } + + /// The socket path of a `unix://` endpoint. + pub fn unix_socket_path(&self) -> Option<&str> { + match self.scheme { + EndpointScheme::Unix => self.url.strip_prefix("unix://"), + EndpointScheme::Ws | EndpointScheme::Wss => None, + } + } + + /// The host of a `ws://` or `wss://` endpoint, brackets kept for IPv6. + pub fn host(&self) -> Option<&str> { + let rest = match self.scheme { + EndpointScheme::Ws => self.url.strip_prefix("ws://")?, + EndpointScheme::Wss => self.url.strip_prefix("wss://")?, + EndpointScheme::Unix => return None, + }; + host_of(rest.split(['/', '?']).next().unwrap_or_default()) + } + + /// True for a Unix socket or a `localhost` or loopback-address host: + /// endpoints whose traffic never leaves the machine. + pub fn is_local(&self) -> bool { + match self.scheme { + EndpointScheme::Unix => true, + EndpointScheme::Ws | EndpointScheme::Wss => self.host().is_some_and(|host| { + let host = host.trim_start_matches('[').trim_end_matches(']'); + host.eq_ignore_ascii_case("localhost") + || host + .parse::() + .is_ok_and(|address| address.is_loopback()) + }), + } + } +} + +fn host_of(authority: &str) -> Option<&str> { + if let Some(rest) = authority.strip_prefix('[') { + let (address, after) = rest.split_once(']')?; + if !after.is_empty() { + after.strip_prefix(':')?.parse::().ok()?; + } + return Some(&authority[..address.len() + 2]); + } + match authority.rsplit_once(':') { + Some((host, port)) => { + port.parse::().ok()?; + Some(host) + } + None => Some(authority), + } +} + +impl fmt::Display for EnvironmentEndpoint { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.url.fmt(formatter) + } +} + +impl FromStr for EnvironmentEndpoint { + type Err = EndpointError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } +} + +impl Serialize for EnvironmentEndpoint { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.url) + } +} + +impl<'de> Deserialize<'de> for EnvironmentEndpoint { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::parse(&value).map_err(de::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_scheme_selects_the_dialer() { + let tcp = EnvironmentEndpoint::parse("wss://incus.example.com:19090/control").unwrap(); + assert_eq!(tcp.scheme(), EndpointScheme::Wss); + assert_eq!(tcp.host(), Some("incus.example.com")); + assert!(!tcp.is_local()); + let local = + EnvironmentEndpoint::parse("ws://127.0.0.1:18080/environment-gateway/routes/u/e/i") + .unwrap(); + assert!(local.is_local()); + let unix = EnvironmentEndpoint::parse("unix:///run/lightspeed/envd.sock").unwrap(); + assert_eq!(unix.scheme(), EndpointScheme::Unix); + assert_eq!(unix.unix_socket_path(), Some("/run/lightspeed/envd.sock")); + assert!(unix.is_local()); + assert!( + EnvironmentEndpoint::parse("ws://[::1]:9/x") + .unwrap() + .is_local() + ); + } + + #[test] + fn other_schemes_and_malformed_urls_are_rejected() { + for bad in [ + "", + "http://host/control", + "ssh://host", + "stdio", + "ws://", + "ws://user@host", + "ws://host:port", + "ws://host/a b", + "ws://host/#frag", + "unix://relative.sock", + "unix:///", + ] { + assert!(EnvironmentEndpoint::parse(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn endpoints_serialize_as_strings_and_validate_on_decode() { + let endpoint: EnvironmentEndpoint = + serde_json::from_value(serde_json::json!("wss://gw.example.com/x")).unwrap(); + assert_eq!( + serde_json::to_value(&endpoint).unwrap(), + "wss://gw.example.com/x" + ); + assert!( + serde_json::from_value::(serde_json::json!("http://x")).is_err() + ); + } +} diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 85001130d..7a10c655e 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -8,10 +8,13 @@ use std::{collections::BTreeMap, fmt, str::FromStr}; use async_trait::async_trait; use auth::{AuthGrantId, AuthProviderId, SecretId, SecretValue}; -pub use environment_protocol::control::targets::PowerState; +pub use environment_protocol::control::target::PowerState; use environment_protocol::{ - control::targets::EnvironmentTemplate, - shared::{EnvironmentTransport, ProviderTargetId}, + control::template::EnvironmentTemplate, + handshake::ImplementationInfo, + ids::{ProviderTargetId, validate_opaque_id}, + metadata::{DescriptiveText, validate_descriptive_text, validate_stored_metadata}, + routes::DataRouteBase, }; pub use harness::EnvironmentId; use harness::{StringIdError, validate_general_string_id}; @@ -186,31 +189,32 @@ pub enum EnvironmentRegistryError { Store { message: String }, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentConnectionSpec { - pub endpoint: String, - pub transport: EnvironmentTransport, -} - -impl EnvironmentConnectionSpec { - pub fn new(endpoint: impl Into, transport: EnvironmentTransport) -> Self { - Self { - endpoint: endpoint.into(), - transport, - } - } - - pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { - validate_endpoint("provider controller endpoint", &self.endpoint) - } +/// Optional controller methods a provider's controller advertised in its +/// last successful `initialize`. Calls and API requests that need an absent +/// feature are refused before they reach the provider. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ControllerFeatures { + /// `target/adopt` is served. + pub adopt_target: bool, + /// `ingress/ensure` and `ingress/remove` are served. + pub ingress: bool, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct EnvironmentProviderRecord { pub provider_id: EnvironmentProviderId, pub display_name: Option, - pub controller_connection: EnvironmentConnectionSpec, + /// Where Lightspeed dials the provider's controller. + pub controller_endpoint: EnvironmentEndpoint, pub metadata: BTreeMap, + /// Recorded at the controller's last successful `initialize`, together + /// with `data_route_base`; absent until then and after the endpoint + /// changes. + pub controller_features: Option, + /// Base URL of the provider's data routes as its controller advertised + /// it (`routes.data`). The gateway builds every target's data route from + /// it; it is never derived from the controller endpoint. + pub data_route_base: Option, pub created_at_ms: i64, pub updated_at_ms: i64, } @@ -218,38 +222,93 @@ pub struct EnvironmentProviderRecord { impl EnvironmentProviderRecord { pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { validate_nonempty_optional("display_name", self.display_name.as_deref())?; - self.controller_connection.validate()?; validate_metadata(&self.metadata)?; validate_timestamps(self.created_at_ms, self.updated_at_ms)?; + if self.controller_features.is_some() != self.data_route_base.is_some() { + return invalid("controller features and data route base are recorded together"); + } Ok(()) } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Register or replace a provider. The provider credential is write-only: +/// the store keeps it sealed and hands it out only through +/// [`EnvironmentProviderStore::read_provider_credential`]. +#[derive(Clone, Debug, PartialEq, Eq)] pub struct PutEnvironmentProvider { pub provider_id: EnvironmentProviderId, pub display_name: Option, - pub controller_connection: EnvironmentConnectionSpec, + pub controller_endpoint: EnvironmentEndpoint, pub metadata: BTreeMap, + /// The bearer credential Lightspeed presents on the controller and data + /// routes. Required for a new provider; `None` keeps the stored one. + pub credential: Option, pub updated_at_ms: i64, } impl PutEnvironmentProvider { - pub fn into_record(self) -> Result { + /// The record this put writes for a provider whose current record is + /// `existing`: the creation time is kept, and the recorded controller + /// facts survive only while the endpoint stays the same. + pub fn into_record( + self, + existing: Option<&EnvironmentProviderRecord>, + ) -> Result { validate_nonnegative_i64(self.updated_at_ms, "updated_at_ms")?; + if let Some(credential) = &self.credential { + validate_provider_credential(credential)?; + } else if existing.is_none() { + return invalid("a new environment provider requires a credential"); + } + let kept = existing.filter(|record| record.controller_endpoint == self.controller_endpoint); + let created_at_ms = existing.map_or(self.updated_at_ms, |record| record.created_at_ms); let record = EnvironmentProviderRecord { provider_id: self.provider_id, display_name: self.display_name, - controller_connection: self.controller_connection, + controller_endpoint: self.controller_endpoint, metadata: self.metadata, - created_at_ms: self.updated_at_ms, - updated_at_ms: self.updated_at_ms, + controller_features: kept.and_then(|record| record.controller_features), + data_route_base: kept.and_then(|record| record.data_route_base.clone()), + created_at_ms, + updated_at_ms: self.updated_at_ms.max(created_at_ms), }; record.validate()?; Ok(record) } } +/// Longest accepted provider credential, in bytes. +pub const PROVIDER_CREDENTIAL_MAX_BYTES: usize = 4096; + +/// A provider credential travels as an HTTP bearer token: 1 to +/// [`PROVIDER_CREDENTIAL_MAX_BYTES`] bytes of visible ASCII. +pub fn validate_provider_credential( + credential: &SecretValue, +) -> Result<(), EnvironmentRegistryError> { + let value = credential.expose(); + if value.is_empty() + || value.len() > PROVIDER_CREDENTIAL_MAX_BYTES + || !value.bytes().all(|byte| byte.is_ascii_graphic()) + { + return invalid(format!( + "provider credential must be 1..={PROVIDER_CREDENTIAL_MAX_BYTES} bytes of visible ASCII" + )); + } + Ok(()) +} + +/// What a controller advertised at a successful `initialize`. Written only +/// while the provider is still registered at `controller_endpoint`, so an +/// `initialize` against an endpoint the operator has since replaced cannot +/// overwrite the new endpoint's facts. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordProviderController { + pub provider_id: EnvironmentProviderId, + pub controller_endpoint: EnvironmentEndpoint, + pub features: ControllerFeatures, + pub data_route_base: DataRouteBase, +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ListEnvironmentProviders {} @@ -330,9 +389,8 @@ pub enum EnvironmentSource { provider_id: EnvironmentProviderId, binding_id: EnvironmentProviderBindingId, }, - External { - connection: EnvironmentConnectionSpec, - }, + /// An `envd` Lightspeed dials directly at a fixed endpoint. + External { endpoint: EnvironmentEndpoint }, /// An `envd` that dialed the gateway outbound and was admitted by a /// registration key. The daemon public key is the identity: one key /// maps to at most one environment in the deployment, ever, so a closed @@ -421,6 +479,12 @@ impl EnvironmentStatus { pub fn is_powered_down(self) -> bool { matches!(self, Self::Paused | Self::Suspended | Self::Offline) } + + /// True once a close was accepted: no observation may change the status + /// any more except the close's own completion. + pub fn is_closing_or_closed(self) -> bool { + matches!(self, Self::Closing | Self::Closed) + } } /// Staged idle policy: each threshold is measured against the daemon's @@ -546,17 +610,65 @@ pub struct EnvironmentRecord { /// daemon's idle report. pub idle_policy: Option, pub incarnation: EnvironmentIncarnationRecord, + /// Lightspeed-owned public ingress intent. The lifecycle reconciler + /// converges the provider target toward it while the environment is + /// ready; the realized endpoint is `public_endpoint`. pub public_ingress_enabled: bool, + /// The public endpoint the provider last reported realized. It can lag + /// the intent in either direction while the reconciler converges. pub public_endpoint: Option, pub metadata: BTreeMap, /// Registered environments only: when the gateway last saw the daemon's /// control connection (connect, heartbeat). A stale stamp under `Ready` /// means the gateway stopped without recording the disconnect. pub last_seen_at_ms: Option, + /// Why the environment failed, or the latest error while its lifecycle + /// keeps retrying. Cleared once an observation reaches a steady state. + pub lifecycle_error: Option, pub created_at_ms: i64, pub updated_at_ms: i64, } +/// The latest lifecycle error of an environment and how long errors have +/// persisted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentLifecycleError { + /// 1 to 1024 bytes without control characters other than line feed and + /// tab. + pub message: String, + /// When the current run of errors began: the first error recorded since + /// the environment was last steady. + pub first_at_ms: i64, + /// When `message` was recorded. + pub last_at_ms: i64, +} + +impl EnvironmentLifecycleError { + /// The error recorded at `at_ms`, continuing the run of `previous`. + /// `message` is cleaned and cut to the stored bound. + pub fn record(previous: Option<&Self>, message: &str, at_ms: i64) -> Self { + let message = DescriptiveText::lossy(message) + .map(DescriptiveText::into_string) + .unwrap_or_else(|| "unspecified lifecycle error".to_owned()); + let first_at_ms = previous.map_or(at_ms, |previous| previous.first_at_ms.min(at_ms)); + Self { + message, + first_at_ms, + last_at_ms: at_ms.max(first_at_ms), + } + } + + pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { + validate_descriptive_text(&self.message) + .map_err(|error| invalid_error(format!("lifecycle error message: {error}")))?; + validate_nonnegative_i64(self.first_at_ms, "lifecycle_error.first_at_ms")?; + if self.last_at_ms < self.first_at_ms { + return invalid("lifecycle_error.last_at_ms must be >= first_at_ms"); + } + Ok(()) + } +} + impl EnvironmentRecord { pub fn registration_key_id(&self) -> Option<&EnvironmentRegistrationKeyId> { self.source.registration_key_id() @@ -591,6 +703,36 @@ impl EnvironmentRecord { .is_some_and(|observed| observed != self.desired_power) } + /// True for a ready provisioned environment whose realized public + /// endpoint does not match its ingress intent yet. + pub fn ingress_diverges(&self) -> bool { + matches!(self.source, EnvironmentSource::Provisioned { .. }) + && self.status == EnvironmentStatus::Ready + && self.public_ingress_enabled != self.public_endpoint.is_some() + } + + /// True for a provisioned environment at rest in a power state whose + /// provider target is recorded: only a periodic observation notices a + /// target that changed or vanished out of band. + pub fn is_steady_provisioned(&self) -> bool { + matches!(self.source, EnvironmentSource::Provisioned { .. }) + && self.status.power_state().is_some() + && self.incarnation.provider_target_id.is_some() + } + + /// Whether a reconcile pass of `scope` visits this environment. + pub fn needs_reconcile(&self, scope: ReconcileScope) -> bool { + matches!( + self.status, + EnvironmentStatus::Provisioning + | EnvironmentStatus::Booting + | EnvironmentStatus::Closing + | EnvironmentStatus::Unknown + ) || self.power_diverges() + || self.ingress_diverges() + || (scope == ReconcileScope::WithSteady && self.is_steady_provisioned()) + } + pub fn provider_id(&self) -> Option<&EnvironmentProviderId> { self.source.provider_id() } @@ -615,10 +757,7 @@ impl EnvironmentRecord { if let Some(policy) = &self.idle_policy { policy.validate()?; } - if self.public_ingress_enabled != self.public_endpoint.is_some() { - return invalid("public ingress requires exactly one public endpoint when enabled"); - } - if self.public_ingress_enabled + if (self.public_ingress_enabled || self.public_endpoint.is_some()) && !matches!(self.source, EnvironmentSource::Provisioned { .. }) { return invalid("only provisioned environments can have provider-managed ingress"); @@ -629,8 +768,8 @@ impl EnvironmentRecord { if let Some(seen) = self.last_seen_at_ms { validate_nonnegative_i64(seen, "last_seen_at_ms")?; } - if let Some(target_id) = &self.incarnation.provider_target_id { - validate_provider_target_id(target_id)?; + if let Some(error) = &self.lifecycle_error { + error.validate()?; } match &self.source { EnvironmentSource::Provisioned { .. } => { @@ -644,18 +783,11 @@ impl EnvironmentRecord { "provisioned incarnation requires exactly one template or adoption source", ); } - if let Some(source) = self.incarnation.adoption_source_target.as_deref() - && (source.is_empty() - || source.len() > 255 - || source.chars().any(char::is_control)) - { - return invalid( - "adoption_source_target must be non-empty, at most 255 bytes, and contain no control characters", - ); + if let Some(source) = self.incarnation.adoption_source_target.as_deref() { + validate_adoption_source(source)?; } } - EnvironmentSource::External { connection } => { - connection.validate()?; + EnvironmentSource::External { .. } => { self.validate_no_provider_linkage("external")?; } EnvironmentSource::Registered { @@ -663,10 +795,8 @@ impl EnvironmentRecord { daemon_public_key, .. } => { - validate_daemon_public_key(daemon_public_key)?; - if daemon_id - != &EnvironmentDaemonId::from_public_key(&decode_hex(daemon_public_key)?) - { + let public_key = parse_daemon_public_key(daemon_public_key)?; + if daemon_id != &EnvironmentDaemonId::from_public_key(public_key.as_bytes()) { return invalid("registered daemon id does not match its public key"); } if self.request_id != EnvironmentProvisionRequestId::for_daemon(daemon_id) { @@ -931,10 +1061,8 @@ pub struct CreateRegisteredEnvironment { impl CreateRegisteredEnvironment { pub fn daemon_id(&self) -> Result { - validate_daemon_public_key(&self.daemon_public_key)?; - Ok(EnvironmentDaemonId::from_public_key(&decode_hex( - &self.daemon_public_key, - )?)) + let public_key = parse_daemon_public_key(&self.daemon_public_key)?; + Ok(EnvironmentDaemonId::from_public_key(public_key.as_bytes())) } } @@ -963,32 +1091,30 @@ pub struct ObserveRegisteredEnvironment { pub metadata: BTreeMap, } -/// Metadata keys the gateway writes about the daemon serving a registered -/// environment. They carry the reserved prefix, so a daemon cannot set them. +/// Metadata keys Lightspeed writes about the daemon serving an environment. +/// They carry the reserved prefix, so a caller cannot set them. pub const ENVD_VERSION_METADATA_KEY: &str = "lightspeed.envd.version"; pub const ENVD_GIT_SHA_METADATA_KEY: &str = "lightspeed.envd.gitSha"; pub const ENVD_PROTOCOL_VERSION_METADATA_KEY: &str = "lightspeed.envd.protocolVersion"; -/// What the gateway learned about a daemon's build when it registered or -/// reconnected. Recorded on the environment row at every admission so an -/// operator sees which build serves the environment. Never consulted for -/// admission: the protocol version alone decides that, and this only -/// records which number the admitted daemon spoke. +/// What Lightspeed learned about the daemon serving an environment: from +/// `registration/register` for a registered daemon, from the data-plane +/// `initialize` at the Ready transition for a provisioned one. Recorded on +/// the environment row so an operator sees which build serves it, and which +/// protocol version a mismatched daemon spoke. Never consulted for +/// admission: the protocol version alone decides that. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct RegisteredDaemonBuild { +pub struct DaemonBuild { pub version: Option, pub git_sha: Option, pub protocol_version: u32, } -impl RegisteredDaemonBuild { - pub fn from_registration( - implementation: &environment_protocol::shared::ImplementationInfo, - protocol_version: u32, - ) -> Self { +impl DaemonBuild { + pub fn from_implementation(implementation: &ImplementationInfo, protocol_version: u32) -> Self { Self { - version: implementation.version.clone(), - git_sha: implementation.git_sha.clone(), + version: implementation.version().map(str::to_owned), + git_sha: implementation.git_sha().map(str::to_owned), protocol_version, } } @@ -1041,12 +1167,185 @@ pub struct CreateExternalEnvironment { pub request_id: EnvironmentProvisionRequestId, pub environment_id: EnvironmentId, pub incarnation_id: EnvironmentIncarnationId, - pub connection: EnvironmentConnectionSpec, + pub endpoint: EnvironmentEndpoint, pub display_name: Option, pub metadata: BTreeMap, pub created_at_ms: i64, } +impl CreateEnvironment { + /// The validated record this create inserts for a binding of + /// `provider_id`. + pub fn record( + &self, + provider_id: EnvironmentProviderId, + ) -> Result { + let record = new_record( + self.environment_id.clone(), + self.request_id.clone(), + EnvironmentSource::Provisioned { + provider_id, + binding_id: self.binding_id.clone(), + }, + self.display_name.clone(), + EnvironmentStatus::Provisioning, + EnvironmentIncarnationRecord { + provision_request_id: Some(self.request_id.clone()), + template_id: Some(self.template_id.clone()), + ..new_incarnation(self.incarnation_id.clone(), self.created_at_ms) + }, + self.metadata.clone(), + self.created_at_ms, + ); + let record = EnvironmentRecord { + idle_policy: self.idle_policy.clone(), + ..record + }; + record.validate()?; + Ok(record) + } +} + +impl AdoptEnvironment { + /// The validated record this adoption inserts for a binding of + /// `provider_id`. + pub fn record( + &self, + provider_id: EnvironmentProviderId, + ) -> Result { + validate_adoption_source(&self.source_target)?; + let record = new_record( + self.environment_id.clone(), + self.request_id.clone(), + EnvironmentSource::Provisioned { + provider_id, + binding_id: self.binding_id.clone(), + }, + self.display_name.clone(), + EnvironmentStatus::Provisioning, + EnvironmentIncarnationRecord { + provision_request_id: Some(self.request_id.clone()), + adoption_source_target: Some(self.source_target.clone()), + ..new_incarnation(self.incarnation_id.clone(), self.created_at_ms) + }, + self.metadata.clone(), + self.created_at_ms, + ); + record.validate()?; + Ok(record) + } +} + +impl CreateExternalEnvironment { + /// The validated record this create inserts. + pub fn record(&self) -> Result { + let record = new_record( + self.environment_id.clone(), + self.request_id.clone(), + EnvironmentSource::External { + endpoint: self.endpoint.clone(), + }, + self.display_name.clone(), + EnvironmentStatus::Ready, + new_incarnation(self.incarnation_id.clone(), self.created_at_ms), + self.metadata.clone(), + self.created_at_ms, + ); + record.validate()?; + Ok(record) + } +} + +impl CreateRegisteredEnvironment { + /// The validated record this admission inserts under `key`. + pub fn record( + &self, + key: &EnvironmentRegistrationKeyRecord, + ) -> Result { + let daemon_id = self.daemon_id()?; + let record = new_record( + self.environment_id.clone(), + EnvironmentProvisionRequestId::for_daemon(&daemon_id), + EnvironmentSource::Registered { + registration_key_id: key.registration_key_id.clone(), + daemon_id, + daemon_public_key: self.daemon_public_key.clone(), + identity_mode: key.identity_mode, + }, + self.display_name.clone(), + EnvironmentStatus::Ready, + new_incarnation(self.incarnation_id.clone(), self.created_at_ms), + self.metadata.clone(), + self.created_at_ms, + ); + let record = EnvironmentRecord { + last_seen_at_ms: Some(self.created_at_ms), + ..record + }; + record.validate()?; + Ok(record) + } +} + +fn new_incarnation( + incarnation_id: EnvironmentIncarnationId, + created_at_ms: i64, +) -> EnvironmentIncarnationRecord { + EnvironmentIncarnationRecord { + incarnation_id, + provision_request_id: None, + provider_target_id: None, + template_id: None, + adoption_source_target: None, + power_states: Vec::new(), + created_at_ms, + updated_at_ms: created_at_ms, + } +} + +#[allow(clippy::too_many_arguments)] +fn new_record( + environment_id: EnvironmentId, + request_id: EnvironmentProvisionRequestId, + source: EnvironmentSource, + display_name: Option, + status: EnvironmentStatus, + incarnation: EnvironmentIncarnationRecord, + metadata: BTreeMap, + created_at_ms: i64, +) -> EnvironmentRecord { + EnvironmentRecord { + environment_id, + request_id, + source, + display_name, + status, + desired_power: PowerState::Running, + idle_policy: None, + incarnation, + public_ingress_enabled: false, + public_endpoint: None, + metadata, + last_seen_at_ms: None, + lifecycle_error: None, + created_at_ms, + updated_at_ms: created_at_ms, + } +} + +/// Which environments a lifecycle reconcile pass visits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReconcileScope { + /// Environments with convergence pending: provisioning, booting, + /// closing or unknown, a power state that diverges from the intent, or + /// a ready environment whose public endpoint does not match its + /// ingress intent. + Converging, + /// Those plus every steady provisioned environment with a recorded + /// target, observed on a slower cadence. + WithSteady, +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ListEnvironments { pub provider_id: Option, @@ -1059,16 +1358,66 @@ pub struct ListEnvironments { pub metadata: BTreeMap, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// A provider observation of a provisioned environment's target. The target +/// id and power states are recorded whatever the environment's status, so a +/// close that raced the first create can still address the target. The +/// status, the daemon build and the clearing of `lifecycle_error` apply only +/// while the environment is not closing or closed: a close is never +/// overwritten by an observation that was in flight when it was accepted. +#[derive(Clone, Debug, PartialEq, Eq)] pub struct ObserveProvisionedEnvironment { pub environment_id: EnvironmentId, - pub provider_target_id: ProviderTargetId, + /// Recorded when present; absent while an asynchronous provider has not + /// assigned one yet, which keeps the recorded id. A different id than + /// the recorded one is refused. + pub provider_target_id: Option, pub status: EnvironmentStatus, /// Provider-reported supported power states for this target. pub power_states: Vec, + /// The daemon found by the data-plane `initialize` of a Ready + /// transition. + pub daemon_build: Option, pub observed_at_ms: i64, } +impl ObserveProvisionedEnvironment { + /// Apply this observation to `record`. Stale observations (older than + /// the incarnation's last update) leave it unchanged. + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result<(), EnvironmentRegistryError> { + validate_nonnegative_i64(self.observed_at_ms, "observed_at_ms")?; + if !matches!(record.source, EnvironmentSource::Provisioned { .. }) { + return invalid("provider observations apply only to provisioned environments"); + } + if self.observed_at_ms < record.incarnation.updated_at_ms { + return Ok(()); + } + if let (Some(recorded), Some(observed)) = ( + record.incarnation.provider_target_id.as_ref(), + self.provider_target_id.as_ref(), + ) && recorded != observed + { + return invalid("provider target conflicts with current incarnation"); + } + if let Some(target_id) = &self.provider_target_id { + record.incarnation.provider_target_id = Some(target_id.clone()); + } + record.incarnation.power_states = self.power_states.clone(); + record.incarnation.updated_at_ms = self.observed_at_ms; + if record.status.is_closing_or_closed() { + return Ok(()); + } + record.status = self.status; + if let Some(build) = &self.daemon_build { + record.metadata.extend(build.metadata()); + } + if self.status.power_state().is_some() { + record.lifecycle_error = None; + } + record.updated_at_ms = record.updated_at_ms.max(self.observed_at_ms); + Ok(()) + } +} + /// Set the Lightspeed-owned power intent. Rejected for external, closing, /// closed, and failed environments; the caller validates provider support. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1086,13 +1435,66 @@ pub struct SetEnvironmentIdlePolicy { pub updated_at_ms: i64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Move an environment to `failed` with `message` as its lifecycle error. +/// A closing or closed environment is left as it is. +#[derive(Clone, Debug, PartialEq, Eq)] pub struct FailEnvironmentLifecycle { + pub environment_id: EnvironmentId, + pub message: String, + /// The daemon whose build made the environment unusable, such as one + /// that speaks another protocol version. + pub daemon_build: Option, + pub observed_at_ms: i64, +} + +impl FailEnvironmentLifecycle { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result<(), EnvironmentRegistryError> { + validate_nonnegative_i64(self.observed_at_ms, "observed_at_ms")?; + if record.status.is_closing_or_closed() { + return Ok(()); + } + record.status = EnvironmentStatus::Failed; + record.lifecycle_error = Some(EnvironmentLifecycleError::record( + record.lifecycle_error.as_ref(), + &self.message, + self.observed_at_ms, + )); + if let Some(build) = &self.daemon_build { + record.metadata.extend(build.metadata()); + } + record.incarnation.updated_at_ms = + record.incarnation.updated_at_ms.max(self.observed_at_ms); + record.updated_at_ms = record.updated_at_ms.max(self.observed_at_ms); + Ok(()) + } +} + +/// Record a lifecycle error the reconciler will retry past, without +/// changing the environment's status. A closed environment is left as it +/// is. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordEnvironmentLifecycleError { pub environment_id: EnvironmentId, pub message: String, pub observed_at_ms: i64, } +impl RecordEnvironmentLifecycleError { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result<(), EnvironmentRegistryError> { + validate_nonnegative_i64(self.observed_at_ms, "observed_at_ms")?; + if record.status == EnvironmentStatus::Closed { + return Ok(()); + } + record.lifecycle_error = Some(EnvironmentLifecycleError::record( + record.lifecycle_error.as_ref(), + &self.message, + self.observed_at_ms, + )); + record.updated_at_ms = record.updated_at_ms.max(self.observed_at_ms); + Ok(()) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct BeginCloseEnvironment { pub environment_id: EnvironmentId, @@ -1105,14 +1507,42 @@ pub struct FinishCloseEnvironment { pub observed_at_ms: i64, } +/// Record the public ingress intent of a provisioned environment; the +/// lifecycle reconciler converges the provider toward it. Enabling is +/// refused for a closing or closed environment. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct SetEnvironmentIngress { pub environment_id: EnvironmentId, pub enabled: bool, - pub public_endpoint: Option, pub updated_at_ms: i64, } +/// Record the public endpoint a provider observation reports realized, +/// `None` when the target has no public ingress. A closing or closed +/// environment is left as it is. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecordEnvironmentIngress { + pub environment_id: EnvironmentId, + pub public_endpoint: Option, + pub observed_at_ms: i64, +} + +impl RecordEnvironmentIngress { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result<(), EnvironmentRegistryError> { + validate_nonnegative_i64(self.observed_at_ms, "observed_at_ms")?; + validate_nonempty_optional("public_endpoint", self.public_endpoint.as_deref())?; + if !matches!(record.source, EnvironmentSource::Provisioned { .. }) { + return invalid("provider-managed ingress requires a provisioned environment"); + } + if record.status.is_closing_or_closed() { + return Ok(()); + } + record.public_endpoint = self.public_endpoint.clone(); + record.updated_at_ms = record.updated_at_ms.max(self.observed_at_ms); + Ok(()) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct EnvironmentCredentialRecord { pub environment_id: EnvironmentId, @@ -1172,6 +1602,18 @@ pub trait EnvironmentProviderStore: Send + Sync { &self, provider_id: &EnvironmentProviderId, ) -> Result; + /// The provider's bearer credential, unsealed. + async fn read_provider_credential( + &self, + provider_id: &EnvironmentProviderId, + ) -> Result; + /// Record what the controller advertised at a successful `initialize`. + /// Returns the record unchanged when the provider has moved to another + /// endpoint since. + async fn record_provider_controller( + &self, + request: RecordProviderController, + ) -> Result; async fn list_providers( &self, request: ListEnvironmentProviders, @@ -1246,12 +1688,17 @@ pub trait EnvironmentStore: Send + Sync { &self, request_id: &EnvironmentProvisionRequestId, ) -> Result; + /// Rows that no longer decode are left out and logged by the store, so + /// one bad row never hides the others; every list method does the same. async fn list_environments( &self, request: ListEnvironments, ) -> Result, EnvironmentRegistryError>; + /// The environments a reconcile pass of `scope` visits; see + /// [`EnvironmentRecord::needs_reconcile`]. async fn list_environments_needing_reconcile( &self, + scope: ReconcileScope, ) -> Result, EnvironmentRegistryError>; async fn observe_provisioned_environment( &self, @@ -1261,6 +1708,10 @@ pub trait EnvironmentStore: Send + Sync { &self, request: FailEnvironmentLifecycle, ) -> Result; + async fn record_environment_lifecycle_error( + &self, + request: RecordEnvironmentLifecycleError, + ) -> Result; async fn begin_close_environment( &self, request: BeginCloseEnvironment, @@ -1273,6 +1724,10 @@ pub trait EnvironmentStore: Send + Sync { &self, request: SetEnvironmentIngress, ) -> Result; + async fn record_environment_ingress( + &self, + request: RecordEnvironmentIngress, + ) -> Result; async fn set_environment_power( &self, request: SetEnvironmentPower, @@ -1309,7 +1764,7 @@ pub fn template_record( value: &EnvironmentTemplate, ) -> Result { let template_id = - EnvironmentTemplateId::try_new(value.template_id.clone()).map_err(|error| { + EnvironmentTemplateId::try_new(value.template_id.as_str()).map_err(|error| { EnvironmentRegistryError::InvalidInput { message: format!("invalid template id: {error}"), } @@ -1317,6 +1772,8 @@ pub fn template_record( Ok(template_id) } +pub mod endpoint; +pub use endpoint::{EndpointError, EndpointScheme, EnvironmentEndpoint}; mod memory; pub use memory::{InMemoryEnvironmentRegistryStore, apply_registered_observation}; @@ -1338,20 +1795,11 @@ fn validate_timestamps( Ok(()) } -fn validate_endpoint(name: &'static str, value: &str) -> Result<(), EnvironmentRegistryError> { - validate_nonempty_string(name, value)?; - if value.chars().any(char::is_whitespace) { - return invalid(format!("{name} must not contain whitespace")); - } - Ok(()) -} - -fn validate_provider_target_id(value: &ProviderTargetId) -> Result<(), EnvironmentRegistryError> { - validate_general_string_id("target_id", value.as_str()).map_err(|error| { - EnvironmentRegistryError::InvalidInput { - message: error.to_string(), - } - }) +/// An adoption source is a provider-native reference under the protocol's +/// opaque rule: 1 to 255 bytes without control characters. +pub fn validate_adoption_source(value: &str) -> Result<(), EnvironmentRegistryError> { + validate_opaque_id("adoption_source_target", value) + .map_err(|error| invalid_error(error.to_string())) } fn validate_registration_display_name(value: &str) -> Result<(), EnvironmentRegistryError> { @@ -1369,27 +1817,14 @@ fn validate_registration_display_name(value: &str) -> Result<(), EnvironmentRegi /// A daemon public key is the raw 32-byte Ed25519 key as lowercase hex. pub fn validate_daemon_public_key(value: &str) -> Result<(), EnvironmentRegistryError> { - if value.len() != 64 - || !value - .bytes() - .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) - { - return invalid("daemon public key must be 64 lowercase hex characters"); - } - Ok(()) + parse_daemon_public_key(value).map(|_| ()) } -fn decode_hex(value: &str) -> Result, EnvironmentRegistryError> { - if !value.len().is_multiple_of(2) { - return invalid("hex value has odd length"); - } - (0..value.len()) - .step_by(2) - .map(|index| { - u8::from_str_radix(&value[index..index + 2], 16) - .map_err(|_| invalid_error("hex value contains a non-hex character")) - }) - .collect() +fn parse_daemon_public_key( + value: &str, +) -> Result { + environment_protocol::registration::DaemonPublicKey::parse(value) + .map_err(|_| invalid_error("daemon public key must be 64 lowercase hex characters")) } fn invalid_error(message: impl Into) -> EnvironmentRegistryError { @@ -1398,29 +1833,12 @@ fn invalid_error(message: impl Into) -> EnvironmentRegistryError { } } -/// Stored records obey the registration handshake's bounds (entry count, -/// key and value bytes, no control characters). The reserved-prefix check -/// applies to caller input at the API boundary, not here: Lightspeed itself -/// annotates records under that prefix, and those entries sit on top of a -/// caller's full allowance rather than eating into it, so the entry count -/// applies to the caller's keys alone. +/// Stored records obey the descriptive metadata bounds. The reserved-prefix +/// check applies to caller input at the API boundary, not here: Lightspeed +/// itself annotates records under that prefix, and those entries sit on top +/// of a caller's full allowance rather than eating into it. fn validate_metadata(value: &BTreeMap) -> Result<(), EnvironmentRegistryError> { - use environment_protocol::registration::{ - MAX_METADATA_ENTRIES, RESERVED_METADATA_PREFIX, validate_metadata_entry, - }; - let caller_entries = value - .keys() - .filter(|key| !key.starts_with(RESERVED_METADATA_PREFIX)) - .count(); - if caller_entries > MAX_METADATA_ENTRIES { - return Err(invalid_error(format!( - "metadata has more than {MAX_METADATA_ENTRIES} caller entries" - ))); - } - for (key, entry) in value { - validate_metadata_entry(key, entry).map_err(invalid_error)?; - } - Ok(()) + validate_stored_metadata(value).map_err(|error| invalid_error(error.to_string())) } fn validate_nonempty_optional( diff --git a/crates/environments/src/memory.rs b/crates/environments/src/memory.rs index fe9693f2e..4b9c28c5b 100644 --- a/crates/environments/src/memory.rs +++ b/crates/environments/src/memory.rs @@ -16,6 +16,7 @@ struct StoredRegistrationKey { #[derive(Default)] struct RegistryState { providers: BTreeMap, + provider_credentials: BTreeMap, bindings: BTreeMap, environments: BTreeMap, requests: BTreeMap, @@ -77,10 +78,14 @@ impl EnvironmentProviderStore for InMemoryEnvironmentRegistryStore { &self, request: PutEnvironmentProvider, ) -> Result { - let mut record = request.into_record()?; let mut state = self.write_state()?; - if let Some(existing) = state.providers.get(&record.provider_id) { - record.created_at_ms = existing.created_at_ms; + let credential = request.credential.clone(); + let existing = state.providers.get(&request.provider_id).cloned(); + let record = request.into_record(existing.as_ref())?; + if let Some(credential) = credential { + state + .provider_credentials + .insert(record.provider_id.clone(), credential); } state .providers @@ -88,6 +93,33 @@ impl EnvironmentProviderStore for InMemoryEnvironmentRegistryStore { Ok(record) } + async fn read_provider_credential( + &self, + provider_id: &EnvironmentProviderId, + ) -> Result { + self.read_state()? + .provider_credentials + .get(provider_id) + .cloned() + .ok_or_else(|| not_found("environment_provider", provider_id)) + } + + async fn record_provider_controller( + &self, + request: RecordProviderController, + ) -> Result { + let mut state = self.write_state()?; + let record = state + .providers + .get_mut(&request.provider_id) + .ok_or_else(|| not_found("environment_provider", &request.provider_id))?; + if record.controller_endpoint == request.controller_endpoint { + record.controller_features = Some(request.features); + record.data_route_base = Some(request.data_route_base); + } + Ok(record.clone()) + } + async fn read_provider( &self, provider_id: &EnvironmentProviderId, @@ -118,10 +150,12 @@ impl EnvironmentProviderStore for InMemoryEnvironmentRegistryStore { { return invalid("environment provider is referenced by a universe binding"); } - state + let record = state .providers .remove(provider_id) - .ok_or_else(|| not_found("environment_provider", provider_id)) + .ok_or_else(|| not_found("environment_provider", provider_id))?; + state.provider_credentials.remove(provider_id); + Ok(record) } } @@ -250,35 +284,7 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { id: request.environment_id.to_string(), }); } - let record = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request.request_id.clone(), - source: EnvironmentSource::Provisioned { - provider_id: binding.provider_id, - binding_id: request.binding_id, - }, - display_name: request.display_name, - status: EnvironmentStatus::Provisioning, - desired_power: PowerState::Running, - idle_policy: request.idle_policy, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id, - provision_request_id: Some(request.request_id.clone()), - provider_target_id: None, - template_id: Some(request.template_id), - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata, - last_seen_at_ms: None, - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - record.validate()?; + let record = request.record(binding.provider_id)?; state .requests .insert(request.request_id, request.environment_id.clone()); @@ -293,14 +299,6 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { request: AdoptEnvironment, ) -> Result { validate_nonnegative_i64(request.created_at_ms, "created_at_ms")?; - if request.source_target.is_empty() - || request.source_target.len() > 255 - || request.source_target.chars().any(char::is_control) - { - return invalid( - "adoption source must be non-empty, at most 255 bytes, and contain no control characters", - ); - } let mut state = self.write_state()?; if let Some(environment_id) = state.requests.get(&request.request_id) { return state @@ -323,35 +321,7 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { id: request.environment_id.to_string(), }); } - let record = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request.request_id.clone(), - source: EnvironmentSource::Provisioned { - provider_id: binding.provider_id, - binding_id: request.binding_id, - }, - display_name: request.display_name, - status: EnvironmentStatus::Provisioning, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id, - provision_request_id: Some(request.request_id.clone()), - provider_target_id: None, - template_id: None, - adoption_source_target: Some(request.source_target), - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata, - last_seen_at_ms: None, - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - record.validate()?; + let record = request.record(binding.provider_id)?; state .requests .insert(request.request_id, request.environment_id.clone()); @@ -366,7 +336,6 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { request: CreateExternalEnvironment, ) -> Result { validate_nonnegative_i64(request.created_at_ms, "created_at_ms")?; - request.connection.validate()?; let mut state = self.write_state()?; if let Some(environment_id) = state.requests.get(&request.request_id) { return state @@ -381,34 +350,7 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { id: request.environment_id.to_string(), }); } - let record = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request.request_id.clone(), - source: EnvironmentSource::External { - connection: request.connection, - }, - display_name: request.display_name, - status: EnvironmentStatus::Ready, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id, - provision_request_id: None, - provider_target_id: None, - template_id: None, - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata, - last_seen_at_ms: None, - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - record.validate()?; + let record = request.record()?; state .requests .insert(request.request_id, request.environment_id.clone()); @@ -472,37 +414,7 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { id: request.environment_id.to_string(), }); } - let record = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request_id.clone(), - source: EnvironmentSource::Registered { - registration_key_id: key.registration_key_id, - daemon_id, - daemon_public_key: request.daemon_public_key, - identity_mode: key.identity_mode, - }, - display_name: request.display_name, - status: EnvironmentStatus::Ready, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id, - provision_request_id: None, - provider_target_id: None, - template_id: None, - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata, - last_seen_at_ms: Some(request.created_at_ms), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - record.validate()?; + let record = request.record(&key)?; state .requests .insert(request_id, request.environment_id.clone()); @@ -533,18 +445,14 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { ) -> Result { validate_nonnegative_i64(request.observed_at_ms, "observed_at_ms")?; let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - apply_registered_observation( - record, - request.observation, - request.observed_at_ms, - request.metadata, - )?; - record.validate()?; - Ok(record.clone()) + update_environment(&mut state, &request.environment_id, |record| { + apply_registered_observation( + record, + request.observation, + request.observed_at_ms, + request.metadata, + ) + }) } async fn list_open_registered_environments( @@ -628,20 +536,13 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { async fn list_environments_needing_reconcile( &self, + scope: ReconcileScope, ) -> Result, EnvironmentRegistryError> { Ok(self .read_state()? .environments .values() - .filter(|record| { - matches!( - record.status, - EnvironmentStatus::Provisioning - | EnvironmentStatus::Booting - | EnvironmentStatus::Closing - | EnvironmentStatus::Unknown - ) || record.power_diverges() - }) + .filter(|record| record.needs_reconcile(scope)) .cloned() .collect()) } @@ -665,28 +566,9 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { request: ObserveProvisionedEnvironment, ) -> Result { let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - if request.observed_at_ms < record.incarnation.updated_at_ms { - return Ok(record.clone()); - } - if record - .incarnation - .provider_target_id - .as_ref() - .is_some_and(|existing| existing != &request.provider_target_id) - { - return invalid("provider target conflicts with current incarnation"); - } - record.incarnation.provider_target_id = Some(request.provider_target_id); - record.incarnation.power_states = request.power_states; - record.incarnation.updated_at_ms = request.observed_at_ms; - record.updated_at_ms = request.observed_at_ms; - record.status = request.status; - record.validate()?; - Ok(record.clone()) + update_environment(&mut state, &request.environment_id, |record| { + request.apply(record) + }) } async fn fail_environment_lifecycle( @@ -694,17 +576,19 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { request: FailEnvironmentLifecycle, ) -> Result { let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - record.status = EnvironmentStatus::Failed; - record - .metadata - .insert("lifecycleError".to_owned(), request.message); - record.incarnation.updated_at_ms = request.observed_at_ms; - record.updated_at_ms = request.observed_at_ms; - Ok(record.clone()) + update_environment(&mut state, &request.environment_id, |record| { + request.apply(record) + }) + } + + async fn record_environment_lifecycle_error( + &self, + request: RecordEnvironmentLifecycleError, + ) -> Result { + let mut state = self.write_state()?; + update_environment(&mut state, &request.environment_id, |record| { + request.apply(record) + }) } async fn begin_close_environment( @@ -712,15 +596,13 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { request: BeginCloseEnvironment, ) -> Result { let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - if record.status != EnvironmentStatus::Closed { - record.status = EnvironmentStatus::Closing; - record.updated_at_ms = request.updated_at_ms; - } - Ok(record.clone()) + update_environment(&mut state, &request.environment_id, |record| { + if record.status != EnvironmentStatus::Closed { + record.status = EnvironmentStatus::Closing; + record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); + } + Ok(()) + }) } async fn finish_close_environment( @@ -728,16 +610,15 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { request: FinishCloseEnvironment, ) -> Result { let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - record.status = EnvironmentStatus::Closed; - record.public_ingress_enabled = false; - record.public_endpoint = None; - record.incarnation.updated_at_ms = request.observed_at_ms; - record.updated_at_ms = request.observed_at_ms; - Ok(record.clone()) + update_environment(&mut state, &request.environment_id, |record| { + record.status = EnvironmentStatus::Closed; + record.public_ingress_enabled = false; + record.public_endpoint = None; + record.incarnation.updated_at_ms = + record.incarnation.updated_at_ms.max(request.observed_at_ms); + record.updated_at_ms = record.updated_at_ms.max(request.observed_at_ms); + Ok(()) + }) } async fn set_environment_ingress( @@ -746,30 +627,27 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { ) -> Result { validate_nonnegative_i64(request.updated_at_ms, "updated_at_ms")?; let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - if !matches!(record.source, EnvironmentSource::Provisioned { .. }) { - return invalid("provider-managed ingress requires a provisioned environment"); - } - if matches!( - record.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) && request.enabled - { - return invalid("cannot enable ingress for a closing environment"); - } - if request.enabled != request.public_endpoint.is_some() { - return invalid( - "enabled ingress requires a public endpoint and disabled ingress forbids one", - ); - } - record.public_ingress_enabled = request.enabled; - record.public_endpoint = request.public_endpoint; - record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); - record.validate()?; - Ok(record.clone()) + update_environment(&mut state, &request.environment_id, |record| { + if !matches!(record.source, EnvironmentSource::Provisioned { .. }) { + return invalid("provider-managed ingress requires a provisioned environment"); + } + if record.status.is_closing_or_closed() && request.enabled { + return invalid("cannot enable ingress for a closing environment"); + } + record.public_ingress_enabled = request.enabled; + record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); + Ok(()) + }) + } + + async fn record_environment_ingress( + &self, + request: RecordEnvironmentIngress, + ) -> Result { + let mut state = self.write_state()?; + update_environment(&mut state, &request.environment_id, |record| { + request.apply(record) + }) } async fn set_environment_power( @@ -778,15 +656,12 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { ) -> Result { validate_nonnegative_i64(request.updated_at_ms, "updated_at_ms")?; let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - check_power_mutable(record)?; - record.desired_power = request.desired_power; - record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); - record.validate()?; - Ok(record.clone()) + update_environment(&mut state, &request.environment_id, |record| { + check_power_mutable(record)?; + record.desired_power = request.desired_power; + record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); + Ok(()) + }) } async fn set_environment_idle_policy( @@ -798,18 +673,34 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { policy.validate()?; } let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - check_power_mutable(record)?; - record.idle_policy = request.idle_policy; - record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); - record.validate()?; - Ok(record.clone()) + update_environment(&mut state, &request.environment_id, |record| { + check_power_mutable(record)?; + record.idle_policy = request.idle_policy; + record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); + Ok(()) + }) } } +/// Change one environment through `change` on a copy, validate the result, +/// and store it only when it is valid: a refused change leaves the stored +/// record untouched. +fn update_environment( + state: &mut RegistryState, + environment_id: &EnvironmentId, + change: impl FnOnce(&mut EnvironmentRecord) -> Result<(), EnvironmentRegistryError>, +) -> Result { + let stored = state + .environments + .get_mut(environment_id) + .ok_or_else(|| not_found("environment", environment_id))?; + let mut candidate = stored.clone(); + change(&mut candidate)?; + candidate.validate()?; + *stored = candidate.clone(); + Ok(candidate) +} + /// Apply a gateway connection observation to a registered environment. /// Closing and closed environments keep their state: a late heartbeat must /// not resurrect them. @@ -822,10 +713,7 @@ pub fn apply_registered_observation( if !record.is_registered() { return invalid("connection observations apply only to registered environments"); } - if matches!( - record.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) { + if record.status.is_closing_or_closed() { return Ok(()); } record.metadata.extend(metadata); diff --git a/crates/environments/src/tests.rs b/crates/environments/src/tests.rs index 605df8215..d89568621 100644 --- a/crates/environments/src/tests.rs +++ b/crates/environments/src/tests.rs @@ -1,23 +1,45 @@ use std::collections::BTreeMap; -use environment_protocol::shared::{EnvironmentTransport, ProviderTargetId}; +use environment_protocol::{ids::ProviderTargetId, metadata::MAX_METADATA_ENTRIES}; use uuid::Uuid; use super::*; +fn endpoint(value: &str) -> EnvironmentEndpoint { + EnvironmentEndpoint::parse(value).expect("endpoint") +} + +fn target(value: &str) -> ProviderTargetId { + ProviderTargetId::new(value).expect("target id") +} + fn provider() -> PutEnvironmentProvider { PutEnvironmentProvider { provider_id: EnvironmentProviderId::new("incus-local"), display_name: Some("Local Incus".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "ws://127.0.0.1:19090/control", - EnvironmentTransport::WebSocket, - ), + controller_endpoint: endpoint("ws://127.0.0.1:19090/control"), metadata: BTreeMap::new(), + credential: Some(SecretValue::new("provider-secret")), updated_at_ms: 1_000, } } +fn observe( + environment_id: &EnvironmentId, + target_id: Option<&str>, + status: EnvironmentStatus, + at: i64, +) -> ObserveProvisionedEnvironment { + ObserveProvisionedEnvironment { + environment_id: environment_id.clone(), + provider_target_id: target_id.map(target), + status, + power_states: vec![PowerState::Running, PowerState::Paused], + daemon_build: None, + observed_at_ms: at, + } +} + fn binding(universe_id: Uuid, expected_revision: Option) -> PutEnvironmentProviderBinding { PutEnvironmentProviderBinding { universe_id, @@ -200,6 +222,7 @@ async fn lightspeed_does_not_enforce_provider_quota() { .fail_environment_lifecycle(FailEnvironmentLifecycle { environment_id: first.environment_id.clone(), message: "provider failed".to_owned(), + daemon_build: None, observed_at_ms: 3_000, }) .await @@ -217,15 +240,14 @@ async fn provider_observation_populates_only_the_current_incarnation() { .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) .await .expect("create"); - let target_id = ProviderTargetId::new("target-1"); + let target_id = target("target-1"); let observed = store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: target_id.clone(), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 3_000, - }) + .observe_provisioned_environment(observe( + &environment.environment_id, + Some("target-1"), + EnvironmentStatus::Ready, + 3_000, + )) .await .expect("observe"); assert_eq!(observed.status, EnvironmentStatus::Ready); @@ -241,37 +263,76 @@ async fn provider_observation_populates_only_the_current_incarnation() { } #[tokio::test(flavor = "current_thread")] -async fn public_ingress_is_a_realized_environment_facet() { +async fn public_ingress_intent_and_realized_endpoint_are_recorded_apart() { let (_, store) = store().await; let environment = store .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) .await .expect("create"); + store + .observe_provisioned_environment(observe( + &environment.environment_id, + Some("target-1"), + EnvironmentStatus::Ready, + 2_500, + )) + .await + .expect("observe"); let enabled = store .set_environment_ingress(SetEnvironmentIngress { environment_id: environment.environment_id.clone(), enabled: true, - public_endpoint: Some("https://opaque.env.example".to_owned()), updated_at_ms: 3_000, }) .await .expect("enable ingress"); assert!(enabled.public_ingress_enabled); + assert_eq!(enabled.public_endpoint, None); + // The intent is pending until the provider reports an endpoint. + assert!(enabled.ingress_diverges()); + assert_eq!( + store + .list_environments_needing_reconcile(ReconcileScope::Converging) + .await + .unwrap() + .len(), + 1 + ); + let realized = store + .record_environment_ingress(RecordEnvironmentIngress { + environment_id: environment.environment_id.clone(), + public_endpoint: Some("https://opaque.env.example".to_owned()), + observed_at_ms: 3_500, + }) + .await + .expect("record realized ingress"); assert_eq!( - enabled.public_endpoint.as_deref(), + realized.public_endpoint.as_deref(), Some("https://opaque.env.example") ); + assert!(!realized.ingress_diverges()); let disabled = store .set_environment_ingress(SetEnvironmentIngress { - environment_id: environment.environment_id, + environment_id: environment.environment_id.clone(), enabled: false, - public_endpoint: None, updated_at_ms: 4_000, }) .await .expect("disable ingress"); assert!(!disabled.public_ingress_enabled); - assert_eq!(disabled.public_endpoint, None); + // The endpoint stays recorded until the provider removed it. + assert!(disabled.public_endpoint.is_some()); + assert!(disabled.ingress_diverges()); + let removed = store + .record_environment_ingress(RecordEnvironmentIngress { + environment_id: environment.environment_id, + public_endpoint: None, + observed_at_ms: 4_500, + }) + .await + .expect("record removed ingress"); + assert_eq!(removed.public_endpoint, None); + assert!(!removed.ingress_diverges()); } #[tokio::test(flavor = "current_thread")] @@ -282,10 +343,7 @@ async fn external_environment_cannot_enable_provider_managed_ingress() { request_id: EnvironmentProvisionRequestId::new("external-request"), environment_id: EnvironmentId::new("external-environment"), incarnation_id: EnvironmentIncarnationId::new("external-incarnation"), - connection: EnvironmentConnectionSpec::new( - "ws://envd.example", - EnvironmentTransport::WebSocket, - ), + endpoint: endpoint("ws://envd.example"), display_name: None, metadata: BTreeMap::new(), created_at_ms: 1_000, @@ -297,7 +355,6 @@ async fn external_environment_cannot_enable_provider_managed_ingress() { .set_environment_ingress(SetEnvironmentIngress { environment_id: environment.environment_id, enabled: true, - public_endpoint: Some("https://invalid.example".to_owned()), updated_at_ms: 2_000, }) .await, @@ -306,27 +363,24 @@ async fn external_environment_cannot_enable_provider_managed_ingress() { } #[tokio::test(flavor = "current_thread")] -async fn external_environment_persists_a_typed_connection() { +async fn external_environment_persists_its_endpoint() { let store = InMemoryEnvironmentRegistryStore::new(); let environment = store .create_external_environment(CreateExternalEnvironment { request_id: EnvironmentProvisionRequestId::new("external-request"), environment_id: EnvironmentId::new("external-environment"), incarnation_id: EnvironmentIncarnationId::new("external-incarnation"), - connection: EnvironmentConnectionSpec::new( - "ws://envd.example:19091", - EnvironmentTransport::WebSocket, - ), + endpoint: endpoint("unix:///run/lightspeed/envd.sock"), display_name: None, metadata: BTreeMap::new(), created_at_ms: 1_000, }) .await .expect("create external environment"); - let EnvironmentSource::External { connection } = environment.source else { + let EnvironmentSource::External { endpoint } = environment.source else { panic!("external source") }; - assert_eq!(connection.endpoint, "ws://envd.example:19091"); + assert_eq!(endpoint.as_str(), "unix:///run/lightspeed/envd.sock"); } #[tokio::test(flavor = "current_thread")] @@ -450,23 +504,31 @@ async fn power_intent_and_idle_policy_are_provisioned_only_and_drive_reconcile_l assert!(environment.idle_policy.is_none()); store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 3_000, - }) + .observe_provisioned_environment(observe( + &environment.environment_id, + Some("target-1"), + EnvironmentStatus::Ready, + 3_000, + )) .await .expect("observe"); - // Ready and converged: nothing to reconcile. + // Ready and converged: nothing to reconcile, but a steady pass still + // observes the target. assert!( store - .list_environments_needing_reconcile() + .list_environments_needing_reconcile(ReconcileScope::Converging) .await .unwrap() .is_empty() ); + assert_eq!( + store + .list_environments_needing_reconcile(ReconcileScope::WithSteady) + .await + .unwrap() + .len(), + 1 + ); let paused = store .set_environment_power(SetEnvironmentPower { environment_id: environment.environment_id.clone(), @@ -477,23 +539,25 @@ async fn power_intent_and_idle_policy_are_provisioned_only_and_drive_reconcile_l .expect("set power"); assert_eq!(paused.desired_power, PowerState::Paused); assert!(paused.power_diverges()); - let pending = store.list_environments_needing_reconcile().await.unwrap(); + let pending = store + .list_environments_needing_reconcile(ReconcileScope::Converging) + .await + .unwrap(); assert_eq!(pending.len(), 1); // Observed paused: converged again. store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Paused, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 5_000, - }) + .observe_provisioned_environment(observe( + &environment.environment_id, + Some("target-1"), + EnvironmentStatus::Paused, + 5_000, + )) .await .expect("observe paused"); assert!( store - .list_environments_needing_reconcile() + .list_environments_needing_reconcile(ReconcileScope::Converging) .await .unwrap() .is_empty() @@ -520,13 +584,12 @@ async fn power_intent_and_idle_policy_are_provisioned_only_and_drive_reconcile_l .is_empty() ); store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 7_000, - }) + .observe_provisioned_environment(observe( + &environment.environment_id, + Some("target-1"), + EnvironmentStatus::Ready, + 7_000, + )) .await .expect("observe ready"); let candidates = store.list_environments_with_idle_policy().await.unwrap(); @@ -539,10 +602,7 @@ async fn power_intent_and_idle_policy_are_provisioned_only_and_drive_reconcile_l request_id: EnvironmentProvisionRequestId::new("external-1"), environment_id: EnvironmentId::new("external-env"), incarnation_id: EnvironmentIncarnationId::new("external-inc"), - connection: EnvironmentConnectionSpec::new( - "ws://envd.test:9000", - environment_protocol::shared::EnvironmentTransport::WebSocket, - ), + endpoint: endpoint("ws://envd.test:9000"), display_name: None, metadata: BTreeMap::new(), created_at_ms: 8_000, @@ -752,7 +812,7 @@ async fn registered_admission_is_keyed_by_daemon_identity() { environment_id: created.environment_id.clone(), observation: RegisteredConnectionObservation::Connected, observed_at_ms: 4_000, - metadata: RegisteredDaemonBuild { + metadata: DaemonBuild { version: Some("0.2.0".to_owned()), git_sha: Some("bbbb".to_owned()), protocol_version: 3, @@ -994,8 +1054,6 @@ async fn registered_environments_list_by_key() { /// itself stays enforced on the caller's keys. #[tokio::test(flavor = "current_thread")] async fn reserved_metadata_sits_on_top_of_the_caller_allowance() { - use environment_protocol::registration::MAX_METADATA_ENTRIES; - let (_universe_id, store) = store().await; minted_key( &store, @@ -1003,7 +1061,7 @@ async fn reserved_metadata_sits_on_top_of_the_caller_allowance() { registration_policy(RegisteredIdentityMode::Persistent), ) .await; - let reserved = RegisteredDaemonBuild { + let reserved = DaemonBuild { version: Some("0.1.0".to_owned()), git_sha: Some("a".repeat(40)), protocol_version: 2, @@ -1011,7 +1069,7 @@ async fn reserved_metadata_sits_on_top_of_the_caller_allowance() { .metadata(); let mut full = register("rk-meta", "env-meta", &daemon_key(0x31), 1_000); - full.metadata = (0..MAX_METADATA_ENTRIES) + full.metadata = (0..MAX_METADATA_ENTRIES as usize) .map(|index| (format!("caller.key{index}"), "v".to_owned())) .collect(); full.metadata.extend(reserved.clone()); @@ -1021,11 +1079,11 @@ async fn reserved_metadata_sits_on_top_of_the_caller_allowance() { .expect("full caller allowance plus reserved entries"); assert_eq!( created.metadata.len(), - MAX_METADATA_ENTRIES + reserved.len() + MAX_METADATA_ENTRIES as usize + reserved.len() ); let mut over = register("rk-meta", "env-over", &daemon_key(0x32), 1_000); - over.metadata = (0..=MAX_METADATA_ENTRIES) + over.metadata = (0..=MAX_METADATA_ENTRIES as usize) .map(|index| (format!("caller.key{index}"), "v".to_owned())) .collect(); assert!(matches!( @@ -1033,3 +1091,278 @@ async fn reserved_metadata_sits_on_top_of_the_caller_allowance() { Err(EnvironmentRegistryError::InvalidInput { .. }) )); } + +#[tokio::test(flavor = "current_thread")] +async fn provider_credentials_are_required_once_and_kept_across_puts() { + let store = InMemoryEnvironmentRegistryStore::new(); + let provider_id = EnvironmentProviderId::new("incus-local"); + let mut without = provider(); + without.credential = None; + assert!(matches!( + store.put_provider(without.clone()).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + let mut unprintable = provider(); + unprintable.credential = Some(SecretValue::new("two words")); + assert!(matches!( + store.put_provider(unprintable).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + + store.put_provider(provider()).await.expect("create"); + without.updated_at_ms = 2_000; + let replaced = store.put_provider(without).await.expect("replace"); + assert_eq!(replaced.created_at_ms, 1_000); + assert_eq!( + store + .read_provider_credential(&provider_id) + .await + .expect("credential") + .expose(), + "provider-secret" + ); + let debug = format!("{:?}", provider()); + assert!(!debug.contains("provider-secret")); +} + +#[tokio::test(flavor = "current_thread")] +async fn controller_facts_follow_the_registered_endpoint() { + let store = InMemoryEnvironmentRegistryStore::new(); + let provider_id = EnvironmentProviderId::new("incus-local"); + let created = store.put_provider(provider()).await.expect("create"); + assert_eq!(created.controller_features, None); + assert_eq!(created.data_route_base, None); + + let base = DataRouteBase::new("ws://127.0.0.1:19090/data").expect("base"); + let features = ControllerFeatures { + adopt_target: true, + ingress: false, + }; + let recorded = store + .record_provider_controller(RecordProviderController { + provider_id: provider_id.clone(), + controller_endpoint: created.controller_endpoint.clone(), + features, + data_route_base: base.clone(), + }) + .await + .expect("record"); + assert_eq!(recorded.controller_features, Some(features)); + assert_eq!(recorded.data_route_base, Some(base.clone())); + + // A put that keeps the endpoint keeps the facts. + let mut same = provider(); + same.credential = None; + same.display_name = Some("Renamed".to_owned()); + let kept = store.put_provider(same).await.expect("rename"); + assert_eq!(kept.data_route_base, Some(base.clone())); + + // A new endpoint clears them, and an initialize that raced the change + // cannot write facts of the old endpoint back. + let mut moved = provider(); + moved.credential = None; + moved.controller_endpoint = endpoint("wss://incus.example.com/control"); + let moved = store.put_provider(moved).await.expect("move"); + assert_eq!(moved.controller_features, None); + assert_eq!(moved.data_route_base, None); + let stale = store + .record_provider_controller(RecordProviderController { + provider_id, + controller_endpoint: created.controller_endpoint, + features, + data_route_base: base, + }) + .await + .expect("stale record"); + assert_eq!(stale.data_route_base, None); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_racing_observation_never_overwrites_a_close() { + let (_, store) = store().await; + let environment = store + .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) + .await + .expect("create"); + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: environment.environment_id.clone(), + updated_at_ms: 2_500, + }) + .await + .expect("begin close"); + + // The create that was in flight when the close arrived answers: the + // target id is still recorded so the close can address it, but the + // status stays closing. + let observed = store + .observe_provisioned_environment(observe( + &environment.environment_id, + Some("ls-3f2a"), + EnvironmentStatus::Ready, + 3_000, + )) + .await + .expect("observe"); + assert_eq!(observed.status, EnvironmentStatus::Closing); + assert_eq!( + observed.incarnation.provider_target_id, + Some(target("ls-3f2a")) + ); + + let failed = store + .fail_environment_lifecycle(FailEnvironmentLifecycle { + environment_id: environment.environment_id.clone(), + message: "power change failed".to_owned(), + daemon_build: None, + observed_at_ms: 3_500, + }) + .await + .expect("fail"); + assert_eq!(failed.status, EnvironmentStatus::Closing); + assert_eq!(failed.lifecycle_error, None); +} + +#[tokio::test(flavor = "current_thread")] +async fn observations_without_a_target_id_keep_the_recorded_one() { + let (_, store) = store().await; + let environment = store + .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) + .await + .expect("create"); + let pending = store + .observe_provisioned_environment(observe( + &environment.environment_id, + None, + EnvironmentStatus::Provisioning, + 2_500, + )) + .await + .expect("no id yet"); + assert_eq!(pending.incarnation.provider_target_id, None); + // Natural provider ids are opaque: slashes and colons are fine. + let natural = "arn:aws:ec2:eu-west-1:123456789012:instance/i-0abc"; + store + .observe_provisioned_environment(observe( + &environment.environment_id, + Some(natural), + EnvironmentStatus::Booting, + 3_000, + )) + .await + .expect("id assigned"); + let kept = store + .observe_provisioned_environment(observe( + &environment.environment_id, + None, + EnvironmentStatus::Booting, + 3_500, + )) + .await + .expect("id kept"); + assert_eq!(kept.incarnation.provider_target_id, Some(target(natural))); + assert!(matches!( + store + .observe_provisioned_environment(observe( + &environment.environment_id, + Some("another-target"), + EnvironmentStatus::Ready, + 4_000, + )) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); +} + +#[tokio::test(flavor = "current_thread")] +async fn lifecycle_errors_accumulate_until_a_steady_observation() { + let (_, store) = store().await; + let environment = store + .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) + .await + .expect("create"); + let first = store + .record_environment_lifecycle_error(RecordEnvironmentLifecycleError { + environment_id: environment.environment_id.clone(), + message: "controller unavailable".to_owned(), + observed_at_ms: 3_000, + }) + .await + .expect("first error"); + assert_eq!(first.status, EnvironmentStatus::Provisioning); + let second = store + .record_environment_lifecycle_error(RecordEnvironmentLifecycleError { + environment_id: environment.environment_id.clone(), + message: format!("connect failed\u{7}{}", "x".repeat(2_000)), + observed_at_ms: 4_000, + }) + .await + .expect("second error"); + let error = second.lifecycle_error.expect("recorded"); + assert_eq!(error.first_at_ms, 3_000); + assert_eq!(error.last_at_ms, 4_000); + assert!(error.message.len() <= 1024); + assert!(!error.message.contains('\u{7}')); + + let ready = store + .observe_provisioned_environment(ObserveProvisionedEnvironment { + daemon_build: Some(DaemonBuild { + version: Some("0.5.0".to_owned()), + git_sha: None, + protocol_version: 3, + }), + ..observe( + &environment.environment_id, + Some("target-1"), + EnvironmentStatus::Ready, + 5_000, + ) + }) + .await + .expect("ready"); + assert_eq!(ready.lifecycle_error, None); + assert_eq!( + ready + .metadata + .get(ENVD_PROTOCOL_VERSION_METADATA_KEY) + .map(String::as_str), + Some("3") + ); +} + +/// A change whose result fails validation must leave the stored record as +/// it was, so a bad value can never become durable. +#[tokio::test(flavor = "current_thread")] +async fn refused_changes_leave_the_stored_record_untouched() { + let (_universe_id, store) = store().await; + minted_key( + &store, + "rk-poison", + registration_policy(RegisteredIdentityMode::Persistent), + ) + .await; + let created = store + .create_registered_environment(register("rk-poison", "env-p", &daemon_key(0x41), 1_000)) + .await + .expect("register"); + let refused = store + .observe_registered_environment(ObserveRegisteredEnvironment { + environment_id: created.environment_id.clone(), + observation: RegisteredConnectionObservation::Connected, + observed_at_ms: 2_000, + metadata: BTreeMap::from([( + ENVD_VERSION_METADATA_KEY.to_owned(), + "bad\u{0}version".to_owned(), + )]), + }) + .await; + assert!(matches!( + refused, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + let stored = store + .read_environment(&created.environment_id) + .await + .expect("still readable"); + assert_eq!(stored, created); +} diff --git a/crates/harness/src/storage/blobs.rs b/crates/harness/src/storage/blobs.rs index 25f9fdd2f..580488ed3 100644 --- a/crates/harness/src/storage/blobs.rs +++ b/crates/harness/src/storage/blobs.rs @@ -126,19 +126,26 @@ pub trait BlobSource: Send { async fn read_chunk(&mut self, max_bytes: usize) -> Result, BlobStoreError>; } +/// Largest range one [`BlobStore::read_blob_range`] call may request; stores +/// refuse larger ranges, so callers moving bigger chunks read several ranges. +pub const MAX_BLOB_RANGE_BYTES: usize = 1024 * 1024; + #[async_trait] pub trait BlobStore: Send + Sync { async fn put_bytes(&self, bytes: Vec) -> Result; - /// Bounded range access for transfer. Ranges are verified by the receiving whole-file - /// hasher. The fallback deliberately rejects large blobs instead of buffering them. + /// Bounded range access for transfer, at most [`MAX_BLOB_RANGE_BYTES`] per call. Ranges + /// are verified by the receiving whole-file hasher. The fallback deliberately rejects + /// large blobs instead of buffering them. async fn read_blob_range( &self, blob_ref: &BlobRef, offset: u64, max_bytes: usize, ) -> Result, BlobStoreError> { - if max_bytes > 1024 * 1024 || self.stat_blob(blob_ref).await?.byte_len > 8 * 1024 * 1024 { + if max_bytes > MAX_BLOB_RANGE_BYTES + || self.stat_blob(blob_ref).await?.byte_len > 8 * 1024 * 1024 + { return Err(BlobStoreError::Store { message: "streaming blob reads unsupported by this store".into(), }); diff --git a/crates/harness/src/storage/mod.rs b/crates/harness/src/storage/mod.rs index 983605a77..486021833 100644 --- a/crates/harness/src/storage/mod.rs +++ b/crates/harness/src/storage/mod.rs @@ -14,7 +14,8 @@ pub use crate::session::{ pub use blobs::{ BlobCacheLimits, BlobCacheStats, BlobEdge, BlobGraphStore, BlobInfo, BlobSource, BlobStore, BlobStoreError, CachedBlobStore, ENGINE_BLOB_CONTENTS, InMemoryBlobCache, InMemoryBlobStore, - collect_blob_refs, ensure_harness_blobs, harness_blob_refs, record_contains_edges, + MAX_BLOB_RANGE_BYTES, collect_blob_refs, ensure_harness_blobs, harness_blob_refs, + record_contains_edges, }; pub use session::{ AdvanceSessionCheckpoint, AppendSessionEvents, AppendSessionEventsResult, CreateClonedSession, diff --git a/crates/llm-runtime/tests/fixtures/builtin_catalogs.json b/crates/llm-runtime/tests/fixtures/builtin_catalogs.json index fdb51e733..65234b66b 100644 --- a/crates/llm-runtime/tests/fixtures/builtin_catalogs.json +++ b/crates/llm-runtime/tests/fixtures/builtin_catalogs.json @@ -3304,7 +3304,7 @@ "type": "object" }, "job_id": { - "description": "Your name for the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read.", + "description": "Your name for this run of the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read. An id names one run: to run a command again, choose a new id.", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", "type": "string" }, @@ -6672,7 +6672,7 @@ "type": "object" }, "job_id": { - "description": "Your name for the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read.", + "description": "Your name for this run of the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read. An id names one run: to run a command again, choose a new id.", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", "type": "string" }, @@ -10112,7 +10112,7 @@ "type": "object" }, "job_id": { - "description": "Your name for the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read.", + "description": "Your name for this run of the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read. An id names one run: to run a command again, choose a new id.", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", "type": "string" }, diff --git a/crates/profiles/src/lib.rs b/crates/profiles/src/lib.rs index 0b88113b7..9e9c9f8ea 100644 --- a/crates/profiles/src/lib.rs +++ b/crates/profiles/src/lib.rs @@ -161,10 +161,11 @@ impl ProfileSourceExt for ProfileSource { } pub fn validate_profile_document(document: &ProfileDocument) -> Result<(), ProfileError> { - environment_protocol::registration::validate_registration_metadata(None, &document.metadata) - .map_err(|message| ProfileError::InvalidInput { - message: format!("invalid metadata: {message}"), - })?; + environment_protocol::metadata::validate_caller_metadata(&document.metadata).map_err( + |error| ProfileError::InvalidInput { + message: format!("invalid metadata: {error}"), + }, + )?; if let Some(value) = document .retention .as_ref() diff --git a/crates/store-fs/src/blob.rs b/crates/store-fs/src/blob.rs index 0023d17f7..a1c91a93f 100644 --- a/crates/store-fs/src/blob.rs +++ b/crates/store-fs/src/blob.rs @@ -93,7 +93,7 @@ impl BlobStore for FsBlobStore { max_bytes: usize, ) -> Result, BlobStoreError> { use tokio::io::{AsyncReadExt, AsyncSeekExt}; - if max_bytes > 1024 * 1024 { + if max_bytes > harness::storage::MAX_BLOB_RANGE_BYTES { return Err(BlobStoreError::Store { message: "blob range limit exceeded".into(), }); diff --git a/crates/store-pg/migrations/012_environment_endpoints.sql b/crates/store-pg/migrations/012_environment_endpoints.sql new file mode 100644 index 000000000..1b39a1663 --- /dev/null +++ b/crates/store-pg/migrations/012_environment_endpoints.sql @@ -0,0 +1,138 @@ +-- Environment protocol v3 storage. +-- +-- Design notes: +-- - Providers and external environments are reached at one validated +-- endpoint URL (ws://, wss:// or unix://) whose scheme selects the dialer; +-- the transport JSON documents are gone. +-- - The provider credential is the bearer token Lightspeed presents on the +-- controller and data routes. It is sealed with AES-256-GCM by the store +-- adapter, exactly like auth_secrets, and never stored in plaintext. +-- - controller_adopt_target, controller_ingress and data_route_base are what +-- the controller advertised at its last successful initialize: all three +-- or none. +-- - lifecycle_error records why an environment failed, or the latest error +-- its lifecycle is retrying past, instead of a caller metadata entry. +-- - Provider target ids follow the protocol's opaque rule. +-- - public_ingress_enabled is the ingress intent and public_endpoint what +-- the provider last reported realized; the reconciler converges the +-- endpoint toward the intent, so either may lead the other for a while. +-- +-- Rows written in the earlier shape are not converted: purge providers and +-- external environments before migrating a database that has them. + +ALTER TABLE environment_providers + DROP CONSTRAINT environment_providers_json_objects; + +ALTER TABLE environment_providers + DROP COLUMN controller_connection_json, + ADD COLUMN controller_endpoint text NOT NULL, + ADD COLUMN credential_key_id text NOT NULL, + ADD COLUMN credential_nonce bytea NOT NULL, + ADD COLUMN credential_ciphertext bytea NOT NULL, + ADD COLUMN controller_adopt_target boolean, + ADD COLUMN controller_ingress boolean, + ADD COLUMN data_route_base text, + ADD CONSTRAINT environment_providers_metadata_object + CHECK (jsonb_typeof(metadata_json) = 'object'), + ADD CONSTRAINT environment_providers_controller_endpoint_format CHECK ( + controller_endpoint ~ '^(wss?|unix)://[^[:space:][:cntrl:]]+$' + AND octet_length(controller_endpoint) <= 2048 + ), + ADD CONSTRAINT environment_providers_credential_sealed CHECK ( + credential_key_id <> '' AND octet_length(credential_nonce) = 12 + ), + ADD CONSTRAINT environment_providers_controller_facts CHECK ( + (controller_adopt_target IS NULL) = (controller_ingress IS NULL) + AND (controller_ingress IS NULL) = (data_route_base IS NULL) + ), + ADD CONSTRAINT environment_providers_data_route_base_format CHECK ( + data_route_base IS NULL + OR (data_route_base ~ '^wss?://' AND octet_length(data_route_base) <= 2048) + ); + +ALTER TABLE environments + DROP CONSTRAINT environments_source_fields, + DROP CONSTRAINT environments_public_ingress_fields; + +ALTER TABLE environments + DROP COLUMN daemon_connection_json, + ADD COLUMN daemon_endpoint text, + ADD COLUMN lifecycle_error text, + ADD COLUMN lifecycle_error_first_at_ms bigint, + ADD COLUMN lifecycle_error_last_at_ms bigint, + ADD CONSTRAINT environments_source_fields CHECK ( + ( + source_kind = 'provisioned' + AND provider_id IS NOT NULL AND binding_id IS NOT NULL + AND daemon_endpoint IS NULL + AND registration_key_id IS NULL AND daemon_id IS NULL + AND daemon_public_key IS NULL AND identity_mode IS NULL + AND last_seen_at_ms IS NULL + ) + OR ( + source_kind = 'external' + AND provider_id IS NULL AND binding_id IS NULL + AND daemon_endpoint ~ '^(wss?|unix)://[^[:space:][:cntrl:]]+$' + AND octet_length(daemon_endpoint) <= 2048 + AND registration_key_id IS NULL AND daemon_id IS NULL + AND daemon_public_key IS NULL AND identity_mode IS NULL + AND last_seen_at_ms IS NULL + ) + OR ( + source_kind = 'registered' + AND provider_id IS NULL AND binding_id IS NULL + AND daemon_endpoint IS NULL + AND registration_key_id IS NOT NULL + AND daemon_id ~ '^daemon_[0-9a-f]{64}$' + AND daemon_public_key ~ '^[0-9a-f]{64}$' + AND identity_mode IN ('persistent', 'ephemeral') + AND (last_seen_at_ms IS NULL OR last_seen_at_ms >= 0) + ) + ), + ADD CONSTRAINT environments_public_ingress_fields CHECK ( + (NOT public_ingress_enabled AND public_endpoint IS NULL) + OR ( + source_kind = 'provisioned' + AND (public_endpoint IS NULL OR public_endpoint <> '') + ) + ), + ADD CONSTRAINT environments_lifecycle_error_fields CHECK ( + ( + lifecycle_error IS NULL + AND lifecycle_error_first_at_ms IS NULL + AND lifecycle_error_last_at_ms IS NULL + ) + OR ( + octet_length(lifecycle_error) BETWEEN 1 AND 1024 + AND lifecycle_error_first_at_ms >= 0 + AND lifecycle_error_last_at_ms >= lifecycle_error_first_at_ms + ) + ); + +ALTER TABLE environment_incarnations + ADD CONSTRAINT environment_incarnations_provider_target_id_opaque CHECK ( + provider_target_id IS NULL + OR ( + octet_length(provider_target_id) BETWEEN 1 AND 255 + AND provider_target_id !~ '[[:cntrl:]]' + ) + ); + +COMMENT ON TABLE environment_providers IS + 'Operator-registered provider identity, controller endpoint and sealed credential, plus the controller facts observed at its last initialize.'; +COMMENT ON COLUMN environment_providers.controller_endpoint IS + 'Validated ws://, wss:// or unix:// URL of the provider controller; the scheme selects the dialer.'; +COMMENT ON COLUMN environment_providers.credential_ciphertext IS + 'AES-256-GCM sealed bearer credential for the controller and data routes; AAD binds it to the provider id.'; +COMMENT ON COLUMN environment_providers.data_route_base IS + 'Data route base (routes.data) the controller advertised at its last successful initialize; target data routes are built from it.'; +COMMENT ON TABLE environments IS + 'Universe-owned logical environments. External environments store a Lightspeed-reachable envd endpoint URL; provisioned environments retain provider routing linkage.'; +COMMENT ON COLUMN environments.public_ingress_enabled IS + 'Lightspeed-owned public ingress intent; the lifecycle reconciler converges the provider target toward it.'; +COMMENT ON COLUMN environments.public_endpoint IS + 'Public endpoint the provider last reported realized; may lag public_ingress_enabled while the reconciler converges.'; +COMMENT ON COLUMN environments.lifecycle_error IS + 'Why the environment failed, or the latest lifecycle error being retried; first/last instants bound the current run of errors.'; +COMMENT ON COLUMN environment_incarnations.provider_target_id IS + 'Opaque provider-scoped target handle (1-255 bytes, no control characters); interpreted with the environment provider identity.'; diff --git a/crates/store-pg/src/auth.rs b/crates/store-pg/src/auth.rs index f97f23b03..7462f4205 100644 --- a/crates/store-pg/src/auth.rs +++ b/crates/store-pg/src/auth.rs @@ -21,11 +21,11 @@ use sqlx::Row; use crate::PgStore; -const LOCAL_KEY_ID: &str = "local-v1"; +pub(crate) const LOCAL_KEY_ID: &str = "local-v1"; const NONCE_LEN: usize = 12; impl PgStore { - fn secrets_cipher(&self) -> Result<(Aes256Gcm, &'static str), AuthRegistryError> { + pub(crate) fn secrets_cipher(&self) -> Result<(Aes256Gcm, &'static str), AuthRegistryError> { let Some(key) = &self.config.secrets_master_key else { return Err(AuthRegistryError::Store { message: "secrets master key is not configured; set LIGHTSPEED_SECRETS_MASTER_KEY" @@ -49,7 +49,7 @@ impl PgStore { } } -fn seal_secret( +pub(crate) fn seal_secret( cipher: &Aes256Gcm, aad: &[u8], value: &SecretValue, @@ -72,7 +72,7 @@ fn seal_secret( Ok((nonce.to_vec(), ciphertext)) } -fn open_secret( +pub(crate) fn open_secret( cipher: &Aes256Gcm, aad: &[u8], nonce: &[u8], diff --git a/crates/store-pg/src/blob.rs b/crates/store-pg/src/blob.rs index 6b2863b53..1d62008c0 100644 --- a/crates/store-pg/src/blob.rs +++ b/crates/store-pg/src/blob.rs @@ -214,7 +214,7 @@ impl BlobStore for PgStore { max_bytes: usize, ) -> Result, BlobStoreError> { use object_store::ObjectStoreExt; - if max_bytes > 1024 * 1024 { + if max_bytes > harness::storage::MAX_BLOB_RANGE_BYTES { return Err(BlobStoreError::Store { message: "blob range limit exceeded".into(), }); diff --git a/crates/store-pg/src/environment.rs b/crates/store-pg/src/environment.rs index 61d20da40..8f60050a8 100644 --- a/crates/store-pg/src/environment.rs +++ b/crates/store-pg/src/environment.rs @@ -1,12 +1,13 @@ // Formatted SQL uses internal schema fragments; request values use bind parameters. use async_trait::async_trait; -use auth::{AuthGrantId, AuthProviderId, SecretId}; -use environment_protocol::shared::ProviderTargetId; +use auth::{AuthGrantId, AuthProviderId, SecretId, SecretValue}; +use environment_protocol::{ids::ProviderTargetId, routes::DataRouteBase}; use environments::{ - AdoptEnvironment, BeginCloseEnvironment, CreateEnvironment, CreateExternalEnvironment, - CreateRegisteredEnvironment, EnvironmentCredentialRecord, EnvironmentCredentialSource, - EnvironmentCredentialStore, EnvironmentDaemonId, EnvironmentId, EnvironmentIncarnationId, - EnvironmentIncarnationRecord, EnvironmentProviderBindingId, EnvironmentProviderBindingRecord, + AdoptEnvironment, BeginCloseEnvironment, ControllerFeatures, CreateEnvironment, + CreateExternalEnvironment, CreateRegisteredEnvironment, EnvironmentCredentialRecord, + EnvironmentCredentialSource, EnvironmentCredentialStore, EnvironmentDaemonId, + EnvironmentEndpoint, EnvironmentId, EnvironmentIncarnationId, EnvironmentIncarnationRecord, + EnvironmentLifecycleError, EnvironmentProviderBindingId, EnvironmentProviderBindingRecord, EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderRecord, EnvironmentProviderStore, EnvironmentProvisionRequestId, EnvironmentRecord, EnvironmentRegistrationKeyId, EnvironmentRegistryError, EnvironmentSource, @@ -14,16 +15,22 @@ use environments::{ FinishCloseEnvironment, ListEnvironmentCredentials, ListEnvironmentProviders, ListEnvironments, ObserveProvisionedEnvironment, ObserveRegisteredEnvironment, PowerState, PutEnvironmentCredential, PutEnvironmentProvider, PutEnvironmentProviderBinding, - RegisteredConnectionObservation, RegisteredIdentityMode, SetEnvironmentIdlePolicy, - SetEnvironmentIngress, SetEnvironmentPower, + ReconcileScope, RecordEnvironmentIngress, RecordEnvironmentLifecycleError, + RecordProviderController, RegisteredConnectionObservation, RegisteredIdentityMode, + SetEnvironmentIdlePolicy, SetEnvironmentIngress, SetEnvironmentPower, + apply_registered_observation, }; use sqlx::{Postgres, Row, Transaction}; use uuid::Uuid; -use crate::PgStore; +use crate::{ + PgStore, + auth::{LOCAL_KEY_ID, open_secret, seal_secret}, +}; const PROVIDER_COLUMNS: &str = r#" - provider_id, display_name, controller_connection_json, metadata_json, + provider_id, display_name, controller_endpoint, metadata_json, + controller_adopt_target, controller_ingress, data_route_base, created_at_ms, updated_at_ms "#; @@ -33,10 +40,11 @@ const BINDING_COLUMNS: &str = r#" "#; const ENVIRONMENT_COLUMNS: &str = r#" - e.environment_id, e.request_id, e.source_kind, e.provider_id, e.binding_id, e.daemon_connection_json, + e.environment_id, e.request_id, e.source_kind, e.provider_id, e.binding_id, e.daemon_endpoint, e.registration_key_id, e.daemon_id, e.daemon_public_key, e.identity_mode, e.last_seen_at_ms, e.display_name, e.status, e.desired_power, e.idle_policy_json, e.public_ingress_enabled, e.public_endpoint, e.metadata_json, + e.lifecycle_error, e.lifecycle_error_first_at_ms, e.lifecycle_error_last_at_ms, e.created_at_ms, e.updated_at_ms, i.incarnation_id, i.provision_request_id, i.provider_target_id, i.template_id, i.adoption_source_target, i.power_states_json, @@ -46,13 +54,23 @@ const ENVIRONMENT_COLUMNS: &str = r#" /// SQL predicate: the observed steady power state differs from the desired /// one, so the lifecycle reconciler has power work to do. -const POWER_DIVERGES_SQL: &str = r#"( - e.source_kind = 'provisioned' AND ( +/// `EnvironmentRecord::needs_reconcile` over `environments e` without the +/// steady clause: convergence pending (status, power or ingress). +pub(crate) const CONVERGING_SQL: &str = r#"( + e.status IN ('provisioning','booting','closing','unknown') + OR (e.source_kind = 'provisioned' AND ( (e.status = 'ready' AND e.desired_power <> 'running') OR (e.status = 'paused' AND e.desired_power <> 'paused') OR (e.status = 'suspended' AND e.desired_power <> 'suspended') OR (e.status = 'offline' AND e.desired_power <> 'stopped') - ) + OR (e.status = 'ready' AND e.public_ingress_enabled <> (e.public_endpoint IS NOT NULL)) + )) +)"#; + +/// Steady provisioned environments, over `environments e`; with the +/// incarnation join the recorded target is required too. +pub(crate) const STEADY_SQL: &str = r#"( + e.source_kind = 'provisioned' AND e.status IN ('ready','offline','paused','suspended') )"#; const ENVIRONMENT_JOIN: &str = r#" @@ -74,34 +92,101 @@ impl EnvironmentProviderStore for PgStore { &self, request: PutEnvironmentProvider, ) -> Result { - let record = request.into_record()?; + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin environment provider put", error))?; let query = format!( - r#" - INSERT INTO environment_providers ( - provider_id, display_name, controller_connection_json, - metadata_json, created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,$4,$5,$6) - ON CONFLICT (provider_id) DO UPDATE SET - display_name = EXCLUDED.display_name, - controller_connection_json = EXCLUDED.controller_connection_json, - metadata_json = EXCLUDED.metadata_json, - updated_at_ms = EXCLUDED.updated_at_ms - RETURNING {PROVIDER_COLUMNS} - "# + "SELECT {PROVIDER_COLUMNS} FROM environment_providers WHERE provider_id = $1 FOR UPDATE" ); - let row = sqlx::query(sqlx::AssertSqlSafe(query)) - .bind(record.provider_id.as_str()) - .bind(record.display_name.as_deref()) - .bind(json_value( - "encode provider controller connection", - &record.controller_connection, - )?) - .bind(json_value("encode provider metadata", &record.metadata)?) - .bind(record.created_at_ms) - .bind(record.updated_at_ms) - .fetch_one(&self.pool) + // A row that no longer decodes is replaced as if it were absent, so + // an operator can repair it with a put that carries a credential. + let existing = sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(request.provider_id.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|error| sql_error("lock environment provider", error))? + .and_then(|row| decoded_or_logged(provider_from_row(&row), "environment provider")); + let credential = request.credential.clone(); + let record = request.into_record(existing.as_ref())?; + let features = record.controller_features; + let row = match credential { + Some(credential) => { + let (key_id, nonce, ciphertext) = + self.seal_provider_credential(&record.provider_id, &credential)?; + let query = format!( + r#" + INSERT INTO environment_providers ( + provider_id, display_name, controller_endpoint, metadata_json, + controller_adopt_target, controller_ingress, data_route_base, + credential_key_id, credential_nonce, credential_ciphertext, + created_at_ms, updated_at_ms + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) + ON CONFLICT (provider_id) DO UPDATE SET + display_name = EXCLUDED.display_name, + controller_endpoint = EXCLUDED.controller_endpoint, + metadata_json = EXCLUDED.metadata_json, + controller_adopt_target = EXCLUDED.controller_adopt_target, + controller_ingress = EXCLUDED.controller_ingress, + data_route_base = EXCLUDED.data_route_base, + credential_key_id = EXCLUDED.credential_key_id, + credential_nonce = EXCLUDED.credential_nonce, + credential_ciphertext = EXCLUDED.credential_ciphertext, + created_at_ms = EXCLUDED.created_at_ms, + updated_at_ms = EXCLUDED.updated_at_ms + RETURNING {PROVIDER_COLUMNS} + "# + ); + sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(record.provider_id.as_str()) + .bind(record.display_name.as_deref()) + .bind(record.controller_endpoint.as_str()) + .bind(json_value("encode provider metadata", &record.metadata)?) + .bind(features.map(|features| features.adopt_target)) + .bind(features.map(|features| features.ingress)) + .bind(record.data_route_base.as_ref().map(DataRouteBase::as_str)) + .bind(key_id) + .bind(nonce) + .bind(ciphertext) + .bind(record.created_at_ms) + .bind(record.updated_at_ms) + .fetch_one(&mut *tx) + .await + .map_err(|error| sql_error("put environment provider", error))? + } + None => { + let query = format!( + r#" + UPDATE environment_providers SET + display_name = $2, controller_endpoint = $3, metadata_json = $4, + controller_adopt_target = $5, controller_ingress = $6, + data_route_base = $7, created_at_ms = $8, updated_at_ms = $9 + WHERE provider_id = $1 + RETURNING {PROVIDER_COLUMNS} + "# + ); + sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(record.provider_id.as_str()) + .bind(record.display_name.as_deref()) + .bind(record.controller_endpoint.as_str()) + .bind(json_value("encode provider metadata", &record.metadata)?) + .bind(features.map(|features| features.adopt_target)) + .bind(features.map(|features| features.ingress)) + .bind(record.data_route_base.as_ref().map(DataRouteBase::as_str)) + .bind(record.created_at_ms) + .bind(record.updated_at_ms) + .fetch_optional(&mut *tx) + .await + .map_err(|error| sql_error("update environment provider", error))? + .ok_or_else(|| { + invalid_error("a new environment provider requires a credential") + })? + } + }; + tx.commit() .await - .map_err(|error| sql_error("put environment provider", error))?; + .map_err(|error| sql_error("commit environment provider put", error))?; provider_from_row(&row) } @@ -120,6 +205,66 @@ impl EnvironmentProviderStore for PgStore { provider_from_row(&row) } + async fn read_provider_credential( + &self, + provider_id: &EnvironmentProviderId, + ) -> Result { + let row = sqlx::query( + "SELECT credential_key_id, credential_nonce, credential_ciphertext FROM environment_providers WHERE provider_id = $1", + ) + .bind(provider_id.as_str()) + .fetch_optional(&self.pool) + .await + .map_err(|error| sql_error("read environment provider credential", error))? + .ok_or_else(|| not_found("environment_provider", provider_id))?; + let key_id: String = row + .try_get("credential_key_id") + .map_err(|error| sql_error("decode provider credential key id", error))?; + if key_id != LOCAL_KEY_ID { + return Err(store_message(format!( + "unsupported provider credential key id '{key_id}'" + ))); + } + let nonce: Vec = row + .try_get("credential_nonce") + .map_err(|error| sql_error("decode provider credential nonce", error))?; + let ciphertext: Vec = row + .try_get("credential_ciphertext") + .map_err(|error| sql_error("decode provider credential", error))?; + let (cipher, _) = self + .secrets_cipher() + .map_err(|error| store_message(error.to_string()))?; + open_secret( + &cipher, + &provider_credential_aad(provider_id), + &nonce, + &ciphertext, + ) + .map_err(|error| store_message(format!("open provider credential: {error}"))) + } + + async fn record_provider_controller( + &self, + request: RecordProviderController, + ) -> Result { + sqlx::query( + r#" + UPDATE environment_providers SET + controller_adopt_target = $3, controller_ingress = $4, data_route_base = $5 + WHERE provider_id = $1 AND controller_endpoint = $2 + "#, + ) + .bind(request.provider_id.as_str()) + .bind(request.controller_endpoint.as_str()) + .bind(request.features.adopt_target) + .bind(request.features.ingress) + .bind(request.data_route_base.as_str()) + .execute(&self.pool) + .await + .map_err(|error| sql_error("record environment provider controller", error))?; + self.read_provider(&request.provider_id).await + } + async fn list_providers( &self, _request: ListEnvironmentProviders, @@ -130,7 +275,11 @@ impl EnvironmentProviderStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list environment providers", error))?; - rows.iter().map(provider_from_row).collect() + Ok(decode_rows( + &rows, + provider_from_row, + "environment provider", + )) } async fn delete_provider( @@ -160,6 +309,28 @@ impl EnvironmentProviderStore for PgStore { } } +impl PgStore { + fn seal_provider_credential( + &self, + provider_id: &EnvironmentProviderId, + credential: &SecretValue, + ) -> Result<(&'static str, Vec, Vec), EnvironmentRegistryError> { + let (cipher, key_id) = self + .secrets_cipher() + .map_err(|error| store_message(error.to_string()))?; + let (nonce, ciphertext) = + seal_secret(&cipher, &provider_credential_aad(provider_id), credential) + .map_err(|error| store_message(format!("seal provider credential: {error}")))?; + Ok((key_id, nonce, ciphertext)) + } +} + +/// Binds a sealed provider credential to its provider, so ciphertexts +/// cannot be moved between provider rows. +fn provider_credential_aad(provider_id: &EnvironmentProviderId) -> Vec { + format!("environment-provider/{}/credential", provider_id.as_str()).into_bytes() +} + #[async_trait] impl EnvironmentProviderBindingStore for PgStore { async fn put_provider_binding( @@ -264,7 +435,11 @@ impl EnvironmentProviderBindingStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list provider bindings", error))?; - rows.iter().map(binding_from_row).collect() + Ok(decode_rows( + &rows, + binding_from_row, + "environment provider binding", + )) } async fn delete_provider_binding( @@ -340,6 +515,7 @@ impl EnvironmentStore for PgStore { if binding.status != EnvironmentProviderBindingStatus::Enabled { return invalid("environment provider binding is disabled"); } + request.record(binding.provider_id.clone())?; sqlx::query( r#" INSERT INTO environments ( @@ -404,14 +580,6 @@ impl EnvironmentStore for PgStore { if request.created_at_ms < 0 { return invalid("environment timestamp must be nonnegative"); } - if request.source_target.is_empty() - || request.source_target.len() > 255 - || request.source_target.chars().any(char::is_control) - { - return invalid( - "adoption source must be non-empty, at most 255 bytes, and contain no control characters", - ); - } let mut tx = self .pool .begin() @@ -447,6 +615,7 @@ impl EnvironmentStore for PgStore { if binding.status != EnvironmentProviderBindingStatus::Enabled { return invalid("environment provider binding is disabled"); } + request.record(binding.provider_id.clone())?; sqlx::query( r#" INSERT INTO environments ( @@ -504,34 +673,7 @@ impl EnvironmentStore for PgStore { if request.created_at_ms < 0 { return invalid("environment timestamp must be nonnegative"); } - let candidate = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request.request_id.clone(), - source: EnvironmentSource::External { - connection: request.connection.clone(), - }, - display_name: request.display_name.clone(), - status: EnvironmentStatus::Ready, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id.clone(), - provision_request_id: None, - provider_target_id: None, - template_id: None, - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata.clone(), - last_seen_at_ms: None, - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - candidate.validate()?; + request.record()?; let mut tx = self .pool .begin() @@ -556,7 +698,7 @@ impl EnvironmentStore for PgStore { sqlx::query( r#" INSERT INTO environments ( - universe_id, environment_id, request_id, source_kind, daemon_connection_json, + universe_id, environment_id, request_id, source_kind, daemon_endpoint, display_name, status, current_incarnation_id, metadata_json, created_at_ms, updated_at_ms ) VALUES ($1,$2,$3,'external',$4,$5,'ready',$6,$7,$8,$8) @@ -565,7 +707,7 @@ impl EnvironmentStore for PgStore { .bind(self.config.universe_id) .bind(request.environment_id.as_str()) .bind(request.request_id.as_str()) - .bind(json_value("encode daemon connection", &request.connection)?) + .bind(request.endpoint.as_str()) .bind(request.display_name.as_deref()) .bind(request.incarnation_id.as_str()) .bind(json_value( @@ -654,6 +796,7 @@ impl EnvironmentStore for PgStore { }); } } + request.record(&key)?; sqlx::query( r#" INSERT INTO environments ( @@ -725,9 +868,17 @@ impl EnvironmentStore for PgStore { return invalid("observed_at_ms must be nonnegative"); } let current = self.read_environment(&request.environment_id).await?; - if !current.is_registered() { - return invalid("connection observations apply only to registered environments"); - } + // Validate the merged result before anything becomes durable: a + // build fact that breaks the record's bounds is refused here rather + // than poisoning the row for every later read. + let mut candidate = current.clone(); + apply_registered_observation( + &mut candidate, + request.observation, + request.observed_at_ms, + request.metadata.clone(), + )?; + candidate.validate()?; // One conditional statement rather than read-modify-write: a close // that lands between a read and a write must never be overwritten // by a late connect or disconnect observation. @@ -782,7 +933,7 @@ impl EnvironmentStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list open registered environments", error))?; - rows.iter().map(environment_from_row).collect() + Ok(decode_rows(&rows, environment_from_row, "environment")) } async fn read_environment( @@ -861,21 +1012,27 @@ impl EnvironmentStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list environments", error))?; - rows.iter().map(environment_from_row).collect() + Ok(decode_rows(&rows, environment_from_row, "environment")) } async fn list_environments_needing_reconcile( &self, + scope: ReconcileScope, ) -> Result, EnvironmentRegistryError> { - let query = format!( - "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1 AND (e.status IN ('provisioning','booting','closing','unknown') OR {POWER_DIVERGES_SQL}) ORDER BY e.updated_at_ms, e.environment_id" - ); + let query = match scope { + ReconcileScope::Converging => format!( + "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1 AND {CONVERGING_SQL} ORDER BY e.updated_at_ms, e.environment_id" + ), + ReconcileScope::WithSteady => format!( + "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1 AND ({CONVERGING_SQL} OR ({STEADY_SQL} AND i.provider_target_id IS NOT NULL)) ORDER BY e.updated_at_ms, e.environment_id" + ), + }; let rows = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(self.config.universe_id) .fetch_all(&self.pool) .await .map_err(|error| sql_error("list environments needing reconcile", error))?; - rows.iter().map(environment_from_row).collect() + Ok(decode_rows(&rows, environment_from_row, "environment")) } async fn list_environments_with_idle_policy( @@ -889,7 +1046,7 @@ impl EnvironmentStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list environments with idle policy", error))?; - rows.iter().map(environment_from_row).collect() + Ok(decode_rows(&rows, environment_from_row, "environment")) } async fn observe_provisioned_environment( @@ -897,26 +1054,81 @@ impl EnvironmentStore for PgStore { request: ObserveProvisionedEnvironment, ) -> Result { let current = self.read_environment(&request.environment_id).await?; - if request.observed_at_ms < current.incarnation.updated_at_ms { + let mut candidate = current.clone(); + request.apply(&mut candidate)?; + candidate.validate()?; + if candidate == current { return Ok(current); } - if current - .incarnation - .provider_target_id - .as_ref() - .is_some_and(|id| id != &request.provider_target_id) - { - return invalid("provider target conflicts with current incarnation"); - } - sqlx::query("UPDATE environment_incarnations SET provider_target_id=$4, power_states_json=$5, updated_at_ms=$6 WHERE universe_id=$1 AND environment_id=$2 AND incarnation_id=$3") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(current.incarnation.incarnation_id.as_str()) - .bind(request.provider_target_id.as_str()) - .bind(json_value("encode power states", &request.power_states)?) - .bind(request.observed_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("observe environment incarnation", error))?; - sqlx::query("UPDATE environments SET status=$3, updated_at_ms=$4 WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(environment_status_to_str(request.status)).bind(request.observed_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("observe environment", error))?; + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin environment observation", error))?; + // The target id and power states are recorded whatever the status, + // so a close that raced the first create can still address the + // target; the predicates refuse stale and conflicting observations + // that raced this one. + sqlx::query( + r#" + UPDATE environment_incarnations SET + provider_target_id = COALESCE($4, provider_target_id), + power_states_json = $5, + updated_at_ms = $6 + WHERE universe_id = $1 AND environment_id = $2 AND incarnation_id = $3 + AND updated_at_ms <= $6 + AND ($4::text IS NULL OR provider_target_id IS NULL OR provider_target_id = $4) + "#, + ) + .bind(self.config.universe_id) + .bind(request.environment_id.as_str()) + .bind(current.incarnation.incarnation_id.as_str()) + .bind( + request + .provider_target_id + .as_ref() + .map(ProviderTargetId::as_str), + ) + .bind(json_value("encode power states", &request.power_states)?) + .bind(request.observed_at_ms) + .execute(&mut *tx) + .await + .map_err(|error| sql_error("observe environment incarnation", error))?; + // A close accepted while this observation was in flight wins. + sqlx::query( + r#" + UPDATE environments SET + status = $3, + metadata_json = metadata_json || $4, + lifecycle_error = CASE WHEN $5 THEN NULL ELSE lifecycle_error END, + lifecycle_error_first_at_ms = + CASE WHEN $5 THEN NULL ELSE lifecycle_error_first_at_ms END, + lifecycle_error_last_at_ms = + CASE WHEN $5 THEN NULL ELSE lifecycle_error_last_at_ms END, + updated_at_ms = GREATEST(updated_at_ms, $6) + WHERE universe_id = $1 AND environment_id = $2 + AND status NOT IN ('closing', 'closed') + "#, + ) + .bind(self.config.universe_id) + .bind(request.environment_id.as_str()) + .bind(environment_status_to_str(request.status)) + .bind(json_value( + "encode daemon build metadata", + &request + .daemon_build + .as_ref() + .map(|build| build.metadata()) + .unwrap_or_default(), + )?) + .bind(request.status.power_state().is_some()) + .bind(request.observed_at_ms) + .execute(&mut *tx) + .await + .map_err(|error| sql_error("observe environment", error))?; + tx.commit() + .await + .map_err(|error| sql_error("commit environment observation", error))?; self.read_environment(&request.environment_id).await } @@ -924,14 +1136,101 @@ impl EnvironmentStore for PgStore { &self, request: FailEnvironmentLifecycle, ) -> Result { - let mut metadata = self - .read_environment(&request.environment_id) - .await? - .metadata; - metadata.insert("lifecycleError".to_owned(), request.message); - sqlx::query("UPDATE environments SET status='failed', metadata_json=$3, updated_at_ms=$4 WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(json_value("encode environment metadata", &metadata)?).bind(request.observed_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("fail environment", error))?; + let current = self.read_environment(&request.environment_id).await?; + let mut candidate = current.clone(); + request.apply(&mut candidate)?; + candidate.validate()?; + let Some(error) = candidate + .lifecycle_error + .as_ref() + .filter(|_| candidate != current) + else { + return Ok(current); + }; + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin environment failure", error))?; + let failed = sqlx::query( + r#" + UPDATE environments SET + status = 'failed', + lifecycle_error = $3, + lifecycle_error_first_at_ms = LEAST(COALESCE(lifecycle_error_first_at_ms, $4), $4), + lifecycle_error_last_at_ms = $4, + metadata_json = metadata_json || $5, + updated_at_ms = GREATEST(updated_at_ms, $4) + WHERE universe_id = $1 AND environment_id = $2 + AND status NOT IN ('closing', 'closed') + "#, + ) + .bind(self.config.universe_id) + .bind(request.environment_id.as_str()) + .bind(&error.message) + .bind(request.observed_at_ms) + .bind(json_value( + "encode daemon build metadata", + &request + .daemon_build + .as_ref() + .map(|build| build.metadata()) + .unwrap_or_default(), + )?) + .execute(&mut *tx) + .await + .map_err(|error| sql_error("fail environment", error))?; + if failed.rows_affected() == 1 { + // Observations older than the failure are stale from now on. + sqlx::query( + "UPDATE environment_incarnations SET updated_at_ms = GREATEST(updated_at_ms, $4) WHERE universe_id = $1 AND environment_id = $2 AND incarnation_id = $3", + ) + .bind(self.config.universe_id) + .bind(request.environment_id.as_str()) + .bind(current.incarnation.incarnation_id.as_str()) + .bind(request.observed_at_ms) + .execute(&mut *tx) + .await + .map_err(|error| sql_error("fail environment incarnation", error))?; + } + tx.commit() + .await + .map_err(|error| sql_error("commit environment failure", error))?; + self.read_environment(&request.environment_id).await + } + + async fn record_environment_lifecycle_error( + &self, + request: RecordEnvironmentLifecycleError, + ) -> Result { + let current = self.read_environment(&request.environment_id).await?; + let mut candidate = current.clone(); + request.apply(&mut candidate)?; + candidate.validate()?; + let Some(error) = candidate + .lifecycle_error + .as_ref() + .filter(|_| candidate != current) + else { + return Ok(current); + }; + sqlx::query( + r#" + UPDATE environments SET + lifecycle_error = $3, + lifecycle_error_first_at_ms = LEAST(COALESCE(lifecycle_error_first_at_ms, $4), $4), + lifecycle_error_last_at_ms = $4, + updated_at_ms = GREATEST(updated_at_ms, $4) + WHERE universe_id = $1 AND environment_id = $2 AND status <> 'closed' + "#, + ) + .bind(self.config.universe_id) + .bind(request.environment_id.as_str()) + .bind(&error.message) + .bind(request.observed_at_ms) + .execute(&self.pool) + .await + .map_err(|error| sql_error("record environment lifecycle error", error))?; self.read_environment(&request.environment_id).await } @@ -976,29 +1275,45 @@ impl EnvironmentStore for PgStore { if !matches!(environment.source, EnvironmentSource::Provisioned { .. }) { return invalid_store("provider-managed ingress requires a provisioned environment"); } - if request.enabled - && matches!( - environment.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) - { + if request.enabled && environment.status.is_closing_or_closed() { return invalid_store("cannot enable ingress for a closing environment"); } - if request.enabled != request.public_endpoint.is_some() { - return invalid_store( - "enabled ingress requires a public endpoint and disabled ingress forbids one", - ); + EnvironmentRecord { + public_ingress_enabled: request.enabled, + ..environment } - sqlx::query("UPDATE environments SET public_ingress_enabled=$3, public_endpoint=$4, updated_at_ms=GREATEST(updated_at_ms,$5) WHERE universe_id=$1 AND environment_id=$2") + .validate()?; + // Enabling never revives a close accepted in the meantime. + sqlx::query("UPDATE environments SET public_ingress_enabled=$3, updated_at_ms=GREATEST(updated_at_ms,$4) WHERE universe_id=$1 AND environment_id=$2 AND (NOT $3 OR status NOT IN ('closing','closed'))") .bind(self.config.universe_id) .bind(request.environment_id.as_str()) .bind(request.enabled) - .bind(request.public_endpoint) .bind(request.updated_at_ms) .execute(&self.pool).await.map_err(|error| sql_error("set environment ingress", error))?; self.read_environment(&request.environment_id).await } + async fn record_environment_ingress( + &self, + request: RecordEnvironmentIngress, + ) -> Result { + let current = self.read_environment(&request.environment_id).await?; + let mut candidate = current.clone(); + request.apply(&mut candidate)?; + candidate.validate()?; + if candidate == current { + return Ok(current); + } + // A close accepted while this observation was in flight wins. + sqlx::query("UPDATE environments SET public_endpoint=$3, updated_at_ms=GREATEST(updated_at_ms,$4) WHERE universe_id=$1 AND environment_id=$2 AND status NOT IN ('closing','closed')") + .bind(self.config.universe_id) + .bind(request.environment_id.as_str()) + .bind(request.public_endpoint) + .bind(request.observed_at_ms) + .execute(&self.pool).await.map_err(|error| sql_error("record environment ingress", error))?; + self.read_environment(&request.environment_id).await + } + async fn set_environment_power( &self, request: SetEnvironmentPower, @@ -1006,6 +1321,11 @@ impl EnvironmentStore for PgStore { validate_ingress_request_time(request.updated_at_ms)?; let environment = self.read_environment(&request.environment_id).await?; check_power_mutable(&environment)?; + EnvironmentRecord { + desired_power: request.desired_power, + ..environment + } + .validate()?; sqlx::query("UPDATE environments SET desired_power=$3, updated_at_ms=GREATEST(updated_at_ms,$4) WHERE universe_id=$1 AND environment_id=$2") .bind(self.config.universe_id) .bind(request.environment_id.as_str()) @@ -1025,6 +1345,11 @@ impl EnvironmentStore for PgStore { } let environment = self.read_environment(&request.environment_id).await?; check_power_mutable(&environment)?; + EnvironmentRecord { + idle_policy: request.idle_policy.clone(), + ..environment + } + .validate()?; sqlx::query("UPDATE environments SET idle_policy_json=$3, updated_at_ms=GREATEST(updated_at_ms,$4) WHERE universe_id=$1 AND environment_id=$2") .bind(self.config.universe_id) .bind(request.environment_id.as_str()) @@ -1045,10 +1370,7 @@ fn check_power_mutable(record: &EnvironmentRecord) -> Result<(), EnvironmentRegi if !matches!(record.source, EnvironmentSource::Provisioned { .. }) { return invalid_store("only provisioned environments have power control"); } - if matches!( - record.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed | EnvironmentStatus::Failed - ) { + if record.status.is_closing_or_closed() || record.status == EnvironmentStatus::Failed { return invalid_store("cannot change power of a closing, closed, or failed environment"); } Ok(()) @@ -1101,7 +1423,11 @@ impl EnvironmentCredentialStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list environment credentials", error))?; - rows.iter().map(credential_from_row).collect() + Ok(decode_rows( + &rows, + credential_from_row, + "environment credential", + )) } async fn unbind_credential( @@ -1153,15 +1479,31 @@ fn provider_from_row( row: &sqlx::postgres::PgRow, ) -> Result { let provider_id = parse_id(row, "provider_id", EnvironmentProviderId::try_new)?; - let connection: environments::EnvironmentConnectionSpec = - json_column(row, "controller_connection_json")?; + let adopt_target: Option = row + .try_get("controller_adopt_target") + .map_err(|e| sql_error("decode provider controller adoptTarget", e))?; + let ingress: Option = row + .try_get("controller_ingress") + .map_err(|e| sql_error("decode provider controller ingress", e))?; + let controller_features = match (adopt_target, ingress) { + (Some(adopt_target), Some(ingress)) => Some(ControllerFeatures { + adopt_target, + ingress, + }), + (None, None) => None, + _ => return Err(store_message("provider controller features are incomplete")), + }; let record = EnvironmentProviderRecord { provider_id, display_name: row .try_get("display_name") .map_err(|e| sql_error("decode provider display_name", e))?, - controller_connection: connection, + controller_endpoint: parse_id(row, "controller_endpoint", |value| { + EnvironmentEndpoint::parse(&value) + })?, metadata: json_column(row, "metadata_json")?, + controller_features, + data_route_base: optional_id(row, "data_route_base", DataRouteBase::new)?, created_at_ms: scalar(row, "created_at_ms")?, updated_at_ms: scalar(row, "updated_at_ms")?, }; @@ -1210,7 +1552,9 @@ fn environment_from_row( .map_err(|e| store_message(format!("decode binding id: {e}")))?, }, "external" => EnvironmentSource::External { - connection: json_column(row, "daemon_connection_json")?, + endpoint: parse_id(row, "daemon_endpoint", |value| { + EnvironmentEndpoint::parse(&value) + })?, }, "registered" => EnvironmentSource::Registered { registration_key_id: parse_id( @@ -1228,9 +1572,22 @@ fn environment_from_row( ))); } }; - let target: Option = row - .try_get("provider_target_id") - .map_err(|e| sql_error("decode provider target", e))?; + let lifecycle_error = match ( + row.try_get::, _>("lifecycle_error") + .map_err(|e| sql_error("decode lifecycle error", e))?, + row.try_get::, _>("lifecycle_error_first_at_ms") + .map_err(|e| sql_error("decode lifecycle error start", e))?, + row.try_get::, _>("lifecycle_error_last_at_ms") + .map_err(|e| sql_error("decode lifecycle error time", e))?, + ) { + (Some(message), Some(first_at_ms), Some(last_at_ms)) => Some(EnvironmentLifecycleError { + message, + first_at_ms, + last_at_ms, + }), + (None, None, None) => None, + _ => return Err(store_message("environment lifecycle error is incomplete")), + }; let record = EnvironmentRecord { environment_id: parse_id(row, "environment_id", EnvironmentId::try_new)?, request_id: parse_id(row, "request_id", EnvironmentProvisionRequestId::try_new)?, @@ -1248,7 +1605,7 @@ fn environment_from_row( "provision_request_id", EnvironmentProvisionRequestId::try_new, )?, - provider_target_id: target.map(ProviderTargetId::new), + provider_target_id: optional_id(row, "provider_target_id", ProviderTargetId::new)?, template_id: optional_id(row, "template_id", EnvironmentTemplateId::try_new)?, adoption_source_target: row .try_get("adoption_source_target") @@ -1267,6 +1624,7 @@ fn environment_from_row( last_seen_at_ms: row .try_get("last_seen_at_ms") .map_err(|e| sql_error("decode last seen", e))?, + lifecycle_error, created_at_ms: scalar(row, "created_at_ms")?, updated_at_ms: scalar(row, "updated_at_ms")?, }; @@ -1550,6 +1908,28 @@ fn map_provider_delete_error(error: sqlx::Error) -> EnvironmentRegistryError { #[allow(dead_code)] async fn _transaction_marker(_: &mut Transaction<'_, Postgres>) {} +/// Decode every row that still decodes. A row that does not is left out +/// and logged, so one bad row never hides the rest of a listing or stalls +/// a reconciler pass over them. +pub(crate) fn decode_rows( + rows: &[sqlx::postgres::PgRow], + decode: impl Fn(&sqlx::postgres::PgRow) -> Result, + kind: &'static str, +) -> Vec { + rows.iter() + .filter_map(|row| decoded_or_logged(decode(row), kind)) + .collect() +} + +fn decoded_or_logged( + result: Result, + kind: &'static str, +) -> Option { + result + .inspect_err(|error| tracing::error!(%error, kind, "skipping undecodable registry row")) + .ok() +} + /// A string map as the jsonb object the `@>` containment filter compares /// against; building the value directly cannot fail. fn metadata_filter_json( diff --git a/crates/store-pg/src/environment_registration.rs b/crates/store-pg/src/environment_registration.rs index 0aa704097..eb0d568c7 100644 --- a/crates/store-pg/src/environment_registration.rs +++ b/crates/store-pg/src/environment_registration.rs @@ -15,7 +15,9 @@ use uuid::Uuid; use crate::{ PgStore, PgStoreError, - environment::{identity_mode_from_str, invalid, not_found, sql_error, store_message}, + environment::{ + decode_rows, identity_mode_from_str, invalid, not_found, sql_error, store_message, + }, }; const KEY_COLUMNS: &str = r#" @@ -99,7 +101,11 @@ impl EnvironmentRegistrationKeyStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list registration keys", error))?; - rows.iter().map(key_from_row).collect() + Ok(decode_rows( + &rows, + key_from_row, + "environment registration key", + )) } async fn revoke_registration_key( diff --git a/crates/store-pg/src/lib.rs b/crates/store-pg/src/lib.rs index 8a2b44a43..b944ffbc5 100644 --- a/crates/store-pg/src/lib.rs +++ b/crates/store-pg/src/lib.rs @@ -388,21 +388,24 @@ pub async fn list_universes(pool: &PgPool) -> Result)> /// Universes with lifecycle intents requiring provider reconciliation. pub async fn list_universes_with_pending_environments( pool: &PgPool, + scope: environments::ReconcileScope, ) -> Result, PgStoreError> { - // Include pending provider work and registered-environment heartbeat repair. - let rows: Vec<(Uuid,)> = sqlx::query_as( + // Pending provider work, registered-environment heartbeat repair and, + // for a steady pass, every steady provisioned environment. + let converging = environment::CONVERGING_SQL; + let steady = match scope { + environments::ReconcileScope::Converging => String::new(), + environments::ReconcileScope::WithSteady => format!(" OR {}", environment::STEADY_SQL), + }; + let query = format!( "SELECT DISTINCT e.universe_id FROM environments e \ - WHERE e.status IN ('provisioning','booting','closing','unknown') \ - OR (e.source_kind = 'registered' AND e.status NOT IN ('closing','closed')) \ - OR (e.source_kind = 'provisioned' AND ( \ - (e.status = 'ready' AND e.desired_power <> 'running') \ - OR (e.status = 'paused' AND e.desired_power <> 'paused') \ - OR (e.status = 'suspended' AND e.desired_power <> 'suspended') \ - OR (e.status = 'offline' AND e.desired_power <> 'stopped'))) \ - ORDER BY e.universe_id", - ) - .fetch_all(pool) - .await?; + WHERE {converging} \ + OR (e.source_kind = 'registered' AND e.status NOT IN ('closing','closed')){steady} \ + ORDER BY e.universe_id" + ); + let rows: Vec<(Uuid,)> = sqlx::query_as(sqlx::AssertSqlSafe(query)) + .fetch_all(pool) + .await?; Ok(rows.into_iter().map(|(universe_id,)| universe_id).collect()) } diff --git a/crates/store-pg/src/migrations.rs b/crates/store-pg/src/migrations.rs index d0f3bf499..63eafdc32 100644 --- a/crates/store-pg/src/migrations.rs +++ b/crates/store-pg/src/migrations.rs @@ -113,9 +113,14 @@ pub const MIGRATIONS: &[EmbeddedMigration] = &[ name: "session_deletion", sql: include_str!("../migrations/011_session_deletion.sql"), }, + EmbeddedMigration { + version: 12, + name: "environment_endpoints", + sql: include_str!("../migrations/012_environment_endpoints.sql"), + }, ]; -pub const REQUIRED_SCHEMA_REVISION: i64 = 11; +pub const REQUIRED_SCHEMA_REVISION: i64 = 12; #[derive(Clone, Debug, PartialEq, Eq)] pub struct SchemaStatus { diff --git a/crates/store-pg/tests/store_pg_live.rs b/crates/store-pg/tests/store_pg_live.rs index 763041077..654475bf0 100644 --- a/crates/store-pg/tests/store_pg_live.rs +++ b/crates/store-pg/tests/store_pg_live.rs @@ -7,14 +7,14 @@ use auth::{ OAuthClientStore, PutSecretRecord, SECRET_KIND_STATIC_BEARER, SecretId, SecretStore, SecretValue, TokenEndpointAuthMethod, state_hash, }; -use environment_protocol::shared::{EnvironmentTransport, ProviderTargetId}; +use environment_protocol::{ids::ProviderTargetId, routes::DataRouteBase}; use environments::{ - BeginCloseEnvironment, CreateEnvironment, EnvironmentConnectionSpec, EnvironmentId, - EnvironmentIncarnationId, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, - EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, - EnvironmentProvisionRequestId, EnvironmentStatus, EnvironmentStore, EnvironmentTemplateId, - ListEnvironmentProviders, ListEnvironments, ObserveProvisionedEnvironment, - PutEnvironmentProvider, PutEnvironmentProviderBinding, + BeginCloseEnvironment, ControllerFeatures, CreateEnvironment, EnvironmentEndpoint, + EnvironmentId, EnvironmentIncarnationId, EnvironmentProviderBindingId, + EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, + EnvironmentProviderStore, EnvironmentProvisionRequestId, EnvironmentStatus, EnvironmentStore, + EnvironmentTemplateId, ListEnvironmentProviders, ListEnvironments, + ObserveProvisionedEnvironment, PutEnvironmentProvider, PutEnvironmentProviderBinding, }; use harness::{ BlobRef, CORE_AGENT_LIFECYCLE_CLOSED_EVENT_KIND, @@ -2009,7 +2009,7 @@ async fn pg_live_universe_environments_are_independent_of_sessions() { ); } let provider_id = EnvironmentProviderId::new("bridge-local"); - let target_id = ProviderTargetId::new("local-host"); + let target_id = ProviderTargetId::new("local-host").expect("target id"); let environment_id = EnvironmentId::new("instance-local"); let session_id = SessionId::new("session-env"); @@ -2017,11 +2017,10 @@ async fn pg_live_universe_environments_are_independent_of_sessions() { .put_provider(PutEnvironmentProvider { provider_id: provider_id.clone(), display_name: Some("Local bridge".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "ws://127.0.0.1:9000/controller", - EnvironmentTransport::WebSocket, - ), + controller_endpoint: EnvironmentEndpoint::parse("ws://127.0.0.1:9000/controller") + .expect("endpoint"), metadata: Default::default(), + credential: Some(SecretValue::new("bridge-credential")), updated_at_ms: 10, }) .await @@ -2072,7 +2071,8 @@ async fn pg_live_universe_environments_are_independent_of_sessions() { let instance = store .observe_provisioned_environment(ObserveProvisionedEnvironment { environment_id: environment_id.clone(), - provider_target_id: target_id.clone(), + provider_target_id: Some(target_id.clone()), + daemon_build: None, status: EnvironmentStatus::Offline, power_states: vec![ environments::PowerState::Running, @@ -2104,7 +2104,7 @@ async fn pg_live_universe_environments_are_independent_of_sessions() { // Offline while desired running: the reconciler must pick it up. assert!( store - .list_environments_needing_reconcile() + .list_environments_needing_reconcile(environments::ReconcileScope::Converging) .await .expect("needing reconcile") .iter() @@ -2132,7 +2132,16 @@ async fn pg_live_universe_environments_are_independent_of_sessions() { ); assert!( !store - .list_environments_needing_reconcile() + .list_environments_needing_reconcile(environments::ReconcileScope::Converging) + .await + .expect("needing reconcile") + .iter() + .any(|record| record.environment_id == environment_id) + ); + // A steady pass observes it anyway. + assert!( + store + .list_environments_needing_reconcile(environments::ReconcileScope::WithSteady) .await .expect("needing reconcile") .iter() @@ -2141,7 +2150,8 @@ async fn pg_live_universe_environments_are_independent_of_sessions() { let ready = store .observe_provisioned_environment(ObserveProvisionedEnvironment { environment_id: environment_id.clone(), - provider_target_id: target_id.clone(), + provider_target_id: Some(target_id.clone()), + daemon_build: None, status: EnvironmentStatus::Ready, power_states: vec![environments::PowerState::Running], observed_at_ms: 32, @@ -2175,7 +2185,8 @@ async fn pg_live_universe_environments_are_independent_of_sessions() { let instance = store .observe_provisioned_environment(ObserveProvisionedEnvironment { environment_id: environment_id.clone(), - provider_target_id: target_id.clone(), + provider_target_id: Some(target_id.clone()), + daemon_build: None, status: EnvironmentStatus::Offline, power_states: vec![environments::PowerState::Running], observed_at_ms: 35, @@ -2785,6 +2796,211 @@ fn create_oauth_client_record(client_id: &OAuthClientId) -> CreateOAuthClientRec } } +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] +async fn pg_live_environment_lifecycle_guards_and_provider_facts() { + use environments::{ + FailEnvironmentLifecycle, RecordEnvironmentLifecycleError, RecordProviderController, + }; + + let store = live_store("environment-guards", 1024).await; + let suffix = Uuid::new_v4().simple().to_string(); + let provider_id = EnvironmentProviderId::new(format!("guard-provider-{suffix}")); + let endpoint = EnvironmentEndpoint::parse("ws://127.0.0.1:9100/control").expect("endpoint"); + + // The credential is required for a new provider, sealed at rest, and + // kept by a later put that omits it. + let refused = store + .put_provider(PutEnvironmentProvider { + provider_id: provider_id.clone(), + display_name: None, + controller_endpoint: endpoint.clone(), + metadata: Default::default(), + credential: None, + updated_at_ms: 1, + }) + .await; + assert!(matches!( + refused, + Err(environments::EnvironmentRegistryError::InvalidInput { .. }) + )); + store + .put_provider(PutEnvironmentProvider { + provider_id: provider_id.clone(), + display_name: None, + controller_endpoint: endpoint.clone(), + metadata: Default::default(), + credential: Some(SecretValue::new("guard-bearer-secret")), + updated_at_ms: 1, + }) + .await + .expect("register provider"); + let ciphertext: Vec = sqlx::query_scalar( + "SELECT credential_ciphertext FROM environment_providers WHERE provider_id = $1", + ) + .bind(provider_id.as_str()) + .fetch_one(store.pool()) + .await + .expect("read sealed credential"); + assert!( + !ciphertext + .windows(b"guard-bearer-secret".len()) + .any(|window| window == b"guard-bearer-secret") + ); + let base = DataRouteBase::new("ws://127.0.0.1:9100/data").expect("base"); + let recorded = store + .record_provider_controller(RecordProviderController { + provider_id: provider_id.clone(), + controller_endpoint: endpoint.clone(), + features: ControllerFeatures { + adopt_target: false, + ingress: true, + }, + data_route_base: base.clone(), + }) + .await + .expect("record controller"); + assert_eq!(recorded.data_route_base, Some(base)); + let renamed = store + .put_provider(PutEnvironmentProvider { + provider_id: provider_id.clone(), + display_name: Some("Guard".to_owned()), + controller_endpoint: endpoint.clone(), + metadata: Default::default(), + credential: None, + updated_at_ms: 2, + }) + .await + .expect("rename provider"); + assert_eq!(renamed.controller_features, recorded.controller_features); + assert_eq!( + store + .read_provider_credential(&provider_id) + .await + .expect("credential") + .expose(), + "guard-bearer-secret" + ); + + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id: store.config().universe_id, + binding_id: EnvironmentProviderBindingId::new("primary"), + provider_id: provider_id.clone(), + status: EnvironmentProviderBindingStatus::Enabled, + metadata: Default::default(), + expected_revision: None, + updated_at_ms: 3, + }) + .await + .expect("provider binding"); + let create = |request: &str, environment: &str| CreateEnvironment { + request_id: EnvironmentProvisionRequestId::new(request), + environment_id: EnvironmentId::new(environment), + incarnation_id: EnvironmentIncarnationId::new(format!("{environment}-inc")), + binding_id: EnvironmentProviderBindingId::new("primary"), + template_id: EnvironmentTemplateId::new("rust-v1"), + display_name: None, + metadata: Default::default(), + idle_policy: None, + created_at_ms: 10, + }; + + // A close accepted while a create was in flight wins over its answer, + // and the target id is still recorded so the close can address it. + let closing = store + .create_environment(create("guard-request", "guard-env")) + .await + .expect("create"); + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: closing.environment_id.clone(), + updated_at_ms: 11, + }) + .await + .expect("begin close"); + let natural = ProviderTargetId::new("default/pod-abc").expect("opaque target id"); + let observed = store + .observe_provisioned_environment(ObserveProvisionedEnvironment { + environment_id: closing.environment_id.clone(), + provider_target_id: Some(natural.clone()), + status: EnvironmentStatus::Ready, + power_states: vec![environments::PowerState::Running], + daemon_build: None, + observed_at_ms: 12, + }) + .await + .expect("observe"); + assert_eq!(observed.status, EnvironmentStatus::Closing); + assert_eq!(observed.incarnation.provider_target_id, Some(natural)); + let failed = store + .fail_environment_lifecycle(FailEnvironmentLifecycle { + environment_id: closing.environment_id.clone(), + message: "late failure".to_owned(), + daemon_build: None, + observed_at_ms: 13, + }) + .await + .expect("fail"); + assert_eq!(failed.status, EnvironmentStatus::Closing); + + // Lifecycle errors keep the start of their run and leave the status. + let erroring = store + .create_environment(create("guard-request-2", "guard-env-2")) + .await + .expect("create second"); + for at in [20, 30] { + store + .record_environment_lifecycle_error(RecordEnvironmentLifecycleError { + environment_id: erroring.environment_id.clone(), + message: format!("controller unavailable at {at}"), + observed_at_ms: at, + }) + .await + .expect("record error"); + } + let still = store + .read_environment(&erroring.environment_id) + .await + .expect("read"); + assert_eq!(still.status, EnvironmentStatus::Provisioning); + let error = still.lifecycle_error.expect("lifecycle error"); + assert_eq!((error.first_at_ms, error.last_at_ms), (20, 30)); + + // A row that no longer decodes is skipped by listings instead of + // failing them. + let poisoned = store + .create_environment(create("guard-request-3", "guard-env-3")) + .await + .expect("create third"); + let oversized: serde_json::Map = (0..40) + .map(|index| (format!("key{index}"), serde_json::json!("v"))) + .collect(); + sqlx::query( + "UPDATE environments SET metadata_json = $3 WHERE universe_id = $1 AND environment_id = $2", + ) + .bind(store.config().universe_id) + .bind(poisoned.environment_id.as_str()) + .bind(serde_json::Value::Object(oversized)) + .execute(store.pool()) + .await + .expect("poison row"); + let listed = store + .list_environments(ListEnvironments::default()) + .await + .expect("list despite a bad row"); + assert!( + listed + .iter() + .all(|record| record.environment_id != poisoned.environment_id) + ); + assert!( + listed + .iter() + .any(|record| record.environment_id == erroring.environment_id) + ); +} + #[tokio::test(flavor = "current_thread")] #[ignore = "requires ./dev.sh infra or compatible Postgres env"] async fn pg_live_environment_credentials_round_trip() { @@ -2800,11 +3016,10 @@ async fn pg_live_environment_credentials_round_trip() { .put_provider(PutEnvironmentProvider { provider_id: provider_id.clone(), display_name: None, - controller_connection: EnvironmentConnectionSpec::new( - "ws://127.0.0.1:9000/controller", - EnvironmentTransport::WebSocket, - ), + controller_endpoint: EnvironmentEndpoint::parse("ws://127.0.0.1:9000/controller") + .expect("endpoint"), metadata: Default::default(), + credential: Some(SecretValue::new("cred-provider-credential")), updated_at_ms: 1, }) .await diff --git a/crates/temporal-runtime/Cargo.toml b/crates/temporal-runtime/Cargo.toml index 32c74dd9c..c6ffa87fc 100644 --- a/crates/temporal-runtime/Cargo.toml +++ b/crates/temporal-runtime/Cargo.toml @@ -63,6 +63,7 @@ url = "2" [dev-dependencies] test-support = { path = "../test-support" } +environment-client = { path = "../environment-client", features = ["conformance"] } dotenvy = "0.15" environment-daemon = { path = "../environment-daemon" } futures-util = "0.3" diff --git a/crates/temporal-runtime/src/bots/fires.rs b/crates/temporal-runtime/src/bots/fires.rs index 48d5aa120..e34cd051b 100644 --- a/crates/temporal-runtime/src/bots/fires.rs +++ b/crates/temporal-runtime/src/bots/fires.rs @@ -363,10 +363,7 @@ pub(crate) fn exec_request_id(trigger_id: &BotTriggerId, scheduled_at_ms: i64) - pub(crate) fn is_terminal_job_status(status: SessionJobStatusView) -> bool { !matches!( status, - SessionJobStatusView::Accepted - | SessionJobStatusView::Queued - | SessionJobStatusView::Running - | SessionJobStatusView::CancelRequested + SessionJobStatusView::Queued | SessionJobStatusView::Running ) } @@ -808,16 +805,16 @@ async fn fetch_exec_payload( }; let deadline = Instant::now() + Duration::from_millis(budget_ms.saturating_add(EXEC_DEADLINE_SLACK_MS)); - let mut after_seq = None; + let mut after = None; let mut stdout: Vec = Vec::new(); let mut stderr_tail: Vec = Vec::new(); loop { let read = api .read_environment_jobs(EnvironmentJobReadParams { jobs: vec![handle.clone()], - output_bytes: None, - after_seq, - include_artifacts: false, + max_bytes: None, + after, + tail_bytes: None, }) .await .map_err(|error| map_job_api_error("read poll command", error))?; @@ -842,11 +839,13 @@ async fn fetch_exec_payload( } } } - if stdout.len() > MAX_POLL_PAYLOAD_BYTES { + // A gap means the environment dropped output it could not retain, + // so stdout can no longer be a complete payload. + if stdout.len() > MAX_POLL_PAYLOAD_BYTES || !entry.output_gaps.is_empty() { cancel_job(api, &handle).await; return Err(payload_too_large("stdout")); } - after_seq = Some(entry.output_next_seq); + after = Some(entry.output_next); if let Some(summary) = entry.summary && is_terminal_job_status(summary.status) { @@ -1172,12 +1171,7 @@ mod tests { #[test] fn job_statuses_split_into_running_and_terminal() { - for status in [ - SessionJobStatusView::Accepted, - SessionJobStatusView::Queued, - SessionJobStatusView::Running, - SessionJobStatusView::CancelRequested, - ] { + for status in [SessionJobStatusView::Queued, SessionJobStatusView::Running] { assert!(!is_terminal_job_status(status), "{status:?}"); } for status in [ @@ -1187,7 +1181,6 @@ mod tests { SessionJobStatusView::TimedOut, SessionJobStatusView::DependencyFailed, SessionJobStatusView::Interrupted, - SessionJobStatusView::Lost, ] { assert!(is_terminal_job_status(status), "{status:?}"); } diff --git a/crates/temporal-runtime/src/config.rs b/crates/temporal-runtime/src/config.rs index 930632c73..9ee9f059f 100644 --- a/crates/temporal-runtime/src/config.rs +++ b/crates/temporal-runtime/src/config.rs @@ -41,24 +41,6 @@ pub enum GatewayAuthMode { Authenticated, } -/// Optional base URL outbound daemons are told to dial for data connections -/// when it differs from the general public base URL (for example a dedicated -/// hostname in front of the environment gateway). -pub fn environment_public_url_from_env() -> anyhow::Result> { - let Some(value) = optional_env("LIGHTSPEED_ENVIRONMENT_PUBLIC_URL") else { - return Ok(None); - }; - if !["https://", "http://", "wss://", "ws://"] - .iter() - .any(|scheme| value.starts_with(scheme)) - { - anyhow::bail!( - "LIGHTSPEED_ENVIRONMENT_PUBLIC_URL must be an http(s) or ws(s) URL, got {value:?}" - ); - } - Ok(Some(value.trim_end_matches('/').to_owned())) -} - pub fn gateway_auth_mode_from_env() -> anyhow::Result { if optional_env("LIGHTSPEED_UNIVERSE_AUTO_CREATE").is_some() { anyhow::bail!( diff --git a/crates/temporal-runtime/src/credential_injection.rs b/crates/temporal-runtime/src/credential_injection.rs index a12c20ad4..593b11513 100644 --- a/crates/temporal-runtime/src/credential_injection.rs +++ b/crates/temporal-runtime/src/credential_injection.rs @@ -8,8 +8,14 @@ use auth::{ RegistryTokenBroker, SecretStore, TokenAudience, }; use environment_protocol::{ - data::jobs::{StartJobsParams, StartJobsResponse}, - shared::SecretString, + bytes::SecretString, + data::{ + handshake::JobLimits, + job::{ + CancelJobsParams, CancelJobsResponse, ReadJobsParams, ReadJobsResponse, + StartJobsParams, StartJobsResponse, + }, + }, }; use environments::{ EnvironmentCredentialSource, EnvironmentCredentialStore, EnvironmentId, @@ -245,33 +251,29 @@ struct CredentialInjectingJobExecutor { #[async_trait] impl JobExecutor for CredentialInjectingJobExecutor { + fn limits(&self) -> Option { + self.inner.limits() + } + async fn start_jobs(&self, mut request: StartJobsParams) -> JobExecResult { for job in &mut request.jobs { let secret_env = self .resolver - .resolve_secret_env(&self.environment_id, &job.env) + .resolve_secret_env(&self.environment_id, &job.command.env) .await .map_err(|error| JobError::InvalidRequest { message: error.to_string(), })?; - for (name, value) in secret_env { - job.secret_env.insert(name, value); - } + job.command.secret_env.extend(secret_env); } self.inner.start_jobs(request).await } - async fn read_jobs( - &self, - request: environment_protocol::data::jobs::ReadJobsParams, - ) -> JobExecResult { + async fn read_jobs(&self, request: ReadJobsParams) -> JobExecResult { self.inner.read_jobs(request).await } - async fn cancel_jobs( - &self, - request: environment_protocol::data::jobs::CancelJobsParams, - ) -> JobExecResult { + async fn cancel_jobs(&self, request: CancelJobsParams) -> JobExecResult { self.inner.cancel_jobs(request).await } } @@ -411,6 +413,114 @@ mod tests { assert_eq!(resolved["GH_TOKEN"].expose(), "environment-token"); } + #[derive(Default)] + struct RecordingJobExecutor { + started: std::sync::Mutex>, + } + + #[async_trait] + impl JobExecutor for RecordingJobExecutor { + fn limits(&self) -> Option { + Some(JobLimits { + max_read_bytes: 1024, + max_request_jobs: 4, + max_retained_output_bytes: 2048, + record_retention_ms: 1, + cancel_grace_ms: 1, + }) + } + + async fn start_jobs(&self, request: StartJobsParams) -> JobExecResult { + self.started.lock().unwrap().push(request); + Err(JobError::Failed { + message: "recorded".to_owned(), + }) + } + + async fn read_jobs(&self, _request: ReadJobsParams) -> JobExecResult { + unreachable!("not read") + } + + async fn cancel_jobs( + &self, + _request: CancelJobsParams, + ) -> JobExecResult { + unreachable!("not cancelled") + } + } + + #[tokio::test(flavor = "current_thread")] + async fn job_wrapper_injects_credentials_into_each_command_and_keeps_limits() { + let environment_id = EnvironmentId::new("environment_1"); + let secret_id = SecretId::new("secret_1"); + let secrets = Arc::new(InMemorySecretStore::new()); + secrets + .put_secret(PutSecretRecord { + secret_id: secret_id.clone(), + secret_kind: SECRET_KIND_STATIC_BEARER.to_owned(), + value: SecretValue::new("environment-token"), + created_at_ms: 1, + }) + .await + .expect("store secret"); + let resolver = EnvironmentCredentialResolver::new( + Arc::new(FixedCredentialStore { + credentials: vec![EnvironmentCredentialRecord { + environment_id: environment_id.clone(), + env_name: "GH_TOKEN".to_owned(), + source: EnvironmentCredentialSource::DirectSecret { secret_id }, + created_at_ms: 1, + updated_at_ms: 1, + }], + }), + Arc::new(InMemoryAuthGrantStore::new()), + Arc::new(InMemoryAuthProviderStore::new()), + secrets, + None, + ); + let inner = Arc::new(RecordingJobExecutor::default()); + let wrapper = CredentialInjectingJobExecutor { + inner: inner.clone(), + resolver, + environment_id, + }; + assert_eq!( + wrapper.limits().map(|limits| limits.max_request_jobs), + Some(4) + ); + + let job = |job_id: &str| environment_protocol::data::job::JobSpec { + job_id: environment_protocol::ids::JobId::new(job_id).unwrap(), + name: None, + command: environment_protocol::data::exec::CommandSpec { + argv: vec!["true".to_owned()], + cwd: None, + env: BTreeMap::from([("CI".to_owned(), "1".to_owned())]), + secret_env: BTreeMap::new(), + input: Default::default(), + timeout_ms: None, + }, + depends_on: Vec::new(), + dependency_policy: Default::default(), + queue_key: None, + }; + let _ = wrapper + .start_jobs(StartJobsParams { + request_id: environment_protocol::ids::JobRequestId::new("req-1").unwrap(), + jobs: vec![job("a"), job("b")], + }) + .await; + + let started = inner.started.lock().unwrap(); + for job in &started[0].jobs { + assert_eq!( + job.command.secret_env["GH_TOKEN"].expose(), + "environment-token" + ); + assert_eq!(job.command.env["CI"], "1"); + } + } + #[test] fn default_grant_audience_is_environment_scoped() { let grant = AuthGrantRecord { diff --git a/crates/temporal-runtime/src/environments/fake_controller.rs b/crates/temporal-runtime/src/environments/fake_controller.rs new file mode 100644 index 000000000..4879a081d --- /dev/null +++ b/crates/temporal-runtime/src/environments/fake_controller.rs @@ -0,0 +1,754 @@ +//! An in-memory provider controller for tests. +//! +//! It serves the controller plane on `/control` and a minimal data plane on +//! `/routes/{universeId}/{bindingId}/{environmentId}/{incarnationId}/{targetId}` +//! over a loopback WebSocket listener, and checks the provider credential +//! as a bearer token on both, exactly as a real provider must. Nothing +//! registers it: a test starts one and registers its controller URL and +//! credential like any provider's. +//! +//! Targets come up at once: `target/create` answers an `active`, `running` +//! target whose daemon is ready, and power changes and closes complete +//! before they answer. The data route answers `initialize` only, as an envd +//! with no capability families, which is enough for the runtime's +//! data-plane handshake at the Ready transition. + +use std::{ + collections::BTreeMap, + hash::{DefaultHasher, Hash, Hasher}, + sync::Arc, +}; + +use axum::{ + Router, + extract::{ + Path, State, + ws::{Message, WebSocket, WebSocketUpgrade}, + }, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, + routing::get, +}; +use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION, + control::{ + handshake::{ + ControllerCapabilities, ControllerInitializeParams, ControllerInitializeResponse, + ControllerRoutes, + }, + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + AdoptTargetParams, CloseTargetParams, CreateTargetParams, GetTargetParams, + IngressObservation, PowerState, SetTargetPowerParams, TargetObservation, TargetOwner, + TargetPhase, TargetRef, TargetResponse, TargetState, + }, + template::{EnvironmentTemplate, ListTemplatesResponse, TemplateResources}, + }, + data::handshake::{ + DataCapabilities, DataLimits, FilesystemLimits, GlobCeilings, InitializeParams, + InitializeResponse, JobLimits, ProcessLimits, ScanCeilings, SearchCeilings, + TransferCeilings, + }, + error::{Capability, PreconditionReason, ProtocolError, ProtocolErrorKind}, + handshake::{ImplementationInfo, Plane}, + ids::{ + BindingId, DaemonInstanceId, EnvironmentId, IncarnationId, ProviderTargetId, TemplateId, + UniverseId, + }, + jsonrpc::{self, Request, Response as RpcResponse}, + limits::{MAX_MESSAGE_BYTES, MIN_CONCURRENT_REQUESTS}, + manifest::{ + ControllerInitialize, IngressEnsure, IngressRemove, RequestMethod, TargetAdopt, + TargetClose, TargetCreate, TargetGet, TargetSetPower, TemplateList, + }, + metadata::{DescriptiveText, DisplayName, Metadata}, + paths::EnvironmentPath, + routes::{DataRouteBase, PublicUrl}, +}; +use futures_util::{SinkExt, StreamExt}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; +use tokio::{net::TcpListener, sync::Mutex, task::JoinHandle}; + +/// The one template the fake controller offers. +pub const FAKE_TEMPLATE_ID: &str = "rust-v1"; + +/// Public endpoint the fake controller reports for enabled ingress. +pub const FAKE_INGRESS_ENDPOINT: &str = "https://fake.env.test"; + +/// Power states the fake controller supports. Suspended is included so the +/// suspend path has coverage. +pub const FAKE_POWER_STATES: [PowerState; 4] = [ + PowerState::Running, + PowerState::Paused, + PowerState::Suspended, + PowerState::Stopped, +]; + +/// What the fake controller advertises. +#[derive(Clone, Debug)] +pub struct FakeControllerOptions { + /// `capabilities.adoptTarget`. + pub adopt_target: bool, + /// `capabilities.ingress`. + pub ingress: bool, + /// The protocol version the fake envd behind every data route speaks. + pub daemon_protocol_version: u32, +} + +impl Default for FakeControllerOptions { + fn default() -> Self { + Self { + adopt_target: true, + ingress: true, + daemon_protocol_version: ENVD_PROTOCOL_VERSION, + } + } +} + +/// A running fake controller. Dropping it stops the listener. +pub struct FakeController { + controller_url: String, + credential: String, + task: JoinHandle<()>, +} + +impl FakeController { + /// Serve a fake controller with the default options on a loopback port. + pub async fn start() -> std::io::Result { + Self::start_with(FakeControllerOptions::default()).await + } + + pub async fn start_with(options: FakeControllerOptions) -> std::io::Result { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let credential = format!("fake-credential-{}", uuid::Uuid::new_v4().simple()); + let data = DataRouteBase::new(format!("ws://{address}/routes")) + .map_err(|error| std::io::Error::other(error.to_string()))?; + let initialize = ControllerInitializeResponse { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + plane: Plane::Controller, + server_info: implementation("lightspeed-fake-controller"), + capabilities: ControllerCapabilities { + adopt_target: options.adopt_target, + ingress: options.ingress, + }, + routes: ControllerRoutes { data }, + }; + let app = Arc::new(App { + credential: credential.clone(), + options, + initialize, + targets: Mutex::new(Targets::default()), + }); + let router = Router::new() + .route("/control", get(control_upgrade)) + .route( + "/routes/{universe}/{binding}/{environment}/{incarnation}/{target}", + get(data_upgrade), + ) + .with_state(app); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, router).await; + }); + Ok(Self { + controller_url: format!("ws://{address}/control"), + credential, + task, + }) + } + + /// The controller URL to register the provider with. + pub fn controller_url(&self) -> &str { + &self.controller_url + } + + /// The provider credential the controller and its data routes accept. + pub fn credential(&self) -> &str { + &self.credential + } +} + +impl Drop for FakeController { + fn drop(&mut self) { + self.task.abort(); + } +} + +struct App { + credential: String, + options: FakeControllerOptions, + initialize: ControllerInitializeResponse, + targets: Mutex, +} + +#[derive(Default)] +struct Targets { + by_id: BTreeMap, + by_owner: BTreeMap, +} + +#[derive(Clone)] +struct FakeTarget { + target_id: ProviderTargetId, + owner: TargetOwner, + power: PowerState, + ingress: bool, + metadata: Metadata, +} + +impl FakeTarget { + fn observation(&self) -> TargetObservation { + TargetObservation { + target_id: Some(self.target_id.clone()), + owner: self.owner.clone(), + display_name: DisplayName::new(self.owner.environment_id.as_str()).ok(), + state: TargetState { + phase: TargetPhase::Active, + power: Some(self.power), + daemon_ready: self.power == PowerState::Running, + message: None, + }, + supported_power_states: FAKE_POWER_STATES.to_vec(), + ingress: self.ingress.then(|| IngressObservation { + endpoint: PublicUrl::new(FAKE_INGRESS_ENDPOINT).expect("valid fake endpoint"), + }), + metadata: self.metadata.clone(), + } + } +} + +fn implementation(name: &str) -> ImplementationInfo { + ImplementationInfo::new(name) + .and_then(|info| info.with_version(env!("CARGO_PKG_VERSION"))) + .expect("fake implementation info is within bounds") +} + +fn authorized(app: &App, headers: &HeaderMap) -> bool { + headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .is_some_and(|token| constant_time_eq(token.as_bytes(), app.credential.as_bytes())) +} + +fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + left.len() == right.len() + && left + .iter() + .zip(right) + .fold(0_u8, |difference, (left, right)| { + difference | (left ^ right) + }) + == 0 +} + +fn unauthorized() -> Response { + ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, "Bearer")], + ) + .into_response() +} + +fn with_limits(upgrade: WebSocketUpgrade) -> WebSocketUpgrade { + upgrade + .max_message_size(MAX_MESSAGE_BYTES as usize) + .max_frame_size(MAX_MESSAGE_BYTES as usize) +} + +async fn control_upgrade( + State(app): State>, + headers: HeaderMap, + upgrade: WebSocketUpgrade, +) -> Response { + if !authorized(&app, &headers) { + return unauthorized(); + } + with_limits(upgrade) + .on_upgrade(move |socket| serve(socket, Plane::Controller, app)) + .into_response() +} + +async fn data_upgrade( + State(app): State>, + Path((universe, binding, environment, incarnation, target_id)): Path<( + String, + String, + String, + String, + String, + )>, + headers: HeaderMap, + upgrade: WebSocketUpgrade, +) -> Response { + if !authorized(&app, &headers) { + return unauthorized(); + } + let (Ok(universe_id), Ok(binding_id), Ok(environment_id), Ok(incarnation_id), Ok(target_id)) = ( + UniverseId::new(universe), + BindingId::new(binding), + EnvironmentId::new(environment), + IncarnationId::new(incarnation), + ProviderTargetId::new(target_id), + ) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let owner = TargetOwner { + universe_id, + binding_id, + environment_id, + incarnation_id, + }; + match app.targets.lock().await.by_id.get(&target_id) { + Some(target) if target.owner != owner => return StatusCode::NOT_FOUND.into_response(), + Some(target) if target.power != PowerState::Running => { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + Some(_) => {} + None => return StatusCode::NOT_FOUND.into_response(), + } + with_limits(upgrade) + .on_upgrade(move |socket| serve(socket, Plane::Data, app)) + .into_response() +} + +/// One connection: the envelope and the `initialize`-first handshake, +/// requests answered in order. +async fn serve(socket: WebSocket, plane: Plane, app: Arc) { + let (mut sink, mut stream) = socket.split(); + let mut initialized = false; + while let Some(Ok(frame)) = stream.next().await { + let reply = match frame { + Message::Text(text) => match jsonrpc::parse(text.as_str()) { + Err(rejection) => rejection.into_response(), + Ok(jsonrpc::Message::Request(request)) => { + Some(if request.method == ControllerInitialize::NAME { + initialize(&app, plane, &mut initialized, &request) + } else if !initialized { + RpcResponse::failure(Some(request.id), ProtocolError::not_initialized()) + } else { + let result = match plane { + Plane::Controller => controller_call(&app, &request).await, + _ => Err(ProtocolError::method_not_found(&request.method)), + }; + RpcResponse { + id: Some(request.id), + result, + } + }) + } + Ok(jsonrpc::Message::Notification(_) | jsonrpc::Message::Response(_)) => None, + }, + Message::Binary(_) => Some(RpcResponse::failure( + None, + ProtocolError::invalid_request("binary frames are not part of any plane"), + )), + Message::Ping(payload) => { + if sink.send(Message::Pong(payload)).await.is_err() { + break; + } + None + } + Message::Pong(_) => None, + Message::Close(_) => break, + }; + if let Some(reply) = reply + && sink + .send(Message::Text(reply.to_text().into())) + .await + .is_err() + { + break; + } + } + let _ = sink.close().await; +} + +fn initialize(app: &App, plane: Plane, initialized: &mut bool, request: &Request) -> RpcResponse { + let id = Some(request.id.clone()); + if *initialized { + return RpcResponse::failure( + id, + ProtocolError::invalid_request("initialize was already called on this connection"), + ); + } + let expected = match plane { + Plane::Controller => CONTROLLER_PROTOCOL_VERSION, + _ => app.options.daemon_protocol_version, + }; + if let Some(version) = request + .params + .get("protocolVersion") + .and_then(Value::as_u64) + && version != u64::from(expected) + { + return RpcResponse::failure( + id, + ProtocolError::version_mismatch(expected, u32::try_from(version).unwrap_or(u32::MAX)), + ); + } + let result = match plane { + Plane::Controller => request + .params::() + .and_then(|_| encode(&app.initialize)), + _ => request + .params::() + .and_then(|_| encode(&data_initialize(expected))), + }; + if result.is_ok() { + *initialized = true; + } + RpcResponse { id, result } +} + +/// The fake envd's handshake: no capability families, so a client that +/// needs one is refused before calling anything. +fn data_initialize(protocol_version: u32) -> InitializeResponse { + InitializeResponse { + protocol_version, + plane: Plane::Data, + daemon_instance_id: DaemonInstanceId::new(format!( + "envd_{}", + uuid::Uuid::new_v4().simple() + )) + .expect("valid daemon instance id"), + server_info: implementation("lightspeed-fake-envd"), + capabilities: DataCapabilities::default(), + limits: DataLimits { + max_concurrent_requests: MIN_CONCURRENT_REQUESTS, + max_wait_ms: 60_000, + process: ProcessLimits { + max_read_bytes: 1 << 20, + max_retained_output_bytes: 1 << 20, + retention_after_exit_ms: 60_000, + unread_retention_ms: 60_000, + max_exited_retained_bytes: 1 << 20, + terminate_grace_ms: 1_000, + max_input_queue_bytes: 1 << 20, + }, + job: JobLimits { + max_read_bytes: 1 << 20, + max_request_jobs: 1, + max_retained_output_bytes: 1 << 20, + record_retention_ms: 60_000, + cancel_grace_ms: 1_000, + }, + filesystem: FilesystemLimits { + max_read_file_bytes: 1 << 20, + max_write_file_bytes: 1 << 20, + max_directory_entries: 1, + search: SearchCeilings { + max_matches: 1, + max_files: 1, + max_bytes: 1, + max_duration_ms: 1_000, + max_line_bytes: 1, + }, + glob: GlobCeilings { + max_matches: 1, + max_files: 1, + max_duration_ms: 1_000, + }, + scan: ScanCeilings { + max_roots: 1, + max_patterns: 1, + max_entries: 1, + max_file_bytes: 1, + max_total_bytes: 1, + max_duration_ms: 1_000, + }, + transfer: TransferCeilings { + max_active_operations: 1, + max_content_chunk_bytes: 1, + max_inventory_page_entries: 1, + receipt_retention_ms: 60_000, + max_entries: 1, + max_depth: 1, + max_file_bytes: 1, + max_total_bytes: 1, + max_manifest_bytes: 1, + max_duration_ms: 1_000, + }, + }, + }, + default_cwd: EnvironmentPath::new("/").expect("root path"), + home_directory: None, + } +} + +fn encode(value: &T) -> Result { + serde_json::to_value(value).map_err(|error| ProtocolError::internal(error.to_string())) +} + +fn params(request: &Request) -> Result +where + M::Params: DeserializeOwned, +{ + request.params::() +} + +async fn controller_call(app: &App, request: &Request) -> Result { + let method = request.method.as_str(); + if method == TemplateList::NAME { + params::(request)?; + return encode(&list_templates()); + } + if (method == IngressEnsure::NAME || method == IngressRemove::NAME) && !app.options.ingress { + return Err(ProtocolError::capability_unavailable( + Capability::Ingress, + "the fake controller was started without ingress", + )); + } + if method == TargetAdopt::NAME && !app.options.adopt_target { + return Err(ProtocolError::capability_unavailable( + Capability::AdoptTarget, + "the fake controller was started without adoption", + )); + } + let mut targets = app.targets.lock().await; + let observation = match method { + TargetCreate::NAME => targets.create(params::(request)?)?, + TargetAdopt::NAME => targets.adopt(params::(request)?)?, + TargetGet::NAME => { + let GetTargetParams { target } = params::(request)?; + targets.resolve(&target)?.observation() + } + TargetSetPower::NAME => targets.set_power(params::(request)?)?, + TargetClose::NAME => targets.close(params::(request)?)?, + IngressEnsure::NAME => { + let EnsureIngressParams { target } = params::(request)?; + targets.set_ingress(&target, true)? + } + IngressRemove::NAME => { + let RemoveIngressParams { target } = params::(request)?; + targets.set_ingress(&target, false)? + } + other => return Err(ProtocolError::method_not_found(other)), + }; + encode(&TargetResponse { + target: observation, + }) +} + +fn list_templates() -> ListTemplatesResponse { + ListTemplatesResponse { + templates: vec![EnvironmentTemplate { + template_id: TemplateId::new(FAKE_TEMPLATE_ID).expect("valid template id"), + display_name: DisplayName::new("Rust development").expect("valid display name"), + description: DescriptiveText::new("Deterministic fake provider template").ok(), + public_ingress: true, + deprecated: false, + resources: Some(TemplateResources { + vcpus: Some(2), + memory_bytes: Some(4 << 30), + disk_bytes: Some(20 << 30), + }), + metadata: Metadata::default(), + }], + } +} + +fn owner_mismatch() -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::OwnerMismatch, + "the target belongs to another owner", + ) +} + +impl Targets { + /// The owner's target, or the named one after the ownership fence. + fn resolve(&self, target: &TargetRef) -> Result<&FakeTarget, ProtocolError> { + let target_id = match &target.target_id { + Some(target_id) => target_id, + None => self + .by_owner + .get(&target.owner) + .ok_or_else(|| ProtocolError::not_found("the owner holds no target"))?, + }; + let found = self + .by_id + .get(target_id) + .ok_or_else(|| ProtocolError::not_found(format!("unknown target: {target_id}")))?; + if found.owner != target.owner { + return Err(owner_mismatch()); + } + Ok(found) + } + + fn insert(&mut self, owner: TargetOwner, metadata: Metadata) -> TargetObservation { + let mut identity = DefaultHasher::new(); + owner.hash(&mut identity); + let target_id = ProviderTargetId::new(format!("fake-{:016x}", identity.finish())) + .expect("valid fake target id"); + let target = FakeTarget { + target_id: target_id.clone(), + owner: owner.clone(), + power: PowerState::Running, + ingress: false, + metadata, + }; + let observation = target.observation(); + self.by_owner.insert(owner, target_id.clone()); + self.by_id.insert(target_id, target); + observation + } + + fn create(&mut self, params: CreateTargetParams) -> Result { + if let Some(target_id) = self.by_owner.get(¶ms.owner) { + return Ok(self.by_id[target_id].observation()); + } + if params.template_id.as_str() != FAKE_TEMPLATE_ID { + return Err(ProtocolError::not_found(format!( + "unknown template: {}", + params.template_id + ))); + } + let metadata = Metadata::new(BTreeMap::from([( + "imageFingerprint".to_owned(), + format!("fake:{}", params.template_id), + )])) + .expect("valid fake metadata"); + Ok(self.insert(params.owner, metadata)) + } + + fn adopt(&mut self, params: AdoptTargetParams) -> Result { + let fingerprint = format!("fake:adopted:{}", params.source_target); + if let Some(target_id) = self.by_owner.get(¶ms.owner) { + let target = &self.by_id[target_id]; + return if target.metadata.get("imageFingerprint") == Some(fingerprint.as_str()) { + Ok(target.observation()) + } else { + Err(owner_mismatch()) + }; + } + if self + .by_id + .values() + .any(|target| target.metadata.get("imageFingerprint") == Some(fingerprint.as_str())) + { + return Err(owner_mismatch()); + } + let metadata = Metadata::new(BTreeMap::from([( + "imageFingerprint".to_owned(), + fingerprint, + )])) + .map_err(|error| ProtocolError::invalid_params(error.to_string()))?; + Ok(self.insert(params.owner, metadata)) + } + + fn set_power( + &mut self, + params: SetTargetPowerParams, + ) -> Result { + if !FAKE_POWER_STATES.contains(¶ms.power) { + return Err(ProtocolError::invalid_field( + "power", + format!("unsupported power state {}", params.power), + )); + } + let target_id = self.resolve(¶ms.target)?.target_id.clone(); + let target = self.by_id.get_mut(&target_id).expect("resolved target"); + target.power = params.power; + Ok(target.observation()) + } + + fn close(&mut self, params: CloseTargetParams) -> Result { + let closed = |target_id| TargetObservation { + target_id, + owner: params.target.owner.clone(), + display_name: None, + state: TargetState { + phase: TargetPhase::Closed, + power: None, + daemon_ready: false, + message: None, + }, + supported_power_states: Vec::new(), + ingress: None, + metadata: Metadata::default(), + }; + let target_id = match self.resolve(¶ms.target) { + Ok(target) => target.target_id.clone(), + Err(error) if error.kind() == ProtocolErrorKind::NotFound => { + return Ok(closed(None)); + } + Err(error) => return Err(error), + }; + if let Some(target) = self.by_id.remove(&target_id) { + self.by_owner.remove(&target.owner); + } + Ok(closed(Some(target_id))) + } + + fn set_ingress( + &mut self, + target: &TargetRef, + enabled: bool, + ) -> Result { + let target_id = self.resolve(target)?.target_id.clone(); + let target = self.by_id.get_mut(&target_id).expect("resolved target"); + target.ingress = enabled; + Ok(target.observation()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::environments::gateway::runtime_client_info; + use environment_client::{ + ConnectOptions, + conformance::controller::{ControllerConformance, run}, + }; + use std::{ + sync::atomic::{AtomicU64, Ordering}, + time::Duration, + }; + + fn owner_factory() -> Arc TargetOwner + Send + Sync> { + let next = Arc::new(AtomicU64::new(0)); + let universe = uuid::Uuid::new_v4().to_string(); + Arc::new(move || { + let n = next.fetch_add(1, Ordering::SeqCst); + TargetOwner { + universe_id: UniverseId::new(universe.clone()).unwrap(), + binding_id: BindingId::new("conformance").unwrap(), + environment_id: EnvironmentId::new(format!("environment_{n}")).unwrap(), + incarnation_id: IncarnationId::new(format!("incarnation_{n}")).unwrap(), + } + }) + } + + async fn conformance(options: FakeControllerOptions) { + let controller = FakeController::start_with(options).await.unwrap(); + let mut suite = ControllerConformance::new( + controller.controller_url(), + ConnectOptions::new(runtime_client_info()).with_bearer_token(controller.credential()), + TemplateId::new(FAKE_TEMPLATE_ID).unwrap(), + owner_factory(), + ); + suite.prompt_bound = Duration::from_secs(5); + suite.settle_timeout = Duration::from_secs(10); + suite.poll_interval = Duration::from_millis(50); + run(&suite) + .await + .into_result() + .expect("the fake controller conforms"); + } + + #[tokio::test(flavor = "current_thread")] + async fn the_fake_controller_conforms() { + conformance(FakeControllerOptions::default()).await; + } + + #[tokio::test(flavor = "current_thread")] + async fn the_fake_controller_without_optional_features_conforms() { + conformance(FakeControllerOptions { + adopt_target: false, + ingress: false, + ..FakeControllerOptions::default() + }) + .await; + } +} diff --git a/crates/temporal-runtime/src/environments/gateway.rs b/crates/temporal-runtime/src/environments/gateway.rs index 21980c4e2..bae09af3f 100644 --- a/crates/temporal-runtime/src/environments/gateway.rs +++ b/crates/temporal-runtime/src/environments/gateway.rs @@ -1,11 +1,22 @@ -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; -use environment_protocol::shared::{ - EnvironmentCapabilities, EnvironmentDataConnection, EnvironmentScope, EnvironmentTransport, - ProviderTargetId, -}; +use environment_client::{ClientError, ConnectOptions, DataClient}; +use environment_protocol::{data::handshake::InitializeResponse, handshake::ImplementationInfo}; use uuid::Uuid; +/// Path prefix of the worker routes the environment gateway serves: +/// `{prefix}/{universe}/{environment}/{incarnation}`. Runtime-internal: only +/// Lightspeed's own workers dial it, with the deployment token. +pub const ROUTE_PATH_PREFIX: &str = "/environment-gateway/routes"; + +/// Connect bound (TCP, TLS and upgrade) of a worker route. The gateway +/// answers a registered environment's upgrade only once the reverse +/// stream is paired, which takes up to its stream-open bound; the worker +/// waits a little longer so that the gateway's typed refusal, not the +/// client's own timeout, reaches the caller. +pub const WORKER_ROUTE_CONNECT_TIMEOUT: Duration = + Duration::from_secs(crate::gateway::registration::STREAM_OPEN_TIMEOUT.as_secs() + 5); + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct RouteKey { pub universe_id: Uuid, @@ -61,41 +72,59 @@ impl EnvironmentGatewayClientConfig { } } - pub fn connection(&self, key: &RouteKey) -> EnvironmentDataConnection { - let base = websocket_base(&self.base_url); - EnvironmentDataConnection { - target_id: ProviderTargetId::new(key.environment_id.clone()), - endpoint: format!( - "{base}/environment-gateway/routes/{}/{}/{}", - key.universe_id, key.environment_id, key.incarnation_id - ), - transport: EnvironmentTransport::WebSocket, - scope: EnvironmentScope::Default, - default_cwd: None, - capabilities: EnvironmentCapabilities::default(), - } + /// The worker route of one environment incarnation: a data-plane + /// endpoint whatever the environment's source. + pub fn endpoint(&self, key: &RouteKey) -> String { + format!( + "{}{ROUTE_PATH_PREFIX}/{}/{}/{}", + websocket_base(&self.base_url), + key.universe_id, + environment_protocol::routes::encode_path_segment(&key.environment_id), + environment_protocol::routes::encode_path_segment(&key.incarnation_id), + ) } - pub fn connection_for( + pub fn endpoint_for( &self, universe_id: Uuid, environment: &environments::EnvironmentRecord, - ) -> EnvironmentDataConnection { - self.connection(&self.route_key(universe_id, environment)) + ) -> String { + self.endpoint(&self.route_key(universe_id, environment)) + } + + /// Data-plane connect options for a worker route: `clientInfo` names + /// the runtime build, `user_agent` the caller, and the deployment token + /// is the bearer, and the connect bound is + /// [`WORKER_ROUTE_CONNECT_TIMEOUT`]. Callers on background paths add a + /// request timeout. + pub fn connect_options(&self, user_agent: impl Into) -> ConnectOptions { + let mut options = ConnectOptions::new(runtime_client_info()) + .with_bearer_token((*self.deployment_token).clone()) + .with_user_agent(user_agent); + options.connect_timeout = WORKER_ROUTE_CONNECT_TIMEOUT; + options } - pub fn connect_options( + /// Dial the worker route of `key` and complete the data-plane + /// handshake. + pub async fn connect( &self, - user_agent: impl Into, - ) -> environment_client::WebSocketConnectOptions { - environment_client::WebSocketConnectOptions { - bearer_token: Some((*self.deployment_token).clone()), - user_agent: Some(user_agent.into()), - headers: Vec::new(), - } + key: &RouteKey, + options: ConnectOptions, + ) -> Result<(DataClient, InitializeResponse), ClientError> { + DataClient::connect_initialized(&self.endpoint(key), options).await } } +/// `clientInfo` of every data and controller connection this runtime +/// opens. +pub fn runtime_client_info() -> ImplementationInfo { + ImplementationInfo::new("lightspeed-runtime") + .and_then(|info| info.with_version(release_info::VERSION)) + .and_then(|info| info.with_git_sha(release_info::GIT_SHA)) + .expect("the runtime's build facts are valid implementation info") +} + pub fn bearer_matches(headers: &axum::http::HeaderMap, expected: &str) -> bool { headers .get(axum::http::header::AUTHORIZATION) @@ -111,7 +140,7 @@ pub fn close_message(reason: &'static str) -> axum::extract::ws::Message { })) } -pub(crate) fn websocket_base(url: &str) -> String { +fn websocket_base(url: &str) -> String { if let Some(rest) = url.strip_prefix("https://") { format!("wss://{rest}") } else if let Some(rest) = url.strip_prefix("http://") { @@ -145,13 +174,43 @@ mod tests { } #[test] - fn gateway_connection_is_stable_and_has_no_cached_capabilities() { - let config = EnvironmentGatewayClientConfig::new("https://gateway.test", "secret"); - let connection = config.connection(&key()); + fn worker_routes_are_stable_websocket_urls() { + let config = EnvironmentGatewayClientConfig::new("https://gateway.test/", "secret"); assert_eq!( - connection.endpoint, + config.endpoint(&key()), "wss://gateway.test/environment-gateway/routes/00000000-0000-0000-0000-000000000000/environment-a/incarnation-a" ); - assert_eq!(connection.capabilities, EnvironmentCapabilities::default()); + let local = EnvironmentGatewayClientConfig::new("http://127.0.0.1:18080", "secret"); + assert!( + local + .endpoint(&key()) + .starts_with("ws://127.0.0.1:18080/environment-gateway/routes/") + ); + } + + #[test] + fn connect_options_carry_the_deployment_token_and_runtime_build() { + let config = EnvironmentGatewayClientConfig::new("https://gateway.test", "secret"); + let options = config.connect_options("lightspeed-test"); + assert_eq!( + options.bearer_token.as_ref().map(|token| token.expose()), + Some("secret") + ); + assert_eq!(options.user_agent.as_deref(), Some("lightspeed-test")); + assert_eq!(options.client_info.name(), "lightspeed-runtime"); + assert_eq!(options.client_info.version(), Some(release_info::VERSION)); + assert_eq!(options.connect_timeout, WORKER_ROUTE_CONNECT_TIMEOUT); + } + + #[test] + fn bearer_tokens_compare_exactly() { + let mut headers = axum::http::HeaderMap::new(); + headers.insert( + axum::http::header::AUTHORIZATION, + "Bearer secret".parse().unwrap(), + ); + assert!(bearer_matches(&headers, "secret")); + assert!(!bearer_matches(&headers, "secre")); + assert!(!bearer_matches(&axum::http::HeaderMap::new(), "secret")); } } diff --git a/crates/temporal-runtime/src/environments/lifecycle.rs b/crates/temporal-runtime/src/environments/lifecycle.rs index e7f7c8f54..34ef4fd4b 100644 --- a/crates/temporal-runtime/src/environments/lifecycle.rs +++ b/crates/temporal-runtime/src/environments/lifecycle.rs @@ -1,45 +1,218 @@ +use super::provider_controllers::{ + ControllerDisposition, ControllerError, ControllerFeature, ProviderController, ProviderPass, + provider_connect_options, +}; use super::providers::{ - binding_context, environment_view, map_environments_error, - parse_environment_provider_binding_id, registry_idle_policy, registry_lifecycle_status, - registry_power_state, + environment_view, map_environments_error, parse_environment_provider_binding_id, + registry_idle_policy, registry_lifecycle_status, registry_power_state, }; use super::*; use ::environments::{ - BeginCloseEnvironment, CreateEnvironment, CreateExternalEnvironment, EnvironmentId, - EnvironmentIncarnationId, EnvironmentProviderBindingStore, EnvironmentProvisionRequestId, - EnvironmentRecord, EnvironmentSource, EnvironmentStatus, EnvironmentStore, + BeginCloseEnvironment, CreateEnvironment, CreateExternalEnvironment, DaemonBuild, + EnvironmentEndpoint, EnvironmentId, EnvironmentIncarnationId, EnvironmentLifecycleError, + EnvironmentProviderId, EnvironmentProvisionRequestId, EnvironmentRecord, + EnvironmentRegistryError, EnvironmentSource, EnvironmentStatus, EnvironmentStore, EnvironmentTemplateId, FailEnvironmentLifecycle, FinishCloseEnvironment, ListEnvironments, - ObserveProvisionedEnvironment, PowerState, SetEnvironmentIdlePolicy, SetEnvironmentPower, -}; -use environment_protocol::control::ingress::{ - EnsureIngressParams, ProviderIngressStatus, RemoveIngressParams, + ObserveProvisionedEnvironment, PowerState, ReconcileScope, RecordEnvironmentIngress, + RecordEnvironmentLifecycleError, SetEnvironmentIdlePolicy, SetEnvironmentIngress, + SetEnvironmentPower, }; -use environment_protocol::control::targets::{ - AdoptTargetParams, CloseTargetParams, CreateTargetParams, ProviderTargetStatus, - ProviderTargetSummary, SetTargetPowerParams, +use auth::SecretValue; +use environment_client::{ClientError, DataClient}; +use environment_protocol::{ + control::{ + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + AdoptTargetParams, CloseTargetParams, CreateTargetParams, GetTargetParams, + SetTargetPowerParams, TargetObservation, TargetOwner, TargetPhase, TargetRef, + TargetState, + }, + }, + error::PreconditionReason, + ids::{BindingId, IncarnationId, ProviderNativeRef, TemplateId, UniverseId}, + routes::provider_data_route, }; -/// Map a provider target observation to the logical lifecycle status. A -/// passive provider reports Ready only after its private envd is reachable; -/// no provider presence has to register separately. -pub(crate) fn lifecycle_status_from_target(status: ProviderTargetStatus) -> EnvironmentStatus { - match status { - ProviderTargetStatus::Ready => EnvironmentStatus::Ready, - ProviderTargetStatus::Creating | ProviderTargetStatus::Starting => { - EnvironmentStatus::Booting - } - ProviderTargetStatus::Paused => EnvironmentStatus::Paused, - ProviderTargetStatus::Suspended => EnvironmentStatus::Suspended, - ProviderTargetStatus::Stopped => EnvironmentStatus::Offline, - ProviderTargetStatus::Closing => EnvironmentStatus::Closing, - ProviderTargetStatus::Closed => EnvironmentStatus::Closed, - ProviderTargetStatus::Failed => EnvironmentStatus::Failed, - ProviderTargetStatus::Unknown => EnvironmentStatus::Unknown, +/// How long a provisioning, booting or unknown environment may go without +/// converging (transient errors such as an unreachable provider, or +/// observations that make no progress, such as a guest whose envd never +/// answers) before it is failed with the last error. +pub(crate) const PROVISIONING_DEADLINE_MS: i64 = 30 * 60 * 1000; + +/// How often a reconcile pass also observes steady provisioned +/// environments with `target/get`, so a target deleted or changed out of +/// band is noticed and ingress converges. +pub(crate) const STEADY_OBSERVATION_INTERVAL: std::time::Duration = + std::time::Duration::from_secs(120); + +/// A repeated transient error is written again only after this long, so a +/// provider that stays down does not rewrite every row on every pass. +const LIFECYCLE_ERROR_REFRESH_MS: i64 = 30 * 1000; + +/// What one target observation means for its environment. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum TargetVerdict { + /// Record this status. + Status(EnvironmentStatus), + /// Running, and the provider's own probe reached envd: Ready once the + /// runtime's data-plane handshake succeeds. + DaemonReady, + /// Broken or gone for good. + Failed(String), + /// The provider cannot tell; retried like a transient error. + Unknown(String), +} + +/// Map a target's two-axis state to the environment lifecycle. A target +/// that closes while its environment is not closing was closed out of band. +pub(crate) fn target_verdict(state: &TargetState) -> TargetVerdict { + let message = state.message.as_ref().map(|message| message.as_str()); + match state.phase { + TargetPhase::Provisioning => TargetVerdict::Status(EnvironmentStatus::Provisioning), + TargetPhase::Active => match state.power { + Some(PowerState::Running) if state.daemon_ready => TargetVerdict::DaemonReady, + Some(PowerState::Running) => TargetVerdict::Status(EnvironmentStatus::Booting), + Some(PowerState::Paused) => TargetVerdict::Status(EnvironmentStatus::Paused), + Some(PowerState::Suspended) => TargetVerdict::Status(EnvironmentStatus::Suspended), + Some(PowerState::Stopped) => TargetVerdict::Status(EnvironmentStatus::Offline), + None => TargetVerdict::Unknown( + "the provider reported an active target without a power state".to_owned(), + ), + }, + TargetPhase::Closing => { + TargetVerdict::Failed("the provider is closing the environment's target".to_owned()) + } + TargetPhase::Closed => { + TargetVerdict::Failed("the provider closed the environment's target".to_owned()) + } + TargetPhase::Failed => TargetVerdict::Failed( + message + .unwrap_or("the provider reported the target failed") + .to_owned(), + ), + TargetPhase::Unknown => TargetVerdict::Unknown( + message + .unwrap_or("the provider cannot tell the target's state") + .to_owned(), + ), } } +/// The lifecycle note for an observation that leaves a converging +/// environment where it was: recorded like a transient error, so the +/// provisioning deadline runs while nothing progresses. +pub(crate) fn waiting_note(status: EnvironmentStatus, state: &TargetState) -> String { + let waiting = match status { + EnvironmentStatus::Booting => "waiting for envd to answer on the target", + _ => "waiting for the provider to provision the target", + }; + match &state.message { + Some(message) => format!("{waiting}: {}", message.as_str()), + None => waiting.to_owned(), + } +} + +/// True once transient errors or stalled observations have persisted past +/// the provisioning deadline. +pub(crate) fn provisioning_deadline_passed( + error: Option<&EnvironmentLifecycleError>, + now_ms: i64, +) -> bool { + error.is_some_and(|error| now_ms.saturating_sub(error.first_at_ms) >= PROVISIONING_DEADLINE_MS) +} + +/// True when `message` at `now_ms` is worth a write over `recorded`. +fn lifecycle_error_is_news( + recorded: Option<&EnvironmentLifecycleError>, + message: &str, + now_ms: i64, +) -> bool { + let cleaned = EnvironmentLifecycleError::record(None, message, now_ms).message; + recorded.is_none_or(|recorded| { + recorded.message != cleaned + || now_ms.saturating_sub(recorded.last_at_ms) >= LIFECYCLE_ERROR_REFRESH_MS + }) +} + +/// The outcome of the data-plane handshake at the Ready transition. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum DaemonProbe { + Ready(DaemonBuild), + /// envd speaks another protocol version and can never serve this + /// runtime. + Mismatch { + local: u32, + remote: u32, + }, + /// envd did not complete the handshake; retried. + Unreachable(String), +} + +/// Run `initialize` against a target's data route, with the provider +/// credential, and close the connection again. +pub(crate) async fn probe_daemon(endpoint: &str, credential: &SecretValue) -> DaemonProbe { + match DataClient::connect_initialized(endpoint, provider_connect_options(credential)).await { + Ok((client, initialized)) => { + let _ = client.close().await; + DaemonProbe::Ready(DaemonBuild::from_implementation( + &initialized.server_info, + initialized.protocol_version, + )) + } + Err(ClientError::VersionMismatch { local, remote, .. }) => { + DaemonProbe::Mismatch { local, remote } + } + Err(error) => DaemonProbe::Unreachable(format!( + "envd did not complete the data-plane handshake: {error}" + )), + } +} + +/// The controller-plane owner of an environment's current incarnation. +pub(crate) fn target_owner( + universe_id: uuid::Uuid, + environment: &EnvironmentRecord, +) -> Result { + let binding_id = environment + .binding_id() + .ok_or_else(|| invalid_registry("only provisioned environments have a provider target"))?; + let invalid = |error: environment_protocol::ids::IdError| { + invalid_registry(format!( + "environment ids do not fit the controller protocol: {error}" + )) + }; + Ok(TargetOwner { + universe_id: UniverseId::new(universe_id.to_string()).map_err(invalid)?, + binding_id: BindingId::new(binding_id.as_str()).map_err(invalid)?, + environment_id: environment_protocol::ids::EnvironmentId::new( + environment.environment_id.as_str(), + ) + .map_err(invalid)?, + incarnation_id: IncarnationId::new(environment.incarnation.incarnation_id.as_str()) + .map_err(invalid)?, + }) +} + +/// Wall-clock milliseconds for reconciler writes. +fn clock_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +fn invalid_registry(message: impl Into) -> ControllerError { + ControllerError::Registry(EnvironmentRegistryError::InvalidInput { + message: message.into(), + }) +} + impl EnvironmentService { + /// `environments/ingress/put`: record the public ingress intent. The + /// reconciler converges the provider toward it while the environment is + /// ready and records the realized endpoint. Enabling is refused up front + /// when the provider does not offer ingress. pub(crate) async fn put_environment_ingress_record( &self, params: EnvironmentIngressPutParams, @@ -48,90 +221,28 @@ impl EnvironmentService { let environment = EnvironmentStore::read_environment(self.store.as_ref(), &environment_id) .await .map_err(map_environments_error)?; - if !matches!( + if matches!( environment.status, - EnvironmentStatus::Ready - | EnvironmentStatus::Offline - | EnvironmentStatus::Paused - | EnvironmentStatus::Suspended + EnvironmentStatus::Closing | EnvironmentStatus::Closed | EnvironmentStatus::Failed ) { return Err(AgentApiError::rejected( - "public ingress requires a ready or powered-down environment", + "public ingress requires an environment that is not closing, closed or failed", )); } - let EnvironmentSource::Provisioned { - provider_id, - binding_id, - } = &environment.source - else { + let EnvironmentSource::Provisioned { provider_id, .. } = &environment.source else { return Err(AgentApiError::rejected( "provider-managed ingress is unavailable for external environments", )); }; - let target_id = environment - .incarnation - .provider_target_id - .clone() - .ok_or_else(|| AgentApiError::rejected("environment has no provider target"))?; - let provider = self.read_environment_provider(provider_id).await?; - let binding = EnvironmentProviderBindingStore::read_provider_binding( - self.store.as_ref(), - self.store.config().universe_id, - binding_id, - ) - .await - .map_err(map_environments_error)?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) - .await?; - let public_endpoint = async { - if params.enabled { - let response = controller - .ensure_ingress(&EnsureIngressParams { - request_id: format!("ingress-enable:{}", environment.request_id), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id, - }) - .await?; - if response.status != ProviderIngressStatus::Ready { - return Err(AgentApiError::rejected( - "provider did not realize public ingress", - )); - } - Ok(Some(response.public_endpoint.ok_or_else(|| { - AgentApiError::rejected( - "provider returned ready ingress without a public endpoint", - ) - })?)) - } else { - let response = controller - .remove_ingress(&RemoveIngressParams { - request_id: format!("ingress-disable:{}", environment.request_id), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id, - }) - .await?; - if response.status != ProviderIngressStatus::Disabled { - return Err(AgentApiError::rejected( - "provider did not remove public ingress", - )); - } - Ok(None) - } + if params.enabled { + self.require_controller_feature(provider_id, ControllerFeature::Ingress) + .await?; } - .await; - let public_endpoint = finish_provider_controller(controller, public_endpoint).await?; let environment = EnvironmentStore::set_environment_ingress( self.store.as_ref(), - ::environments::SetEnvironmentIngress { + SetEnvironmentIngress { environment_id, enabled: params.enabled, - public_endpoint, updated_at_ms: now_ms()?, }, ) @@ -150,14 +261,16 @@ impl EnvironmentService { EnvironmentProvisionRequestId::try_new(params.request_id).map_err(|error| { AgentApiError::invalid_request(format!("invalid environment request id: {error}")) })?; - let connection = external_connection_from_api(params.connection)?; + let endpoint = EnvironmentEndpoint::parse(¶ms.endpoint).map_err(|error| { + AgentApiError::invalid_request(format!("invalid environment endpoint: {error}")) + })?; let environment = EnvironmentStore::create_external_environment( self.store.as_ref(), CreateExternalEnvironment { request_id, environment_id: allocate_environment_id(), incarnation_id: allocate_incarnation_id(), - connection, + endpoint, display_name: params.display_name, metadata: validated_caller_metadata(params.metadata)?, created_at_ms: now_ms()?, @@ -169,6 +282,7 @@ impl EnvironmentService { environment: environment_view(&environment), }) } + pub(crate) async fn create_environment_record( &self, params: EnvironmentCreateParams, @@ -320,7 +434,7 @@ impl EnvironmentService { metadata: params.metadata, provider_id: params .provider_id - .map(parse_environment_provider_id) + .map(super::providers::parse_environment_provider_id) .transpose()?, binding_id: params .binding_id @@ -362,70 +476,76 @@ impl EnvironmentService { /// Public entry point for one reconciliation pass; used by acceptance /// tests that drive the reconciler deterministically instead of running /// the background loop. + /// The pass includes steady environments. pub async fn reconcile_environments_once(&self) -> Result { - self.reconcile_environment_lifecycle_once().await + let mut pass = ProviderPass::default(); + let result = self + .reconcile_environment_lifecycle_pass(&mut pass, ReconcileScope::WithSteady) + .await; + pass.close().await; + result + } + + /// Reconcile this universe once, sharing `pass`'s controller + /// connections and unhealthy-provider skips with the rest of the + /// deployment pass. The registered sweep and the provisioned pass run + /// independently: an error in one never stops the other, and one + /// environment's error never stops the rest. + pub(crate) async fn reconcile_environment_lifecycle_pass( + &self, + pass: &mut ProviderPass, + scope: ReconcileScope, + ) -> Result { + let registered = self.reconcile_registered_once().await; + let provisioned = self.reconcile_provisioned_once(pass, scope).await; + match (registered, provisioned) { + (Ok(registered), Ok(provisioned)) => Ok(registered + provisioned), + (Err(error), Ok(_)) => Err(AgentApiError { + message: format!("registered environments: {}", error.message), + ..error + }), + (Ok(_), Err(error)) => Err(error), + (Err(registered), Err(provisioned)) => Err(AgentApiError { + message: format!( + "registered environments: {}; {}", + registered.message, provisioned.message + ), + ..provisioned + }), + } } - /// Reconcile every pending environment in this universe once. Calls are - /// stable and idempotent; a crash after controller I/O is recovered by a - /// later pass issuing the same IDs again. - pub(crate) async fn reconcile_environment_lifecycle_once( + /// One pass over every environment `scope` selects. Calls are + /// idempotent per owner, so a crash after controller I/O is recovered + /// by a later pass making the same calls again. + async fn reconcile_provisioned_once( &self, + pass: &mut ProviderPass, + scope: ReconcileScope, ) -> Result { - let mut changed = self.reconcile_registered_once().await?; let environments = - EnvironmentStore::list_environments_needing_reconcile(self.store.as_ref()) + EnvironmentStore::list_environments_needing_reconcile(self.store.as_ref(), scope) .await .map_err(map_environments_error)?; - // One unreachable provider must not stall every other environment in - // the universe: per-environment failures are remembered and the pass - // continues; the first failure is reported once the pass is complete. + let mut changed = 0; let mut first_error: Option = None; for environment in environments { - let outcome = match environment.status { - EnvironmentStatus::Closing => self.reconcile_environment_close(&environment).await, - EnvironmentStatus::Ready - | EnvironmentStatus::Paused - | EnvironmentStatus::Suspended - | EnvironmentStatus::Offline - if environment.power_diverges() => - { - self.reconcile_environment_power(&environment).await - } - EnvironmentStatus::Provisioning - | EnvironmentStatus::Booting - | EnvironmentStatus::Unknown => { - match self.reconcile_environment_create(&environment).await { - Err(error) if error.kind == api::AgentApiErrorKind::Rejected => { - EnvironmentStore::fail_environment_lifecycle( - self.store.as_ref(), - FailEnvironmentLifecycle { - environment_id: environment.environment_id.clone(), - message: error.message, - observed_at_ms: now_ms()?, - }, - ) - .await - .map_err(map_environments_error) - .map(|_| true) - } - other => other, - } - } - _ => Ok(false), - }; - match outcome { + match self.reconcile_environment(pass, &environment).await { Ok(did_change) => changed += usize::from(did_change), Err(error) => { - if first_error.is_none() { - first_error = Some(AgentApiError { - message: format!( - "environment {}: {}", - environment.environment_id, error.message - ), - ..error - }); - } + tracing::debug!( + target: "temporal_runtime", + environment_id = %environment.environment_id, + %error, + "environment reconcile step failed" + ); + first_error.get_or_insert(AgentApiError { + message: format!( + "environment {}: {}", + environment.environment_id, error.message + ), + ..error + }); } } } @@ -435,235 +555,711 @@ impl EnvironmentService { } } - async fn reconcile_environment_create( + async fn reconcile_environment( &self, - environment: &::environments::EnvironmentRecord, + pass: &mut ProviderPass, + environment: &EnvironmentRecord, ) -> Result { - let EnvironmentSource::Provisioned { - provider_id, - binding_id, - } = &environment.source - else { + let EnvironmentSource::Provisioned { provider_id, .. } = &environment.source else { + // External and registered environments hold no provider target. + if environment.status == EnvironmentStatus::Closing { + self.finish_close(environment) + .await + .map_err(map_environments_error)?; + return Ok(true); + } return Ok(false); }; - let provider = self.read_environment_provider(provider_id).await?; - let binding = EnvironmentProviderBindingStore::read_provider_binding( - self.store.as_ref(), - self.store.config().universe_id, - binding_id, - ) - .await - .map_err(map_environments_error)?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) - .await?; - let target = async { - match ( - environment.incarnation.template_id.as_ref(), - environment.incarnation.adoption_source_target.as_ref(), - ) { - (Some(template_id), None) => Ok(controller - .create_target(&CreateTargetParams { - request_id: environment.request_id.to_string(), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - template_id: template_id.to_string(), - }) - .await? - .target), - (None, Some(source_target)) => Ok(controller - .adopt_target(&AdoptTargetParams { - request_id: environment.request_id.to_string(), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - source_target: source_target.clone(), - }) - .await? - .target), - _ => Err(AgentApiError::internal( - "provisioned incarnation has invalid realization", - )), + let step = match environment.status { + EnvironmentStatus::Closing => self.close_target(pass, environment, provider_id).await, + EnvironmentStatus::Provisioning + | EnvironmentStatus::Booting + | EnvironmentStatus::Unknown => { + self.observe_target(pass, environment, provider_id).await + } + _ if environment.power_diverges() => { + self.converge_power(pass, environment, provider_id).await + } + // Steady targets (and ready ones whose ingress diverges) are + // observed with `target/get`: a vanished target fails the + // environment, and ingress converges from the observation. + _ if environment.is_steady_provisioned() => { + self.observe_target(pass, environment, provider_id).await + } + _ => return Ok(false), + }; + match step { + Ok(changed) => Ok(changed), + Err(error) => { + pass.observe_failure(&error).await; + self.controller_failure(environment, error).await } } - .await; - let target = finish_provider_controller(controller, target).await?; - self.record_target_observation(environment, target).await } - /// Persist a provider observation when it changes the record: status, - /// target id, or the provider-reported power states. - async fn record_target_observation( + /// Decide what a failed controller interaction does to the environment. + async fn controller_failure( &self, - environment: &::environments::EnvironmentRecord, - target: ProviderTargetSummary, + environment: &EnvironmentRecord, + error: ControllerError, ) -> Result { - let status = lifecycle_status_from_target(target.status); - let changed = environment.status != status - || environment.incarnation.provider_target_id.as_ref() != Some(&target.target_id) - || environment.incarnation.power_states != target.power_states; - if changed { - EnvironmentStore::observe_provisioned_environment( - self.store.as_ref(), - ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: target.target_id, - status, - power_states: target.power_states, - observed_at_ms: now_ms()?, - }, - ) + if let ControllerError::Registry(error) = error { + return Err(map_environments_error(error)); + } + let converging = matches!( + environment.status, + EnvironmentStatus::Closing + | EnvironmentStatus::Provisioning + | EnvironmentStatus::Booting + | EnvironmentStatus::Unknown + ); + if !converging && error.disposition() != ControllerDisposition::Gone { + // A steady environment whose power change failed keeps serving + // in its current state. The error is reported with the pass + // instead of recorded, so it cannot start a provisioning + // deadline that a later wake would inherit. + return Err(error.into_api_error()); + } + self.decide_controller_failure(environment, error) .await - .map_err(map_environments_error)?; + .map_err(map_environments_error) + } + + async fn decide_controller_failure( + &self, + environment: &EnvironmentRecord, + error: ControllerError, + ) -> Result { + let error = match error { + ControllerError::Registry(error) => return Err(error), + error => error, + }; + let message = error.to_string(); + let disposition = error.disposition(); + match environment.status { + EnvironmentStatus::Closing if disposition == ControllerDisposition::Gone => { + self.finish_close(environment).await?; + Ok(true) + } + EnvironmentStatus::Closing => self.record_lifecycle_error(environment, &message).await, + EnvironmentStatus::Provisioning + | EnvironmentStatus::Booting + | EnvironmentStatus::Unknown => match disposition { + ControllerDisposition::Transient => { + self.transient_failure(environment, &message).await + } + ControllerDisposition::Refused => self.fail(environment, &message, None).await, + ControllerDisposition::Gone => { + let message = if environment.incarnation.provider_target_id.is_some() { + format!("provider target disappeared: {message}") + } else { + message + }; + self.fail(environment, &message, None).await + } + }, + // Only a vanished target reaches here for a steady environment. + _ => { + self.fail( + environment, + &format!("provider target disappeared: {message}"), + None, + ) + .await + } } - Ok(changed) } - /// Converge a steady-state provisioned environment toward its desired - /// power. One idempotent `setTargetPower` per pass; the observed - /// summary is recorded like any other observation, so a transitional - /// answer (`Starting`) flows back through the create/observe path. - async fn reconcile_environment_power( + /// Keep the status and record the error; fail once errors persisted + /// past the provisioning deadline. + async fn transient_failure( &self, - environment: &::environments::EnvironmentRecord, - ) -> Result { - let EnvironmentSource::Provisioned { - provider_id, - binding_id, - } = &environment.source - else { + environment: &EnvironmentRecord, + message: &str, + ) -> Result { + let now = clock_ms(); + if provisioning_deadline_passed(environment.lifecycle_error.as_ref(), now) { + return self + .fail( + environment, + &format!( + "provisioning did not finish within {} minutes; last error: {message}", + PROVISIONING_DEADLINE_MS / 60_000 + ), + None, + ) + .await; + } + self.record_lifecycle_error(environment, message).await + } + + async fn record_lifecycle_error( + &self, + environment: &EnvironmentRecord, + message: &str, + ) -> Result { + let now = clock_ms(); + if !lifecycle_error_is_news(environment.lifecycle_error.as_ref(), message, now) { return Ok(false); + } + EnvironmentStore::record_environment_lifecycle_error( + self.store.as_ref(), + RecordEnvironmentLifecycleError { + environment_id: environment.environment_id.clone(), + message: message.to_owned(), + observed_at_ms: now, + }, + ) + .await?; + Ok(true) + } + + async fn fail( + &self, + environment: &EnvironmentRecord, + message: &str, + daemon_build: Option, + ) -> Result { + EnvironmentStore::fail_environment_lifecycle( + self.store.as_ref(), + FailEnvironmentLifecycle { + environment_id: environment.environment_id.clone(), + message: message.to_owned(), + daemon_build, + observed_at_ms: clock_ms(), + }, + ) + .await?; + Ok(true) + } + + async fn finish_close( + &self, + environment: &EnvironmentRecord, + ) -> Result<(), EnvironmentRegistryError> { + EnvironmentStore::finish_close_environment( + self.store.as_ref(), + FinishCloseEnvironment { + environment_id: environment.environment_id.clone(), + observed_at_ms: clock_ms(), + }, + ) + .await?; + Ok(()) + } + + /// Converging and steady environments: create (or adopt) while no + /// target id is recorded, observe with `target/get` once one is. + /// Creation is never re-sent for a known target. + async fn observe_target( + &self, + pass: &mut ProviderPass, + environment: &EnvironmentRecord, + provider_id: &EnvironmentProviderId, + ) -> Result { + let controller = pass + .controller( + self.store.as_ref(), + self.provider_controller_connector.as_ref(), + provider_id, + ) + .await?; + let owner = target_owner(self.store.config().universe_id, environment)?; + let response = match ( + &environment.incarnation.provider_target_id, + &environment.incarnation.template_id, + &environment.incarnation.adoption_source_target, + ) { + (Some(target_id), _, _) => { + controller + .client + .get_target(&GetTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: Some(target_id.clone()), + }, + }) + .await + } + (None, Some(template_id), None) => { + let template_id = TemplateId::new(template_id.as_str()).map_err(|error| { + invalid_registry(format!("template id does not fit the protocol: {error}")) + })?; + controller + .client + .create_target(&CreateTargetParams { + owner: owner.clone(), + template_id, + }) + .await + } + (None, None, Some(source_target)) => { + controller.require(ControllerFeature::AdoptTarget)?; + let source_target = + ProviderNativeRef::new(source_target.as_str()).map_err(|error| { + invalid_registry(format!("adoption source is not valid: {error}")) + })?; + controller + .client + .adopt_target(&AdoptTargetParams { + owner: owner.clone(), + source_target, + }) + .await + } + _ => { + return Err(invalid_registry( + "provisioned incarnation has neither a template nor an adoption source", + )); + } }; + let observation = response + .map_err(|error| controller.call_error(error))? + .target; + self.apply_observation(&controller, environment, &owner, observation) + .await + } + + /// Steady environments whose power diverges from the intent: one + /// idempotent `target/setPower` per pass. A target that is not active + /// is observed instead. + async fn converge_power( + &self, + pass: &mut ProviderPass, + environment: &EnvironmentRecord, + provider_id: &EnvironmentProviderId, + ) -> Result { let Some(target_id) = environment.incarnation.provider_target_id.clone() else { - return Ok(false); + return self.observe_target(pass, environment, provider_id).await; }; - let provider = self.read_environment_provider(provider_id).await?; - let binding = EnvironmentProviderBindingStore::read_provider_binding( - self.store.as_ref(), - self.store.config().universe_id, - binding_id, - ) - .await - .map_err(map_environments_error)?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) + let controller = pass + .controller( + self.store.as_ref(), + self.provider_controller_connector.as_ref(), + provider_id, + ) .await?; - let response = controller + let owner = target_owner(self.store.config().universe_id, environment)?; + let target = TargetRef { + owner: owner.clone(), + target_id: Some(target_id), + }; + let observation = match controller + .client .set_target_power(&SetTargetPowerParams { - request_id: format!( - "power:{}:{}", - environment.request_id, environment.desired_power - ), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id, + target: target.clone(), power: environment.desired_power, }) - .await; - let response = finish_provider_controller(controller, response).await?; - self.record_target_observation(environment, response.target) + .await + { + Ok(response) => response.target, + Err(error) => { + let error = controller.call_error(error); + if error.precondition() != Some(PreconditionReason::TargetNotActive) { + return Err(error); + } + controller + .client + .get_target(&GetTargetParams { target }) + .await + .map_err(|error| controller.call_error(error))? + .target + } + }; + self.apply_observation(&controller, environment, &owner, observation) .await } - async fn reconcile_environment_close( + /// Close by owner, with the target id when one is recorded: the provider + /// destroys whatever the owner holds, ingress included, and answers + /// `closed` once nothing is left. A closing target is closed again on + /// later passes (close is idempotent). + async fn close_target( &self, - environment: &::environments::EnvironmentRecord, - ) -> Result { - let Some(target_id) = environment.incarnation.provider_target_id.clone() else { - EnvironmentStore::finish_close_environment( + pass: &mut ProviderPass, + environment: &EnvironmentRecord, + provider_id: &EnvironmentProviderId, + ) -> Result { + let controller = match pass + .controller( self.store.as_ref(), - FinishCloseEnvironment { - environment_id: environment.environment_id.clone(), - observed_at_ms: now_ms()?, - }, + self.provider_controller_connector.as_ref(), + provider_id, ) .await - .map_err(map_environments_error)?; - return Ok(true); - }; - let EnvironmentSource::Provisioned { - provider_id, - binding_id, - } = &environment.source - else { - return Ok(false); + { + Err(ControllerError::Registry(EnvironmentRegistryError::NotFound { .. })) => { + // The provider record is gone, so nothing can reach its + // targets any more. + tracing::warn!( + target: "temporal_runtime", + environment_id = %environment.environment_id, + %provider_id, + "closing an environment whose provider is no longer registered" + ); + self.finish_close(environment).await?; + return Ok(true); + } + other => other?, }; - let provider = self.read_environment_provider(provider_id).await?; - let binding = EnvironmentProviderBindingStore::read_provider_binding( + let owner = target_owner(self.store.config().universe_id, environment)?; + let observation = controller + .client + .close_target(&CloseTargetParams { + target: TargetRef { + owner, + target_id: environment.incarnation.provider_target_id.clone(), + }, + }) + .await + .map_err(|error| controller.call_error(error))? + .target; + if observation.state.phase == TargetPhase::Closed { + self.finish_close(environment).await?; + return Ok(true); + } + let (_, changed) = self + .record_observation(environment, &observation, environment.status, None) + .await?; + Ok(changed) + } + + /// Record an observation and act on it: the Ready transition runs the + /// data-plane handshake first, and ingress converges against the + /// recorded intent on running targets. + async fn apply_observation( + &self, + controller: &ProviderController, + environment: &EnvironmentRecord, + owner: &TargetOwner, + observation: TargetObservation, + ) -> Result { + if observation.owner != *owner { + return Err(controller.call_error(ClientError::Decode( + "the provider answered with another owner's target".to_owned(), + ))); + } + match target_verdict(&observation.state) { + // A ready environment became ready through the runtime's own + // data-plane handshake; the provider's liveness probe alone does + // not demote it. + TargetVerdict::Status(EnvironmentStatus::Booting) + if environment.status == EnvironmentStatus::Ready => + { + let (recorded, changed) = self + .record_observation(environment, &observation, EnvironmentStatus::Ready, None) + .await?; + let converged = self + .converge_ingress(controller, &recorded, owner, &observation) + .await; + Ok(changed || converged) + } + TargetVerdict::Status(status) => { + let (recorded, changed) = self + .record_observation(environment, &observation, status, None) + .await?; + let stalled = environment.status == status + && matches!( + status, + EnvironmentStatus::Provisioning | EnvironmentStatus::Booting + ); + if !stalled { + return Ok(changed); + } + let noted = self + .transient_failure(&recorded, &waiting_note(status, &observation.state)) + .await?; + Ok(changed || noted) + } + TargetVerdict::Failed(message) => { + self.record_observation(environment, &observation, environment.status, None) + .await?; + Ok(self.fail(environment, &message, None).await?) + } + TargetVerdict::Unknown(message) => { + let (recorded, _) = self + .record_observation(environment, &observation, EnvironmentStatus::Unknown, None) + .await?; + self.transient_failure(&recorded, &message).await?; + Ok(true) + } + TargetVerdict::DaemonReady if environment.status == EnvironmentStatus::Ready => { + let (recorded, changed) = self + .record_observation(environment, &observation, EnvironmentStatus::Ready, None) + .await?; + let converged = self + .converge_ingress(controller, &recorded, owner, &observation) + .await; + Ok(changed || converged) + } + TargetVerdict::DaemonReady => { + let Some(target_id) = &observation.target_id else { + let (recorded, _) = self + .record_observation( + environment, + &observation, + EnvironmentStatus::Booting, + None, + ) + .await?; + self.transient_failure( + &recorded, + "the provider reported a running target without a targetId", + ) + .await?; + return Ok(true); + }; + let endpoint = provider_data_route(&controller.data_route_base, owner, target_id); + match probe_daemon(&endpoint, &controller.credential).await { + DaemonProbe::Ready(build) => { + let (recorded, _) = self + .record_observation( + environment, + &observation, + EnvironmentStatus::Ready, + Some(build), + ) + .await?; + self.converge_ingress(controller, &recorded, owner, &observation) + .await; + Ok(true) + } + DaemonProbe::Mismatch { local, remote } => { + self.record_observation( + environment, + &observation, + environment.status, + None, + ) + .await?; + self.fail( + environment, + &format!( + "the environment's envd speaks protocol version {remote}; this runtime requires {local}" + ), + Some(DaemonBuild { + version: None, + git_sha: None, + protocol_version: remote, + }), + ) + .await + .map_err(ControllerError::from) + } + DaemonProbe::Unreachable(message) => { + let (recorded, _) = self + .record_observation( + environment, + &observation, + EnvironmentStatus::Booting, + None, + ) + .await?; + self.transient_failure(&recorded, &message).await?; + Ok(true) + } + } + } + } + } + + /// Persist an observation when it changes the record: status, target + /// id, power states or the daemon build. Returns the current record. + async fn record_observation( + &self, + environment: &EnvironmentRecord, + observation: &TargetObservation, + status: EnvironmentStatus, + daemon_build: Option, + ) -> Result<(EnvironmentRecord, bool), ControllerError> { + let target_changed = observation.target_id.is_some() + && environment.incarnation.provider_target_id != observation.target_id; + let changed = environment.status != status + || target_changed + || environment.incarnation.power_states != observation.supported_power_states + || daemon_build.is_some(); + if !changed { + return Ok((environment.clone(), false)); + } + let recorded = EnvironmentStore::observe_provisioned_environment( self.store.as_ref(), - self.store.config().universe_id, - binding_id, + ObserveProvisionedEnvironment { + environment_id: environment.environment_id.clone(), + provider_target_id: observation.target_id.clone(), + status, + power_states: observation.supported_power_states.clone(), + daemon_build, + observed_at_ms: clock_ms(), + }, ) - .await - .map_err(map_environments_error)?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) - .await?; - let response = async { - let ingress_response = controller - .remove_ingress(&RemoveIngressParams { - request_id: format!("ingress-close:{}", environment.request_id), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id: target_id.clone(), - }) - .await?; - if ingress_response.status != ProviderIngressStatus::Disabled { - return Err(AgentApiError::rejected( - "provider did not remove ingress before target close", - )); + .await?; + Ok((recorded, true)) + } + + /// Converge a running target's ingress toward the recorded intent. The + /// recorded endpoint always mirrors what the provider last reported. + /// Ingress methods answer promptly and may continue in the background, + /// so an answer that has not converged yet is pending: the intent stays + /// and a later observation checks again. Only a typed refusal of + /// `ingress/ensure` drops the intent. Ingress failures never fail the + /// environment and are never recorded as lifecycle errors, so they + /// cannot start a provisioning deadline. Returns whether the record + /// changed. + async fn converge_ingress( + &self, + controller: &ProviderController, + environment: &EnvironmentRecord, + owner: &TargetOwner, + observation: &TargetObservation, + ) -> bool { + let realized = realized_ingress(observation); + let mut changed = self + .record_realized_ingress( + environment, + environment.public_endpoint.as_deref(), + &realized, + ) + .await; + let mut recorded = realized; + if environment.public_ingress_enabled == recorded.is_some() { + return changed; + } + let enabling = environment.public_ingress_enabled; + if !controller.features.ingress { + if enabling { + tracing::warn!( + target: "temporal_runtime", + environment_id = %environment.environment_id, + provider_id = %controller.provider_id, + "the provider no longer offers public ingress; the ingress intent is dropped" + ); + changed |= self.drop_ingress_intent(environment).await; } + return changed; + } + let target = TargetRef { + owner: owner.clone(), + target_id: observation.target_id.clone(), + }; + let answer = if enabling { controller - .close_target(&CloseTargetParams { - request_id: format!("close:{}", environment.request_id), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id, - force: false, - }) + .client + .ensure_ingress(&EnsureIngressParams { target }) .await + } else { + controller + .client + .remove_ingress(&RemoveIngressParams { target }) + .await + }; + match answer.map_err(|error| controller.call_error(error)) { + Ok(response) => { + let answered = realized_ingress(&response.target); + changed |= self + .record_realized_ingress(environment, recorded.as_deref(), &answered) + .await; + recorded = answered; + if environment.public_ingress_enabled != recorded.is_some() { + tracing::debug!( + target: "temporal_runtime", + environment_id = %environment.environment_id, + enabling, + "public ingress change is still in progress at the provider" + ); + } + } + Err(error) if enabling && error.disposition() == ControllerDisposition::Refused => { + tracing::warn!( + target: "temporal_runtime", + environment_id = %environment.environment_id, + %error, + "the provider refused public ingress; the ingress intent is dropped" + ); + changed |= self.drop_ingress_intent(environment).await; + } + Err(error) => { + tracing::warn!( + target: "temporal_runtime", + environment_id = %environment.environment_id, + %error, + "{}; retried at the next observation", + if enabling { + "enabling public ingress failed" + } else { + "removing public ingress failed" + } + ); + } } + changed + } + + /// Record `realized` as the environment's public endpoint when it + /// differs from `recorded`. + async fn record_realized_ingress( + &self, + environment: &EnvironmentRecord, + recorded: Option<&str>, + realized: &Option, + ) -> bool { + if recorded == realized.as_deref() { + return false; + } + let written = EnvironmentStore::record_environment_ingress( + self.store.as_ref(), + RecordEnvironmentIngress { + environment_id: environment.environment_id.clone(), + public_endpoint: realized.clone(), + observed_at_ms: clock_ms(), + }, + ) .await; - let response = finish_provider_controller(controller, response).await?; - if response.status == ProviderTargetStatus::Closed { - EnvironmentStore::finish_close_environment( - self.store.as_ref(), - FinishCloseEnvironment { - environment_id: environment.environment_id.clone(), - observed_at_ms: now_ms()?, - }, - ) - .await - .map_err(map_environments_error)?; + match written { + Ok(_) => true, + Err(error) => { + tracing::warn!( + target: "temporal_runtime", + environment_id = %environment.environment_id, + %error, + "recording the realized public ingress failed" + ); + false + } } - Ok(true) } -} -fn external_connection_from_api( - value: EnvironmentConnectionView, -) -> Result<::environments::EnvironmentConnectionSpec, AgentApiError> { - let transport = match value.transport { - EnvironmentConnectionTransportView::WebSocket => { - environment_protocol::shared::EnvironmentTransport::WebSocket - } - _ => { - return Err(AgentApiError::invalid_request( - "external environment connections currently require webSocket transport", - )); + /// Clear an ingress intent the provider cannot realize. + async fn drop_ingress_intent(&self, environment: &EnvironmentRecord) -> bool { + let written = EnvironmentStore::set_environment_ingress( + self.store.as_ref(), + SetEnvironmentIngress { + environment_id: environment.environment_id.clone(), + enabled: false, + updated_at_ms: clock_ms(), + }, + ) + .await; + match written { + Ok(_) => true, + Err(error) => { + tracing::warn!( + target: "temporal_runtime", + environment_id = %environment.environment_id, + %error, + "dropping the public ingress intent failed" + ); + false + } } - }; - Ok(::environments::EnvironmentConnectionSpec { - endpoint: value.endpoint, - transport, - }) + } +} + +/// The public endpoint an observation reports realized. +fn realized_ingress(observation: &TargetObservation) -> Option { + observation + .ingress + .as_ref() + .map(|ingress| ingress.endpoint.as_str().to_owned()) } /// Rate-limits repeated reconcile-pass failures per universe so a provider @@ -728,3 +1324,214 @@ fn validated_caller_metadata( super::validate_caller_metadata(&metadata)?; Ok(metadata) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::environments::fake_controller::{FakeController, FakeControllerOptions}; + use environment_protocol::{ + ENVD_PROTOCOL_VERSION, control::target::TargetRef as Ref, metadata::DescriptiveText, + routes::DataRouteBase, + }; + + fn state( + phase: TargetPhase, + power: Option, + daemon_ready: bool, + message: Option<&str>, + ) -> TargetState { + TargetState { + phase, + power, + daemon_ready, + message: message.and_then(DescriptiveText::lossy), + } + } + + #[test] + fn target_states_map_to_lifecycle_verdicts() { + use TargetPhase::*; + assert_eq!( + target_verdict(&state(Provisioning, None, false, None)), + TargetVerdict::Status(EnvironmentStatus::Provisioning) + ); + assert_eq!( + target_verdict(&state(Active, Some(PowerState::Running), false, None)), + TargetVerdict::Status(EnvironmentStatus::Booting) + ); + assert_eq!( + target_verdict(&state(Active, Some(PowerState::Running), true, None)), + TargetVerdict::DaemonReady + ); + assert_eq!( + target_verdict(&state(Active, Some(PowerState::Stopped), false, None)), + TargetVerdict::Status(EnvironmentStatus::Offline) + ); + assert_eq!( + target_verdict(&state(Failed, None, false, Some("disk full"))), + TargetVerdict::Failed("disk full".to_owned()) + ); + assert!(matches!( + target_verdict(&state(Closed, None, false, None)), + TargetVerdict::Failed(_) + )); + assert!(matches!( + target_verdict(&state(Unknown, None, false, None)), + TargetVerdict::Unknown(_) + )); + assert!(matches!( + target_verdict(&state(Active, None, false, None)), + TargetVerdict::Unknown(_) + )); + } + + #[test] + fn the_provisioning_deadline_counts_from_the_first_error_or_stall() { + let error = EnvironmentLifecycleError::record(None, "provider unreachable", 1_000); + assert!(!provisioning_deadline_passed(None, i64::MAX)); + assert!(!provisioning_deadline_passed( + Some(&error), + 1_000 + PROVISIONING_DEADLINE_MS - 1 + )); + let later = EnvironmentLifecycleError::record(Some(&error), "still unreachable", 50_000); + assert_eq!(later.first_at_ms, 1_000); + assert!(provisioning_deadline_passed( + Some(&later), + 1_000 + PROVISIONING_DEADLINE_MS + )); + } + + #[test] + fn stalled_observations_carry_the_provider_message_into_the_deadline() { + let booting = waiting_note( + EnvironmentStatus::Booting, + &state( + TargetPhase::Active, + Some(PowerState::Running), + false, + Some("guest agent not up"), + ), + ); + assert_eq!( + booting, + "waiting for envd to answer on the target: guest agent not up" + ); + let provisioning = waiting_note( + EnvironmentStatus::Provisioning, + &state(TargetPhase::Provisioning, None, false, None), + ); + assert_eq!( + provisioning, + "waiting for the provider to provision the target" + ); + // A stalled observation starts the deadline like a transient error: + // the run of notes keeps its first instant. + let first = EnvironmentLifecycleError::record(None, &booting, 1_000); + let repeated = EnvironmentLifecycleError::record(Some(&first), &booting, 900_000); + assert!(provisioning_deadline_passed( + Some(&repeated), + 1_000 + PROVISIONING_DEADLINE_MS + )); + } + + #[test] + fn repeated_errors_are_rewritten_only_when_new_or_stale() { + let recorded = EnvironmentLifecycleError::record(None, "provider unreachable", 1_000); + assert!(lifecycle_error_is_news(None, "provider unreachable", 1_000)); + assert!(!lifecycle_error_is_news( + Some(&recorded), + "provider unreachable", + 2_000 + )); + assert!(lifecycle_error_is_news( + Some(&recorded), + "provider refused", + 2_000 + )); + assert!(lifecycle_error_is_news( + Some(&recorded), + "provider unreachable", + 1_000 + LIFECYCLE_ERROR_REFRESH_MS + )); + } + + async fn created_target(controller: &FakeController) -> (TargetOwner, String, SecretValue) { + let credential = SecretValue::new(controller.credential()); + let (client, initialized) = environment_client::ControllerClient::connect_initialized( + controller.controller_url(), + provider_connect_options(&credential), + ) + .await + .expect("controller handshake"); + let owner = TargetOwner { + universe_id: UniverseId::new(uuid::Uuid::new_v4().to_string()).unwrap(), + binding_id: BindingId::new("primary").unwrap(), + environment_id: environment_protocol::ids::EnvironmentId::new("environment_1").unwrap(), + incarnation_id: IncarnationId::new("incarnation_1").unwrap(), + }; + let observation = client + .create_target(&CreateTargetParams { + owner: owner.clone(), + template_id: TemplateId::new( + crate::environments::fake_controller::FAKE_TEMPLATE_ID, + ) + .unwrap(), + }) + .await + .expect("create") + .target; + let target_id = observation.target_id.expect("target id"); + let base: DataRouteBase = initialized.routes.data; + let endpoint = provider_data_route(&base, &owner, &target_id); + // Observation is a pure read of the same target. + let observed = client + .get_target(&GetTargetParams { + target: Ref { + owner: owner.clone(), + target_id: Some(target_id.clone()), + }, + }) + .await + .expect("get") + .target; + assert_eq!(observed.target_id, Some(target_id)); + let _ = client.close().await; + (owner, endpoint, credential) + } + + #[tokio::test(flavor = "current_thread")] + async fn the_ready_transition_handshake_records_the_daemon_build() { + let controller = FakeController::start().await.unwrap(); + let (_, endpoint, credential) = created_target(&controller).await; + match probe_daemon(&endpoint, &credential).await { + DaemonProbe::Ready(build) => { + assert_eq!(build.protocol_version, ENVD_PROTOCOL_VERSION); + assert!(build.version.is_some()); + } + other => panic!("expected a ready daemon, got {other:?}"), + } + // Without the provider credential the data route is refused. + assert!(matches!( + probe_daemon(&endpoint, &SecretValue::new("wrong")).await, + DaemonProbe::Unreachable(_) + )); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_daemon_of_another_protocol_version_is_a_mismatch_naming_both() { + let controller = FakeController::start_with(FakeControllerOptions { + daemon_protocol_version: ENVD_PROTOCOL_VERSION + 1, + ..FakeControllerOptions::default() + }) + .await + .unwrap(); + let (_, endpoint, credential) = created_target(&controller).await; + assert_eq!( + probe_daemon(&endpoint, &credential).await, + DaemonProbe::Mismatch { + local: ENVD_PROTOCOL_VERSION, + remote: ENVD_PROTOCOL_VERSION + 1, + } + ); + } +} diff --git a/crates/temporal-runtime/src/environments/mod.rs b/crates/temporal-runtime/src/environments/mod.rs index 6164bdca0..6d952e7a1 100644 --- a/crates/temporal-runtime/src/environments/mod.rs +++ b/crates/temporal-runtime/src/environments/mod.rs @@ -2,11 +2,12 @@ use crate::environments::gateway::EnvironmentGatewayClientConfig; use api::*; use lifecycle::parse_registry_environment_id; -use provider_controllers::{ProviderControllerConnector, finish_provider_controller}; -use providers::{map_environments_error, parse_environment_provider_id}; +use provider_controllers::ProviderControllerConnector; +use providers::map_environments_error; use std::{collections::BTreeMap, sync::Arc}; use store_pg::PgStore; pub(crate) mod credentials; +pub mod fake_controller; pub(crate) mod lifecycle; pub(crate) mod power; pub(crate) mod provider_controllers; @@ -37,6 +38,6 @@ fn now_ms() -> Result { } fn validate_caller_metadata(metadata: &BTreeMap) -> Result<(), AgentApiError> { - environment_protocol::registration::validate_registration_metadata(None, metadata) - .map_err(|message| AgentApiError::invalid_request(format!("invalid metadata: {message}"))) + environment_protocol::metadata::validate_caller_metadata(metadata) + .map_err(|error| AgentApiError::invalid_request(format!("invalid metadata: {error}"))) } diff --git a/crates/temporal-runtime/src/environments/power.rs b/crates/temporal-runtime/src/environments/power.rs index 1932f8816..7701ee00c 100644 --- a/crates/temporal-runtime/src/environments/power.rs +++ b/crates/temporal-runtime/src/environments/power.rs @@ -10,17 +10,19 @@ use super::providers::map_environments_error; use super::*; use ::environments::{ - BeginCloseEnvironment, EnvironmentRecord, EnvironmentStore, IdleAction, PowerState, - SetEnvironmentPower, -}; -use environment_client::EnvironmentDataClient; -use environment_protocol::{ - data::{ - handshake::{InitializeParams, InitializedParams}, - idle::{IdleParams, IdleResponse}, - }, - shared::CURRENT_PROTOCOL_VERSION, + BeginCloseEnvironment, EnvironmentProviderId, EnvironmentRecord, EnvironmentStore, IdleAction, + PowerState, SetEnvironmentPower, }; +use environment_client::{ClientError, DataClient}; +use environment_protocol::data::idle::{GetIdleParams, GetIdleResponse}; +use std::{collections::HashSet, time::Duration}; + +/// `clientInfo.name` of the reaper's data-plane connections. +const POWER_REAPER_CLIENT: &str = "lightspeed-power-reaper"; + +/// Deadline of one `env/getIdle`, on top of the connect and `initialize` +/// bounds. +const IDLE_REPORT_TIMEOUT: Duration = Duration::from_secs(10); /// Outcome of one reaper pass for reporting/tests. #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -32,15 +34,20 @@ pub struct PowerReaperStats { pub closed: usize, } +/// No process or job is running. +fn is_quiescent(report: &GetIdleResponse) -> bool { + report.running_process_count == 0 && report.running_job_count == 0 +} + /// Decide the idle action for one environment from the daemon's report. /// Pure so it can be tested without a daemon: nothing is due while work /// runs, while a power change is already pending, or before the first /// supported stage threshold. pub(crate) fn decide_idle_action( environment: &EnvironmentRecord, - report: &IdleResponse, + report: &GetIdleResponse, ) -> Option { - if !report.is_quiescent() { + if !is_quiescent(report) { return None; } if environment.desired_power != PowerState::Running { @@ -50,15 +57,27 @@ pub(crate) fn decide_idle_action( policy.due_action(report.idle_for_ms, &environment.incarnation.power_states) } +/// Why an idle report could not be read. +enum IdleReportFailure { + /// The daemon did not answer within its deadlines: its provider's + /// remaining environments are skipped for the rest of the pass. + TimedOut, + Unavailable, +} + impl EnvironmentService { /// Public entry point for one reaper pass; used by acceptance tests that /// drive the reaper deterministically instead of running the loop. pub async fn reap_idle_environments_once(&self) -> Result { - self.reconcile_idle_power_once().await + self.reconcile_idle_power_pass(&mut HashSet::new()).await } - pub(crate) async fn reconcile_idle_power_once( + /// One reaper pass over this universe. `unhealthy` names providers + /// whose daemons timed out earlier in the deployment pass; their + /// environments are left for the next pass. + pub(crate) async fn reconcile_idle_power_pass( &self, + unhealthy: &mut HashSet, ) -> Result { let candidates = EnvironmentStore::list_environments_with_idle_policy(self.store.as_ref()) .await @@ -69,15 +88,27 @@ impl EnvironmentService { }; let universe_id = self.store.config().universe_id; for environment in candidates { + let provider_id = environment.provider_id().cloned(); + if provider_id + .as_ref() + .is_some_and(|provider_id| unhealthy.contains(provider_id)) + { + stats.unreachable += 1; + continue; + } let report = match self.read_idle_report(universe_id, &environment).await { - Some(report) => report, - None => { + Ok(report) => report, + Err(failure) => { + if let (IdleReportFailure::TimedOut, Some(provider_id)) = (failure, provider_id) + { + unhealthy.insert(provider_id); + } stats.unreachable += 1; continue; } }; let Some(action) = decide_idle_action(&environment, &report) else { - if !report.is_quiescent() { + if !is_quiescent(&report) { stats.busy += 1; } continue; @@ -132,37 +163,33 @@ impl EnvironmentService { } /// Ask the environment's daemon for its idle report through the ordinary - /// on-demand route. `None` when the daemon is unreachable or predates - /// `env/idle`; the reaper then leaves the environment alone. + /// on-demand route, within the connect, `initialize` and report + /// deadlines, and close the connection again. The handshake and the + /// report are observations: reading them never resets the idle clock. async fn read_idle_report( &self, universe_id: uuid::Uuid, environment: &EnvironmentRecord, - ) -> Option { - let connection = self + ) -> Result { + let endpoint = self + .environment_gateway + .endpoint_for(universe_id, environment); + let options = self .environment_gateway - .connection_for(universe_id, environment); - let mut client = EnvironmentDataClient::connect( - &connection.endpoint, - self.environment_gateway - .connect_options("lightspeed-power-reaper"), - ) - .await - .ok()?; - let response = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-power-reaper".to_owned(), - scope: connection.scope.clone(), - resume_connection_id: None, - }) + .connect_options(POWER_REAPER_CLIENT) + .with_request_timeout(IDLE_REPORT_TIMEOUT); + let failure = |error: &ClientError| match error { + ClientError::ConnectTimeout { .. } | ClientError::DeadlineExceeded { .. } => { + IdleReportFailure::TimedOut + } + _ => IdleReportFailure::Unavailable, + }; + let (client, _) = DataClient::connect_initialized(&endpoint, options) .await - .ok()?; - if response.protocol_version != CURRENT_PROTOCOL_VERSION { - return None; - } - client.initialized(&InitializedParams {}).await.ok()?; - client.idle(&IdleParams {}).await.ok() + .map_err(|error| failure(&error))?; + let report = client.get_idle(&GetIdleParams {}).await; + let _ = client.close().await; + report.map_err(|error| failure(&error)) } } @@ -196,7 +223,9 @@ mod tests { incarnation: EnvironmentIncarnationRecord { incarnation_id: EnvironmentIncarnationId::new("incarnation-1"), provision_request_id: Some(EnvironmentProvisionRequestId::new("request-1")), - provider_target_id: Some("target-1".into()), + provider_target_id: Some( + environment_protocol::ids::ProviderTargetId::new("target-1").unwrap(), + ), template_id: Some(EnvironmentTemplateId::new("dev")), adoption_source_target: None, power_states, @@ -207,17 +236,18 @@ mod tests { public_endpoint: None, metadata: BTreeMap::new(), last_seen_at_ms: None, + lifecycle_error: None, created_at_ms: 1, updated_at_ms: 1, } } - fn report(idle_for_ms: u64, running_processes: u32, running_jobs: u32) -> IdleResponse { - IdleResponse { + fn report(idle_for_ms: u64, running_processes: u32, running_jobs: u32) -> GetIdleResponse { + GetIdleResponse { idle_for_ms, - running_processes, - running_jobs, - leftover_process_groups: 0, + running_process_count: running_processes, + running_job_count: running_jobs, + leftover_process_group_count: 0, } } @@ -238,6 +268,13 @@ mod tests { ); // Running work blocks every stage regardless of the clock. assert_eq!(decide_idle_action(&env, &report(10_000, 1, 0)), None); + // Leftover process groups are services left behind, not running work. + let mut leftovers = report(1_500, 0, 0); + leftovers.leftover_process_group_count = 3; + assert_eq!( + decide_idle_action(&env, &leftovers), + Some(IdleAction::Pause) + ); assert_eq!(decide_idle_action(&env, &report(10_000, 0, 2)), None); // Suspend is due but unsupported by Incus: pause remains. assert_eq!( diff --git a/crates/temporal-runtime/src/environments/prompts.rs b/crates/temporal-runtime/src/environments/prompts.rs index f87168c1c..7dc62bd2a 100644 --- a/crates/temporal-runtime/src/environments/prompts.rs +++ b/crates/temporal-runtime/src/environments/prompts.rs @@ -20,17 +20,17 @@ pub(crate) async fn refresh( let connection = discovery.connection().await?; let query = tools::environment::sources::scan_query( source.roots.as_deref(), - &connection.cwd, - connection.initialized.home_directory.as_deref(), + connection.cwd.as_str(), + connection + .initialized + .home_directory + .as_ref() + .map(|home| home.as_str()), "prompts", )?; let scan = { let _timer = PhaseTimer::new("prompts_scan"); - connection - .client - .scan(&query) - .await - .map_err(|e| e.to_string())? + connection.scan(query).await? }; tools::prompts::environment::assemble(&scan) }; @@ -53,10 +53,23 @@ mod tests { ENVIRONMENT_PROMPT_CONTEXT_KEY, EnvironmentPromptReport, assemble, publication, }; + fn daemon(root: &std::path::Path, cwd: &std::path::Path) -> environment_daemon::DaemonRuntime { + environment_daemon::DaemonRuntime::new(environment_daemon::config::DaemonConfig { + listen: None, + cwd: cwd.to_owned(), + fs_root: root.to_owned(), + state_dir: root.parent().unwrap().join("state"), + read_only_fs: true, + registration: None, + scrubbed_env: Vec::new(), + }) + .unwrap() + } + #[tokio::test(flavor = "current_thread")] async fn local_environment_prompts_scan_in_order_and_replace_on_failure() { let directory = tempfile::tempdir().unwrap(); - let root = directory.path().canonicalize().unwrap(); + let root = directory.path().canonicalize().unwrap().join("root"); let cwd = root.join("project"); let home = root.join("home"); std::fs::create_dir_all(cwd.join(".agents/prompts")).unwrap(); @@ -85,8 +98,7 @@ mod tests { ) .unwrap(); std::fs::write(cwd.join(".agents/prompts/notes.json"), "Must not load").unwrap(); - let fs = - environment_daemon::filesystem::LocalFileSystem::new(root.clone(), cwd.clone(), true); + let runtime = daemon(&root, &cwd); let query = tools::environment::sources::scan_query( None, cwd.to_str().unwrap(), @@ -94,15 +106,23 @@ mod tests { "prompts", ) .unwrap(); - let scan = fs.scan(query).await.unwrap(); - let (text, paths) = assemble(&scan).unwrap(); + let scan = runtime + .filesystem() + .scan( + &environment_daemon::dispatch::RequestContext::detached(), + query, + ) + .await + .unwrap(); + let observation = assemble(&scan).unwrap(); assert_eq!( - text, + observation.text, "First instructions\n\nLast instructions\n\nBase instructions" ); - assert_eq!(paths.len(), 3); + assert_eq!(observation.paths.len(), 3); + assert!(observation.warnings.is_empty()); let blobs = InMemoryBlobStore::new(); - let entries = publication(&blobs, "machine", Ok((text, paths))) + let entries = publication(&blobs, "machine", Ok(observation.clone())) .await .unwrap(); let entry = &entries[&ContextEntryKey::new(ENVIRONMENT_PROMPT_CONTEXT_KEY)]; @@ -114,10 +134,36 @@ mod tests { ) .unwrap(); assert!(report.available); - let mut incomplete = scan.clone(); - incomplete.complete = false; - assert!(assemble(&incomplete).is_err()); - let failed = publication(&blobs, "other-machine", assemble(&incomplete)) + // A root that cannot be scanned is a warning naming it; the other + // roots' prompts still load. + let mut partial = scan.clone(); + let failed_root = environment_protocol::paths::EnvironmentPath::new( + home.join(".agents/prompts").to_str().unwrap(), + ) + .unwrap(); + partial + .roots + .push(environment_protocol::data::scan::ScanRootResult::Failed { + path: failed_root.clone(), + error: environment_protocol::error::ProtocolError::deadline_exceeded("slow"), + }); + let degraded = assemble(&partial).unwrap(); + assert_eq!(degraded.text, observation.text); + assert_eq!(degraded.warnings.len(), 1); + assert!(degraded.warnings[0].starts_with(failed_root.as_str())); + // Prompts need a full observation of every root. + let mut unchanged = scan.clone(); + unchanged.roots.push( + environment_protocol::data::scan::ScanRootResult::Unchanged { + path: failed_root, + fingerprint: environment_protocol::data::scan::ScanFingerprint::new(format!( + "scan:v1:{}", + "0".repeat(64) + )) + .unwrap(), + }, + ); + let failed = publication(&blobs, "other-machine", assemble(&unchanged)) .await .unwrap(); let failed = &failed[&ContextEntryKey::new(ENVIRONMENT_PROMPT_CONTEXT_KEY)]; @@ -130,10 +176,14 @@ mod tests { assert!(text.contains("unavailable")); assert!(!text.contains("Base instructions")); assert!( - publication(&blobs, "machine", Ok((String::new(), vec![]))) - .await - .unwrap() - .is_empty() + publication( + &blobs, + "machine", + Ok(tools::prompts::environment::PromptObservation::default()) + ) + .await + .unwrap() + .is_empty() ); let feature = EnvironmentsFeature { skills: Some(Default::default()), diff --git a/crates/temporal-runtime/src/environments/provider_controllers.rs b/crates/temporal-runtime/src/environments/provider_controllers.rs index dd91f8b39..d9050e271 100644 --- a/crates/temporal-runtime/src/environments/provider_controllers.rs +++ b/crates/temporal-runtime/src/environments/provider_controllers.rs @@ -1,701 +1,851 @@ +//! Provider controllers: dialing a provider's controller with its +//! credential, recording what its `initialize` advertises, and classifying +//! what goes wrong. +//! +//! Every connection runs the controller handshake through +//! [`ControllerClient::connect_initialized`]. At each successful handshake +//! the advertised `routes.data` passes the downgrade rule and, together with +//! the optional features, is recorded on the provider record; the gateway's +//! provider data proxy and the reconciler's data-plane handshake build +//! every target's data route from that recorded base, never from the +//! controller URL. + use super::*; -use ::environments::EnvironmentConnectionSpec; -use async_trait::async_trait; -use environment_client::{ - EnvironmentClientError, EnvironmentProviderClient, WebSocketConnectOptions, +use ::environments::{ + ControllerFeatures, EndpointScheme, EnvironmentEndpoint, EnvironmentProviderId, + EnvironmentProviderStore, EnvironmentRegistryError, ListEnvironmentProviders, + RecordProviderController, }; +use async_trait::async_trait; +use auth::SecretValue; +use environment_client::{ClientError, ConnectOptions, ControllerClient}; use environment_protocol::{ - control::{ - handshake::{ControllerInitializeParams, ControllerInitializeResponse}, - ingress::{ - EnsureIngressParams, IngressResponse, ProviderIngressStatus, RemoveIngressParams, - }, - targets::{ - AdoptTargetParams, AdoptTargetResponse, CloseTargetParams, CloseTargetResponse, - CreateTargetParams, CreateTargetResponse, EnvironmentTemplate, ListTemplatesParams, - ListTemplatesResponse, PowerState, ProviderTargetStatus, ProviderTargetSummary, - SetTargetPowerParams, SetTargetPowerResponse, - }, - }, - shared::{ - CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentScope, EnvironmentTransport, - ImplementationInfo, ProviderTargetId, - }, + control::handshake::ControllerInitializeResponse, + error::{ErrorData, PreconditionReason, ProtocolErrorKind}, + routes::{DataRouteBase, UrlError}, }; - use std::{ - collections::{BTreeMap, hash_map::DefaultHasher}, - hash::{Hash, Hasher}, - sync::Arc, + collections::HashMap, + time::{Duration, Instant}, }; - -#[async_trait] -pub(crate) trait ProviderController: Send { - async fn close(&mut self) -> Result<(), AgentApiError> { - Ok(()) - } - - async fn initialize( - &mut self, - params: &ControllerInitializeParams, - ) -> Result; - - async fn create_target( - &mut self, - params: &CreateTargetParams, - ) -> Result; - - async fn adopt_target( - &mut self, - params: &AdoptTargetParams, - ) -> Result; - - async fn list_templates( - &mut self, - params: &ListTemplatesParams, - ) -> Result; - - async fn close_target( - &mut self, - params: &CloseTargetParams, - ) -> Result; - async fn set_target_power( - &mut self, - params: &SetTargetPowerParams, - ) -> Result; - async fn ensure_ingress( - &mut self, - params: &EnsureIngressParams, - ) -> Result; - async fn remove_ingress( - &mut self, - params: &RemoveIngressParams, - ) -> Result; +use thiserror::Error; + +/// Deadline of every controller call. Controllers answer target methods +/// promptly and continue slow work in the background, so this bounds a +/// hung controller, not a slow create. +pub(crate) const CONTROLLER_REQUEST_TIMEOUT: Duration = Duration::from_secs(120); + +/// Options for one controller or provider data-route connection: the +/// provider credential as the bearer token, the default connect and +/// `initialize` bounds, and [`CONTROLLER_REQUEST_TIMEOUT`] on every call. +pub(crate) fn provider_connect_options(credential: &SecretValue) -> ConnectOptions { + ConnectOptions::new(super::gateway::runtime_client_info()) + .with_bearer_token(credential.expose()) + .with_request_timeout(CONTROLLER_REQUEST_TIMEOUT) } +/// Dials a provider controller and runs its handshake. Tests substitute an +/// in-process client. #[async_trait] pub(crate) trait ProviderControllerConnector: Send + Sync { async fn connect( &self, - connection: &EnvironmentConnectionSpec, - ) -> Result, AgentApiError>; + endpoint: &EnvironmentEndpoint, + options: ConnectOptions, + ) -> Result; } -#[derive(Default)] -pub(crate) struct WebSocketProviderControllerConnector { - fake_backend: Arc>, -} +/// The production connector: `ws://`, `wss://` and `unix://` controller +/// URLs through [`ControllerClient::connect_initialized`]. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct WebSocketProviderControllerConnector; #[async_trait] impl ProviderControllerConnector for WebSocketProviderControllerConnector { async fn connect( &self, - connection: &EnvironmentConnectionSpec, - ) -> Result, AgentApiError> { - let mut controller: Box = match &connection.transport { - EnvironmentTransport::WebSocket => { - let client = EnvironmentProviderClient::connect( - &connection.endpoint, - WebSocketConnectOptions { - bearer_token: None, - user_agent: Some("lightspeed-temporal-runtime".to_owned()), - headers: Vec::new(), - }, - ) - .await - .map_err(map_environment_client_error)?; - Box::new(client) - } - EnvironmentTransport::Http => return Err(unsupported_transport("http")), - EnvironmentTransport::Stdio => return Err(unsupported_transport("stdio")), - EnvironmentTransport::Ssh => return Err(unsupported_transport("ssh")), - EnvironmentTransport::Provider { provider_type } => { - if provider_type == "fake" { - Box::new(FakeProviderController { - backend: self.fake_backend.clone(), - }) - } else { - return Err(unsupported_transport(format!("provider:{provider_type}"))); - } - } - }; - let initialized = controller - .initialize(&ControllerInitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-temporal-runtime".to_owned(), - }) - .await; - let initialized = match initialized { - Ok(initialized) => initialized, - Err(error) => { - let _ = controller.close().await; - return Err(error); - } - }; - if initialized.protocol_version != CURRENT_PROTOCOL_VERSION { - let _ = controller.close().await; - return Err(AgentApiError::rejected(format!( - "environment provider protocol version {} is incompatible with {}", - initialized.protocol_version, CURRENT_PROTOCOL_VERSION, - ))); - } - Ok(controller) + endpoint: &EnvironmentEndpoint, + options: ConnectOptions, + ) -> Result { + ControllerClient::connect_initialized(endpoint.as_str(), options) + .await + .map(|(client, _)| client) } } -/// Power states the in-process fake provider advertises. Suspended is -/// deliberately included so the Firecracker-shaped path has coverage. -const FAKE_POWER_STATES: [PowerState; 4] = [ - PowerState::Running, - PowerState::Paused, - PowerState::Suspended, - PowerState::Stopped, -]; +/// The facts a controller's `initialize` advertises, checked: the data +/// route base must not downgrade a `wss` controller to `ws`. A controller +/// on a Unix socket is local, so any base passes. +pub(crate) fn controller_facts( + endpoint: &EnvironmentEndpoint, + initialize: &ControllerInitializeResponse, +) -> Result<(ControllerFeatures, DataRouteBase), UrlError> { + if endpoint.scheme() != EndpointScheme::Unix { + initialize + .routes + .data + .check_for_controller(endpoint.as_str())?; + } + Ok(( + ControllerFeatures { + adopt_target: initialize.capabilities.adopt_target, + ingress: initialize.capabilities.ingress, + }, + initialize.routes.data.clone(), + )) +} -#[derive(Default)] -struct FakeBackend { - targets: BTreeMap, - requests: BTreeMap<(String, String, String), String>, +/// An optional controller feature the runtime gates calls on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ControllerFeature { + AdoptTarget, + Ingress, } -#[derive(Clone)] -struct FakeTarget { - universe_id: String, - binding_id: String, - environment_id: String, - incarnation_id: String, - response: CreateTargetResponse, +impl ControllerFeature { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::AdoptTarget => "target adoption", + Self::Ingress => "public ingress", + } + } + + pub(crate) fn in_features(self, features: &ControllerFeatures) -> bool { + match self { + Self::AdoptTarget => features.adopt_target, + Self::Ingress => features.ingress, + } + } } -struct FakeProviderController { - backend: Arc>, +/// One initialized controller connection and what its handshake advertised. +#[derive(Clone)] +pub(crate) struct ProviderController { + pub(crate) provider_id: EnvironmentProviderId, + pub(crate) client: ControllerClient, + pub(crate) features: ControllerFeatures, + pub(crate) data_route_base: DataRouteBase, + /// The provider credential, presented on its data routes too. + pub(crate) credential: SecretValue, } -#[async_trait] -impl ProviderController for FakeProviderController { - async fn initialize( - &mut self, - _params: &ControllerInitializeParams, - ) -> Result { - Ok(ControllerInitializeResponse { - protocol_version: CURRENT_PROTOCOL_VERSION, - capabilities: environment_protocol::control::handshake::ControllerCapabilities { - list_templates: true, - list_targets: true, - create_target: true, - adopt_target: true, - get_target: true, - close_target: true, - set_target_power: true, - ingress: true, - }, - implementation: ImplementationInfo { - name: "lightspeed-fake-environment-provider".to_owned(), - version: Some(env!("CARGO_PKG_VERSION").to_owned()), - ..Default::default() - }, - }) +impl ProviderController { + /// Refuse a call that needs an optional feature the controller did not + /// advertise, before anything is sent. + pub(crate) fn require(&self, feature: ControllerFeature) -> Result<(), ControllerError> { + if feature.in_features(&self.features) { + Ok(()) + } else { + Err(ControllerError::FeatureUnavailable { + provider_id: self.provider_id.to_string(), + feature, + }) + } } - async fn list_templates( - &mut self, - _params: &ListTemplatesParams, - ) -> Result { - Ok(ListTemplatesResponse { - templates: vec![EnvironmentTemplate { - template_id: "rust-v1".to_owned(), - display_name: "Rust development".to_owned(), - description: Some("Deterministic fake provider template".to_owned()), - public_ingress: false, - deprecated: false, - metadata: BTreeMap::new(), - }], - }) - } - - async fn create_target( - &mut self, - params: &CreateTargetParams, - ) -> Result { - if params.template_id != "rust-v1" && !params.template_id.starts_with("adopted:") { - return Err(AgentApiError::rejected( - "fake provider policy rejected the template", - )); - } - let key = ( - params.binding.universe_id.clone(), - params.binding.binding_id.clone(), - params.request_id.clone(), - ); - let mut backend = self.backend.lock().await; - if let Some(target_id) = backend.requests.get(&key) { - return Ok(backend - .targets - .get(target_id) - .expect("fake request index target") - .response - .clone()); + /// Wrap a failed call of this controller. + pub(crate) fn call_error(&self, source: ClientError) -> ControllerError { + ControllerError::Call { + provider_id: self.provider_id.to_string(), + source, } - if backend.targets.values().any(|target| { - target.universe_id == params.binding.universe_id - && target.binding_id == params.binding.binding_id - && target.response.target.status != ProviderTargetStatus::Closed - }) { - return Err(AgentApiError::rejected( - "fake provider policy allows one active target per binding", - )); - } - let mut identity = DefaultHasher::new(); - params.binding.universe_id.hash(&mut identity); - params.binding.binding_id.hash(&mut identity); - params.incarnation_id.hash(&mut identity); - let target_id = format!("fake-{:016x}", identity.finish()); - let provider_target_id = ProviderTargetId::new(target_id.clone()); - let capabilities = EnvironmentCapabilities::filesystem(true, true).with_process(); - let response = CreateTargetResponse { - target: ProviderTargetSummary { - target_id: provider_target_id.clone(), - display_name: Some(params.environment_id.clone()), - status: ProviderTargetStatus::Ready, - scope: EnvironmentScope::Default, - capabilities: capabilities.clone(), - default_cwd: None, - power_states: FAKE_POWER_STATES.to_vec(), - metadata: BTreeMap::from([( - "imageFingerprint".to_owned(), - format!("fake:{}", params.template_id), - )]), - }, - }; - backend.requests.insert(key, target_id.clone()); - backend.targets.insert( - target_id, - FakeTarget { - universe_id: params.binding.universe_id.clone(), - binding_id: params.binding.binding_id.clone(), - environment_id: params.environment_id.clone(), - incarnation_id: params.incarnation_id.clone(), - response: response.clone(), - }, - ); - Ok(response) } - async fn adopt_target( - &mut self, - params: &AdoptTargetParams, - ) -> Result { - let create = CreateTargetParams { - request_id: params.request_id.clone(), - environment_id: params.environment_id.clone(), - incarnation_id: params.incarnation_id.clone(), - binding: params.binding.clone(), - template_id: format!("adopted:{}", params.source_target), - }; - let response = self.create_target(&create).await?; - Ok(AdoptTargetResponse { - target: response.target, - }) + pub(crate) async fn close(&self) { + let _ = self.client.close().await; } +} - async fn close_target( - &mut self, - params: &CloseTargetParams, - ) -> Result { - let mut backend = self.backend.lock().await; - let target = backend - .targets - .get_mut(params.target_id.as_str()) - .ok_or_else(|| { - AgentApiError::not_found(format!("fake target not found: {}", params.target_id)) - })?; - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - { - return Err(AgentApiError::rejected( - "fake target ownership metadata does not match close request", - )); +/// Connect to a provider's controller with its stored credential, check the +/// advertised facts and record them on the provider record when they +/// changed. +pub(crate) async fn connect_provider_controller( + store: &dyn EnvironmentProviderStore, + connector: &dyn ProviderControllerConnector, + provider_id: &EnvironmentProviderId, +) -> Result { + let provider = store.read_provider(provider_id).await?; + let credential = store.read_provider_credential(provider_id).await?; + let client = connector + .connect( + &provider.controller_endpoint, + provider_connect_options(&credential), + ) + .await + .map_err(|source| ControllerError::Connect { + provider_id: provider_id.to_string(), + source, + })?; + let (features, data_route_base) = + match controller_facts(&provider.controller_endpoint, client.initialize_response()) { + Ok(facts) => facts, + Err(source) => { + let _ = client.close().await; + return Err(ControllerError::Routes { + provider_id: provider_id.to_string(), + source, + }); + } + }; + if provider.controller_features != Some(features) + || provider.data_route_base.as_ref() != Some(&data_route_base) + { + let recorded = store + .record_provider_controller(RecordProviderController { + provider_id: provider_id.clone(), + controller_endpoint: provider.controller_endpoint.clone(), + features, + data_route_base: data_route_base.clone(), + }) + .await; + if let Err(error) = recorded { + let _ = client.close().await; + return Err(error.into()); } - target.response.target.status = ProviderTargetStatus::Closed; - Ok(CloseTargetResponse { - target_id: params.target_id.clone(), - status: ProviderTargetStatus::Closed, - }) } + Ok(ProviderController { + provider_id: provider_id.clone(), + client, + features, + data_route_base, + credential, + }) +} - /// The fake provider converges instantly: the requested steady state is - /// the observed state on return. - async fn set_target_power( - &mut self, - params: &SetTargetPowerParams, - ) -> Result { - if !FAKE_POWER_STATES.contains(¶ms.power) { - return Err(AgentApiError::rejected(format!( - "fake provider does not support power state {}", - params.power - ))); - } - let mut backend = self.backend.lock().await; - let target = backend - .targets - .get_mut(params.target_id.as_str()) - .ok_or_else(|| { - AgentApiError::not_found(format!("fake target not found: {}", params.target_id)) - })?; - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - { - return Err(AgentApiError::rejected( - "fake target ownership metadata does not match power request", - )); +/// Check a controller endpoint at provider registration. A controller that +/// answers definitively wrong (an HTTP refusal such as 404 for a URL that +/// names no controller or 401 for a wrong credential, another plane, or a +/// data route base that fails the downgrade rule) is refused, so a +/// mis-registered provider never provisions anything. One that cannot be +/// reached right now is accepted; its facts are recorded at the first +/// successful handshake. +pub(crate) async fn probe_controller( + connector: &dyn ProviderControllerConnector, + endpoint: &EnvironmentEndpoint, + credential: &SecretValue, +) -> Result, AgentApiError> { + match connector + .connect(endpoint, provider_connect_options(credential)) + .await + { + Ok(client) => { + let facts = controller_facts(endpoint, client.initialize_response()); + let _ = client.close().await; + facts.map(Some).map_err(|error| { + AgentApiError::invalid_request(format!( + "the provider controller advertised an unusable data route: {error}" + )) + }) } - if target.response.target.status == ProviderTargetStatus::Closed { - return Err(AgentApiError::rejected("fake target is closed")); + Err(error) if refuses_registration(&error) => Err(AgentApiError::invalid_request(format!( + "the provider controller at {endpoint} refused the handshake: {error}" + ))), + Err(error) => { + tracing::warn!( + target: "temporal_runtime", + %endpoint, + %error, + "provider controller is not reachable at registration; its features are recorded at the first successful handshake" + ); + Ok(None) } - target.response.target.status = match params.power { - PowerState::Running => ProviderTargetStatus::Ready, - PowerState::Paused => ProviderTargetStatus::Paused, - PowerState::Suspended => ProviderTargetStatus::Suspended, - PowerState::Stopped => ProviderTargetStatus::Stopped, - }; - Ok(SetTargetPowerResponse { - target: target.response.target.clone(), - }) } +} - async fn ensure_ingress( - &mut self, - _params: &EnsureIngressParams, - ) -> Result { - Ok(IngressResponse { - status: ProviderIngressStatus::Ready, - public_endpoint: Some("https://fake.env.test".to_owned()), - }) +/// Handshake failures that prove the registration wrong rather than the +/// controller temporarily unavailable. +fn refuses_registration(error: &ClientError) -> bool { + match error { + ClientError::UpgradeRejected { status } => { + (400..500).contains(status) && !matches!(status, 408 | 429) + } + ClientError::WrongPlane { .. } | ClientError::Configuration { .. } => true, + _ => false, } +} - async fn remove_ingress( - &mut self, - _params: &RemoveIngressParams, - ) -> Result { - Ok(IngressResponse { - status: ProviderIngressStatus::Disabled, - public_endpoint: None, - }) - } +/// How the reconciler treats a failed controller interaction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ControllerDisposition { + /// May succeed later unchanged: keep the status, record the error and + /// retry next pass, until the provisioning deadline. + Transient, + /// The provider refused the request; repeating it cannot help. + Refused, + /// The provider holds no such target. + Gone, } -#[async_trait] -impl ProviderController for EnvironmentProviderClient -where - T: environment_client::JsonRpcTransport + Send, -{ - async fn close(&mut self) -> Result<(), AgentApiError> { - EnvironmentProviderClient::close(self) - .await - .map_err(map_environment_client_error) +#[derive(Debug, Error)] +pub(crate) enum ControllerError { + /// Dialing the controller or its `initialize` failed, a version or + /// plane mismatch included. Always transient: a provider restart or a + /// rollout that upgrades the runtime first must not fail environments. + #[error("provider controller {provider_id} is unreachable: {source}")] + Connect { + provider_id: String, + #[source] + source: ClientError, + }, + + /// The controller advertised a data route base that fails the + /// downgrade rule; treated like a failed `initialize`. + #[error("provider controller {provider_id} advertised an unusable data route: {source}")] + Routes { + provider_id: String, + #[source] + source: UrlError, + }, + + /// A call on an initialized connection failed. + #[error("provider controller {provider_id}: {source}")] + Call { + provider_id: String, + #[source] + source: ClientError, + }, + + /// The call needs an optional feature the controller did not + /// advertise; nothing was sent. + #[error("provider {provider_id} does not offer {}", feature.as_str())] + FeatureUnavailable { + provider_id: String, + feature: ControllerFeature, + }, + + /// An earlier failure in this reconcile pass marked the provider + /// unhealthy; its remaining environments are not attempted. + #[error("provider controller {provider_id} is skipped for this pass: {message}")] + Skipped { + provider_id: String, + message: String, + }, + + #[error(transparent)] + Registry(#[from] EnvironmentRegistryError), +} + +impl ControllerError { + /// Refusals are the controller's typed answers that no retry changes. + /// Connection failures, deadlines, undecodable answers and retryable or + /// `internal` answers are transient; `notFound` means the target is + /// gone. Registry failures are the runtime's own and transient. + pub(crate) fn disposition(&self) -> ControllerDisposition { + match self { + Self::Call { source, .. } => call_disposition(source), + Self::FeatureUnavailable { .. } => ControllerDisposition::Refused, + Self::Connect { .. } | Self::Routes { .. } | Self::Skipped { .. } => { + ControllerDisposition::Transient + } + Self::Registry(_) => ControllerDisposition::Transient, + } } - async fn initialize( - &mut self, - params: &ControllerInitializeParams, - ) -> Result { - EnvironmentProviderClient::initialize(self, params) - .await - .map_err(map_environment_client_error) + /// True when the controller connection itself failed (a dial, a + /// deadline, a dropped socket): the provider's remaining environments + /// are skipped for the rest of the pass, so one hung provider costs one + /// deadline per pass instead of one per environment. + pub(crate) fn is_connection_failure(&self) -> bool { + match self { + Self::Connect { .. } | Self::Routes { .. } | Self::Skipped { .. } => true, + Self::Call { source, .. } => !matches!( + source, + ClientError::Protocol(_) | ClientError::Decode(_) | ClientError::Encode(_) + ), + Self::FeatureUnavailable { .. } | Self::Registry(_) => false, + } } - async fn create_target( - &mut self, - params: &CreateTargetParams, - ) -> Result { - EnvironmentProviderClient::create_target(self, params) - .await - .map_err(map_environment_client_error) + /// The precondition a refusal named, if any. + pub(crate) fn precondition(&self) -> Option { + match self { + Self::Call { + source: ClientError::Protocol(error), + .. + } => match error.data { + ErrorData::FailedPrecondition { reason } => Some(reason), + _ => None, + }, + _ => None, + } } - async fn adopt_target( - &mut self, - params: &AdoptTargetParams, - ) -> Result { - EnvironmentProviderClient::adopt_target(self, params) - .await - .map_err(map_environment_client_error) + pub(crate) fn provider_id(&self) -> Option<&str> { + match self { + Self::Connect { provider_id, .. } + | Self::Routes { provider_id, .. } + | Self::Call { provider_id, .. } + | Self::FeatureUnavailable { provider_id, .. } + | Self::Skipped { provider_id, .. } => Some(provider_id), + Self::Registry(_) => None, + } } - async fn list_templates( - &mut self, - params: &ListTemplatesParams, - ) -> Result { - EnvironmentProviderClient::list_templates(self, params) - .await - .map_err(map_environment_client_error) + /// The API error for a caller waiting on this controller interaction. + pub(crate) fn into_api_error(self) -> AgentApiError { + match self { + Self::Registry(error) => super::providers::map_environments_error(error), + other => match other.disposition() { + ControllerDisposition::Refused => AgentApiError::rejected(other.to_string()), + ControllerDisposition::Gone => AgentApiError::not_found(other.to_string()), + ControllerDisposition::Transient => AgentApiError::internal(other.to_string()), + }, + } } +} - async fn close_target( - &mut self, - params: &CloseTargetParams, - ) -> Result { - EnvironmentProviderClient::close_target(self, params) - .await - .map_err(map_environment_client_error) +/// The disposition of a failed call on an initialized connection. +pub(crate) fn call_disposition(error: &ClientError) -> ControllerDisposition { + match error { + ClientError::Protocol(error) => match error.kind() { + ProtocolErrorKind::NotFound => ControllerDisposition::Gone, + ProtocolErrorKind::Internal => ControllerDisposition::Transient, + kind if kind.is_retryable() => ControllerDisposition::Transient, + _ => ControllerDisposition::Refused, + }, + _ => ControllerDisposition::Transient, } +} - async fn set_target_power( +/// The controllers one reconcile pass talks to: each provider is dialed at +/// most once per pass, and a provider whose connection failed is skipped +/// for the rest of the pass. +#[derive(Default)] +pub(crate) struct ProviderPass { + controllers: HashMap, +} + +enum PassController { + Connected(ProviderController), + Unhealthy(String), +} + +impl ProviderPass { + pub(crate) async fn controller( &mut self, - params: &SetTargetPowerParams, - ) -> Result { - EnvironmentProviderClient::set_target_power(self, params) - .await - .map_err(map_environment_client_error) + store: &dyn EnvironmentProviderStore, + connector: &dyn ProviderControllerConnector, + provider_id: &EnvironmentProviderId, + ) -> Result { + match self.controllers.get(provider_id) { + Some(PassController::Connected(controller)) => return Ok(controller.clone()), + Some(PassController::Unhealthy(message)) => { + return Err(ControllerError::Skipped { + provider_id: provider_id.to_string(), + message: message.clone(), + }); + } + None => {} + } + match connect_provider_controller(store, connector, provider_id).await { + Ok(controller) => { + self.controllers.insert( + provider_id.clone(), + PassController::Connected(controller.clone()), + ); + Ok(controller) + } + Err(error) => { + if error.is_connection_failure() { + self.controllers.insert( + provider_id.clone(), + PassController::Unhealthy(error.to_string()), + ); + } + Err(error) + } + } } - async fn ensure_ingress( - &mut self, - params: &EnsureIngressParams, - ) -> Result { - EnvironmentProviderClient::ensure_ingress(self, params) - .await - .map_err(map_environment_client_error) + /// Record a failure of `error`'s provider: a connection failure skips + /// the provider for the rest of the pass. + pub(crate) async fn observe_failure(&mut self, error: &ControllerError) { + if !error.is_connection_failure() { + return; + } + let Some(provider_id) = error.provider_id() else { + return; + }; + let Ok(provider_id) = EnvironmentProviderId::try_new(provider_id) else { + return; + }; + if let Some(PassController::Connected(controller)) = self + .controllers + .insert(provider_id, PassController::Unhealthy(error.to_string())) + { + controller.close().await; + } } - async fn remove_ingress( - &mut self, - params: &RemoveIngressParams, - ) -> Result { - EnvironmentProviderClient::remove_ingress(self, params) - .await - .map_err(map_environment_client_error) + pub(crate) async fn close(self) { + for controller in self.controllers.into_values() { + if let PassController::Connected(controller) = controller { + controller.close().await; + } + } } } -/// Finish one scoped provider-controller operation and close its transport on -/// both success and failure. A close error does not replace the operation's -/// result: controller calls are already complete when this runs. -pub(crate) async fn finish_provider_controller( - mut controller: Box, - result: Result, -) -> Result { - let _ = controller.close().await; - result +/// Dials providers whose controller facts are not recorded yet (never +/// reached, or their endpoint changed and the registration probe could not +/// reach the new one), so the gateway's data proxy gets a data route base +/// even when none of the provider's environments needs convergence. Failed +/// attempts back off per provider. +#[derive(Default)] +pub(crate) struct ProviderFactsRefresh { + listed_at: Option, + retry: HashMap, } -pub(crate) fn map_environment_client_error(error: EnvironmentClientError) -> AgentApiError { - match error { - EnvironmentClientError::Protocol(error) => { - AgentApiError::rejected(format!("provider controller error: {}", error.message)) - } - EnvironmentClientError::TransportClosed => { - AgentApiError::rejected("provider controller disconnected") - } - EnvironmentClientError::Serialize(error) => { - AgentApiError::rejected(format!("provider controller call failed: {error}")) +impl ProviderFactsRefresh { + /// How often the provider list is read. + const LIST_INTERVAL: Duration = Duration::from_secs(5); + const FIRST_RETRY: Duration = Duration::from_secs(5); + const MAX_RETRY: Duration = Duration::from_secs(300); + + /// One refresh step of a deployment pass; connections stay in `pass`. + pub(crate) async fn run( + &mut self, + pass: &mut ProviderPass, + store: &dyn EnvironmentProviderStore, + connector: &dyn ProviderControllerConnector, + ) { + let now = Instant::now(); + if self + .listed_at + .is_some_and(|listed| now.duration_since(listed) < Self::LIST_INTERVAL) + { + return; } - EnvironmentClientError::Transport(message) - | EnvironmentClientError::InvalidMessage(message) => { - AgentApiError::rejected(format!("provider controller call failed: {message}")) + self.listed_at = Some(now); + let providers = match store + .list_providers(ListEnvironmentProviders::default()) + .await + { + Ok(providers) => providers, + Err(error) => { + tracing::warn!(target: "temporal_runtime", %error, "listing providers for a controller handshake failed"); + return; + } + }; + let unrecorded: Vec = providers + .into_iter() + .filter(|provider| provider.data_route_base.is_none()) + .map(|provider| provider.provider_id) + .collect(); + self.retry + .retain(|provider_id, _| unrecorded.contains(provider_id)); + for provider_id in unrecorded { + if self + .retry + .get(&provider_id) + .is_some_and(|(next, _)| now < *next) + { + continue; + } + match pass.controller(store, connector, &provider_id).await { + Ok(_) => { + self.retry.remove(&provider_id); + } + Err(error) => { + let backoff = self + .retry + .get(&provider_id) + .map_or(Self::FIRST_RETRY, |(_, backoff)| { + (*backoff * 2).min(Self::MAX_RETRY) + }); + tracing::debug!( + target: "temporal_runtime", + %provider_id, + %error, + retry_in_s = backoff.as_secs(), + "provider controller handshake failed; its data route base stays unrecorded" + ); + self.retry.insert(provider_id, (now + backoff, backoff)); + } + } } } } -fn unsupported_transport(transport: impl std::fmt::Display) -> AgentApiError { - AgentApiError::invalid_request(format!( - "provider controller transport is not supported by this gateway: {transport}" - )) +impl EnvironmentService { + /// A controller connection for one API request; the caller closes it. + pub(crate) async fn connect_provider_controller( + &self, + provider_id: &EnvironmentProviderId, + ) -> Result { + connect_provider_controller( + self.store.as_ref(), + self.provider_controller_connector.as_ref(), + provider_id, + ) + .await + } + + /// Refuse an API request that needs an optional controller feature the + /// provider does not offer. Features recorded at the last handshake + /// decide; a provider not reached yet is dialed once to learn them. + pub(crate) async fn require_controller_feature( + &self, + provider_id: &EnvironmentProviderId, + feature: ControllerFeature, + ) -> Result<(), AgentApiError> { + let provider = self.read_environment_provider(provider_id).await?; + let features = match provider.controller_features { + Some(features) => features, + None => { + let controller = self + .connect_provider_controller(provider_id) + .await + .map_err(ControllerError::into_api_error)?; + controller.close().await; + controller.features + } + }; + if feature.in_features(&features) { + Ok(()) + } else { + Err(AgentApiError::rejected(format!( + "provider {provider_id} does not offer {}", + feature.as_str() + ))) + } + } } #[cfg(test)] mod tests { use super::*; - use ::environments::EnvironmentConnectionSpec; - use environment_client::{EnvironmentClientResult, JsonRpcTransport}; - use environment_protocol::control::targets::ProviderBindingContext; - use serde_json::Value; - use std::sync::atomic::{AtomicBool, Ordering}; - - struct CloseTrackingTransport { - closed: Arc, + use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, + control::handshake::{ControllerCapabilities, ControllerRoutes}, + error::ProtocolError, + handshake::{ImplementationInfo, Plane}, + }; + + fn initialize(data: &str) -> ControllerInitializeResponse { + ControllerInitializeResponse { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + plane: Plane::Controller, + server_info: ImplementationInfo::new("controller").unwrap(), + capabilities: ControllerCapabilities { + adopt_target: true, + ingress: false, + }, + routes: ControllerRoutes { + data: DataRouteBase::new(data).unwrap(), + }, + } } - #[async_trait] - impl JsonRpcTransport for CloseTrackingTransport { - async fn send(&mut self, _message: Value) -> EnvironmentClientResult<()> { - Ok(()) + fn call(source: ClientError) -> ControllerError { + ControllerError::Call { + provider_id: "provider".into(), + source, } + } - async fn recv(&mut self) -> EnvironmentClientResult> { - Ok(None) + #[test] + fn connection_failures_and_contract_breaches_are_transient() { + for source in [ + ClientError::Closed { + code: None, + reason: None, + }, + ClientError::Transport("reset".into()), + ClientError::DeadlineExceeded { + elapsed_ms: 120_000, + }, + ClientError::Decode("missing field".into()), + ClientError::InvalidMessage("binary frame".into()), + ClientError::Protocol(ProtocolError::internal("bug")), + ClientError::Protocol(ProtocolError::unavailable("incus api down")), + ] { + assert_eq!( + call(source.clone()).disposition(), + ControllerDisposition::Transient, + "{source}" + ); } - - async fn close(&mut self) -> EnvironmentClientResult<()> { - self.closed.store(true, Ordering::SeqCst); - Ok(()) + // Every handshake failure is transient, a version mismatch included. + for source in [ + ClientError::VersionMismatch { + plane: Plane::Controller, + local: 3, + remote: 4, + }, + ClientError::UpgradeRejected { status: 404 }, + ClientError::ConnectTimeout { + endpoint: "ws://provider".into(), + }, + ] { + let error = ControllerError::Connect { + provider_id: "provider".into(), + source, + }; + assert_eq!(error.disposition(), ControllerDisposition::Transient); + assert!(error.is_connection_failure()); } } - fn connection() -> EnvironmentConnectionSpec { - EnvironmentConnectionSpec::new( - "in-process", - EnvironmentTransport::Provider { - provider_type: "fake".to_owned(), - }, - ) + #[test] + fn typed_refusals_and_missing_targets_are_final() { + let refused = call(ClientError::Protocol(ProtocolError::failed_precondition( + PreconditionReason::TemplateDeprecated, + "deprecated", + ))); + assert_eq!(refused.disposition(), ControllerDisposition::Refused); + assert_eq!( + refused.precondition(), + Some(PreconditionReason::TemplateDeprecated) + ); + assert!(!refused.is_connection_failure()); + let gone = call(ClientError::Protocol(ProtocolError::not_found("no target"))); + assert_eq!(gone.disposition(), ControllerDisposition::Gone); + let unadvertised = ControllerError::FeatureUnavailable { + provider_id: "provider".into(), + feature: ControllerFeature::AdoptTarget, + }; + assert_eq!(unadvertised.disposition(), ControllerDisposition::Refused); + // A deadline or a dropped socket skips the provider for the pass. + assert!(call(ClientError::DeadlineExceeded { elapsed_ms: 1 }).is_connection_failure()); + assert!(!call(ClientError::Decode("bad".into())).is_connection_failure()); } - fn create( - universe: &str, - binding: &str, - request: &str, - environment: &str, - ) -> CreateTargetParams { - CreateTargetParams { - request_id: request.to_owned(), - environment_id: environment.to_owned(), - incarnation_id: format!("incarnation-{environment}"), - binding: ProviderBindingContext { - universe_id: universe.to_owned(), - binding_id: binding.to_owned(), + #[test] + fn registration_refuses_only_definitive_handshake_failures() { + for refused in [ + ClientError::UpgradeRejected { status: 404 }, + ClientError::UpgradeRejected { status: 401 }, + ClientError::WrongPlane { + expected: Plane::Controller, + actual: Plane::Data, }, - template_id: "rust-v1".to_owned(), + ] { + assert!(refuses_registration(&refused), "{refused}"); + } + for accepted in [ + ClientError::UpgradeRejected { status: 503 }, + ClientError::UpgradeRejected { status: 429 }, + ClientError::Connect { + endpoint: "ws://provider".into(), + message: "refused".into(), + }, + ClientError::VersionMismatch { + plane: Plane::Controller, + local: 3, + remote: 4, + }, + ] { + assert!(!refuses_registration(&accepted), "{accepted}"); } } - #[tokio::test(flavor = "current_thread")] - async fn finishing_a_provider_operation_closes_its_transport() { - let closed = Arc::new(AtomicBool::new(false)); - let controller: Box = - Box::new(EnvironmentProviderClient::new(CloseTrackingTransport { - closed: closed.clone(), - })); - - let value = finish_provider_controller(controller, Ok::<_, AgentApiError>(42)) - .await - .expect("operation result"); - - assert_eq!(value, 42); - assert!(closed.load(Ordering::SeqCst)); + /// A connector whose every dial fails, counting the attempts. + #[derive(Default)] + struct UnreachableConnector { + attempts: std::sync::atomic::AtomicUsize, } - #[tokio::test(flavor = "current_thread")] - async fn fake_provider_adopts_retries_and_isolates_bindings() { - let connector = WebSocketProviderControllerConnector::default(); - let mut first = connector.connect(&connection()).await.expect("connect"); - let created = first - .create_target(&create( - "universe-a", - "binding-a", - "request-1", - "environment-1", - )) - .await - .expect("create"); - drop(first); - - let mut after_restart = connector.connect(&connection()).await.expect("reconnect"); - let adopted = after_restart - .create_target(&create( - "universe-a", - "binding-a", - "request-1", - "different-environment", - )) - .await - .expect("adopt"); - assert_eq!(adopted.target.target_id, created.target.target_id); - - let isolated = after_restart - .create_target(&create( - "universe-b", - "binding-b", - "request-1", - "environment-2", - )) - .await - .expect("isolated create"); - assert_ne!(isolated.target.target_id, created.target.target_id); - - let imported = after_restart - .adopt_target(&AdoptTargetParams { - request_id: "adopt-request-1".to_owned(), - environment_id: "environment-imported".to_owned(), - incarnation_id: "incarnation-imported".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-c".to_owned(), - binding_id: "binding-c".to_owned(), - }, - source_target: "legacy/hand-built-vm".to_owned(), + #[async_trait] + impl ProviderControllerConnector for UnreachableConnector { + async fn connect( + &self, + endpoint: &EnvironmentEndpoint, + _options: ConnectOptions, + ) -> Result { + self.attempts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Err(ClientError::Connect { + endpoint: endpoint.to_string(), + message: "refused".into(), }) - .await - .expect("explicit adoption"); - assert_eq!( - imported.target.metadata.get("imageFingerprint"), - Some(&"fake:adopted:legacy/hand-built-vm".to_owned()) - ); + } + } - let wrong_owner = after_restart - .close_target(&CloseTargetParams { - request_id: "close-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-environment-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-b".to_owned(), - binding_id: "binding-b".to_owned(), - }, - target_id: created.target.target_id.clone(), - force: false, + async fn provider_store( + endpoint: &str, + credential: &str, + ) -> ::environments::InMemoryEnvironmentRegistryStore { + let store = ::environments::InMemoryEnvironmentRegistryStore::new(); + store + .put_provider(::environments::PutEnvironmentProvider { + provider_id: EnvironmentProviderId::new("provider"), + display_name: None, + controller_endpoint: EnvironmentEndpoint::parse(endpoint).unwrap(), + metadata: Default::default(), + credential: Some(SecretValue::new(credential)), + updated_at_ms: 1, }) .await - .expect_err("ownership mismatch"); - assert_eq!(wrong_owner.kind, AgentApiErrorKind::Rejected); - - let power = |power: PowerState| SetTargetPowerParams { - request_id: format!("power-{power}"), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-environment-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-a".to_owned(), - binding_id: "binding-a".to_owned(), - }, - target_id: created.target.target_id.clone(), - power, - }; - assert_eq!(created.target.power_states, FAKE_POWER_STATES.to_vec()); - let paused = after_restart - .set_target_power(&power(PowerState::Paused)) - .await - .expect("pause"); - assert_eq!(paused.target.status, ProviderTargetStatus::Paused); - let resumed = after_restart - .set_target_power(&power(PowerState::Running)) - .await - .expect("resume"); - assert_eq!(resumed.target.status, ProviderTargetStatus::Ready); - assert_eq!(resumed.target.target_id, created.target.target_id); - - let closed = after_restart - .close_target(&CloseTargetParams { - request_id: "close-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-environment-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-a".to_owned(), - binding_id: "binding-a".to_owned(), - }, - target_id: created.target.target_id.clone(), - force: false, - }) + .unwrap(); + store + } + + #[tokio::test(flavor = "current_thread")] + async fn the_refresh_records_facts_of_a_provider_never_reached() { + let controller = crate::environments::fake_controller::FakeController::start() .await - .expect("close"); - assert_eq!(closed.status, ProviderTargetStatus::Closed); + .unwrap(); + let store = provider_store(controller.controller_url(), controller.credential()).await; + let provider_id = EnvironmentProviderId::new("provider"); assert!( - after_restart - .set_target_power(&power(PowerState::Running)) + store + .read_provider(&provider_id) .await - .is_err() + .unwrap() + .data_route_base + .is_none() + ); + let mut pass = ProviderPass::default(); + ProviderFactsRefresh::default() + .run(&mut pass, &store, &WebSocketProviderControllerConnector) + .await; + pass.close().await; + let provider = store.read_provider(&provider_id).await.unwrap(); + assert!(provider.data_route_base.is_some()); + assert!(provider.controller_features.is_some()); + } + + #[tokio::test(flavor = "current_thread")] + async fn an_unreachable_provider_is_retried_with_backoff() { + let store = provider_store("ws://127.0.0.1:9/control", "credential").await; + let connector = UnreachableConnector::default(); + let mut refresh = ProviderFactsRefresh::default(); + let mut pass = ProviderPass::default(); + refresh.run(&mut pass, &store, &connector).await; + assert_eq!( + connector.attempts.load(std::sync::atomic::Ordering::SeqCst), + 1 ); + // The next listing still finds it unrecorded, but its retry is not + // due yet. + refresh.listed_at = None; + let mut pass = ProviderPass::default(); + refresh.run(&mut pass, &store, &connector).await; + assert_eq!( + connector.attempts.load(std::sync::atomic::Ordering::SeqCst), + 1 + ); + let (_, backoff) = refresh.retry[&EnvironmentProviderId::new("provider")]; + assert_eq!(backoff, ProviderFactsRefresh::FIRST_RETRY); + } + + #[test] + fn controller_facts_enforce_the_downgrade_rule() { + let secure = EnvironmentEndpoint::parse("wss://provider.test/control").unwrap(); + assert!(matches!( + controller_facts(&secure, &initialize("ws://provider.test/routes")), + Err(UrlError::Downgrade(_)) + )); + let (features, base) = + controller_facts(&secure, &initialize("wss://provider.test/routes")).unwrap(); + assert!(features.adopt_target && !features.ingress); + assert_eq!(base.as_str(), "wss://provider.test/routes"); + let plain = EnvironmentEndpoint::parse("ws://127.0.0.1:9/control").unwrap(); + assert!(controller_facts(&plain, &initialize("ws://127.0.0.1:9/routes")).is_ok()); + let local = EnvironmentEndpoint::parse("unix:///run/provider.sock").unwrap(); + assert!(controller_facts(&local, &initialize("ws://localhost/routes")).is_ok()); } } diff --git a/crates/temporal-runtime/src/environments/providers.rs b/crates/temporal-runtime/src/environments/providers.rs index 21204d8c7..0e891459e 100644 --- a/crates/temporal-runtime/src/environments/providers.rs +++ b/crates/temporal-runtime/src/environments/providers.rs @@ -1,3 +1,4 @@ +use super::provider_controllers::ControllerError; use super::*; use ::environments::{ @@ -5,8 +6,12 @@ use ::environments::{ EnvironmentProviderBindingStatus, EnvironmentProviderId, EnvironmentProviderRecord, EnvironmentRecord, EnvironmentRegistryError, EnvironmentSource, EnvironmentStatus, }; -use environment_protocol::control::targets::{ - EnvironmentTemplate, ListTemplatesParams, ProviderBindingContext, +use environment_protocol::{ + control::{ + target::BindingRef, + template::{EnvironmentTemplate, ListTemplatesParams}, + }, + ids::{BindingId, UniverseId}, }; impl EnvironmentService { @@ -76,17 +81,20 @@ impl EnvironmentService { .into_iter() .filter(|binding| binding.status == EnvironmentProviderBindingStatus::Enabled) { - let provider = self.read_environment_provider(&binding.provider_id).await?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) - .await?; - let response = controller - .list_templates(&ListTemplatesParams { - binding: binding_context(&binding), - }) - .await; - let response = finish_provider_controller(controller, response).await?; + let controller = self + .connect_provider_controller(&binding.provider_id) + .await + .map_err(ControllerError::into_api_error)?; + let response = match binding_ref(&binding) { + Ok(binding) => controller + .client + .list_templates(&ListTemplatesParams { binding }) + .await + .map_err(|error| controller.call_error(error).into_api_error()), + Err(error) => Err(error), + }; + controller.close().await; + let response = response?; templates.extend( response .templates @@ -130,11 +138,19 @@ impl EnvironmentService { } } -pub(crate) fn binding_context(record: &EnvironmentProviderBindingRecord) -> ProviderBindingContext { - ProviderBindingContext { - universe_id: record.universe_id.to_string(), - binding_id: record.binding_id.to_string(), - } +/// The controller-plane reference to a binding. +pub(crate) fn binding_ref( + record: &EnvironmentProviderBindingRecord, +) -> Result { + let invalid = |error: environment_protocol::ids::IdError| { + AgentApiError::internal(format!( + "binding ids do not fit the controller protocol: {error}" + )) + }; + Ok(BindingRef { + universe_id: UniverseId::new(record.universe_id.to_string()).map_err(invalid)?, + binding_id: BindingId::new(record.binding_id.as_str()).map_err(invalid)?, + }) } pub(crate) fn parse_environment_provider_id( @@ -178,14 +194,25 @@ pub(crate) fn environment_template_view( template: &EnvironmentTemplate, ) -> EnvironmentTemplateView { EnvironmentTemplateView { - template_id: template.template_id.clone(), + template_id: template.template_id.to_string(), provider_id: binding.provider_id.to_string(), binding_id: binding.binding_id.to_string(), - display_name: template.display_name.clone(), - description: template.description.clone(), + display_name: template.display_name.to_string(), + description: template + .description + .as_ref() + .map(|description| description.to_string()), public_ingress: template.public_ingress, deprecated: template.deprecated, - metadata: template.metadata.clone(), + resources: template + .resources + .as_ref() + .map(|resources| EnvironmentTemplateResourcesView { + vcpus: resources.vcpus, + memory_bytes: resources.memory_bytes, + disk_bytes: resources.disk_bytes, + }), + metadata: template.metadata.as_map().clone(), } } @@ -201,29 +228,8 @@ pub(crate) fn environment_view(record: &EnvironmentRecord) -> EnvironmentView { provider_id: provider_id.to_string(), binding_id: binding_id.to_string(), }, - EnvironmentSource::External { connection } => EnvironmentSourceView::External { - connection: EnvironmentConnectionView { - endpoint: connection.endpoint.clone(), - transport: match &connection.transport { - environment_protocol::shared::EnvironmentTransport::WebSocket => { - EnvironmentConnectionTransportView::WebSocket - } - environment_protocol::shared::EnvironmentTransport::Http => { - EnvironmentConnectionTransportView::Http - } - environment_protocol::shared::EnvironmentTransport::Stdio => { - EnvironmentConnectionTransportView::Stdio - } - environment_protocol::shared::EnvironmentTransport::Ssh => { - EnvironmentConnectionTransportView::Ssh - } - environment_protocol::shared::EnvironmentTransport::Provider { - provider_type, - } => EnvironmentConnectionTransportView::Provider { - provider_type: provider_type.clone(), - }, - }, - }, + EnvironmentSource::External { endpoint } => EnvironmentSourceView::External { + endpoint: endpoint.as_str().to_owned(), }, EnvironmentSource::Registered { registration_key_id, @@ -271,6 +277,13 @@ pub(crate) fn environment_view(record: &EnvironmentRecord) -> EnvironmentView { public_endpoint: record.public_endpoint.clone(), metadata: record.metadata.clone(), last_seen_at_ms: record.last_seen_at_ms, + lifecycle_error: record.lifecycle_error.as_ref().map(|error| { + EnvironmentLifecycleErrorView { + message: error.message.clone(), + first_at_ms: error.first_at_ms, + last_at_ms: error.last_at_ms, + } + }), created_at_ms: record.created_at_ms, updated_at_ms: record.updated_at_ms, } diff --git a/crates/temporal-runtime/src/environments/registration.rs b/crates/temporal-runtime/src/environments/registration.rs index 9317b8276..82a12b9ee 100644 --- a/crates/temporal-runtime/src/environments/registration.rs +++ b/crates/temporal-runtime/src/environments/registration.rs @@ -7,17 +7,21 @@ use std::collections::BTreeMap; use ::environments::{ BeginCloseEnvironment, CreateEnvironmentRegistrationKey, EnvironmentRecord, EnvironmentRegistrationKeyRecord, EnvironmentRegistrationKeyStatus, - EnvironmentRegistrationKeyStore, EnvironmentStatus, EnvironmentStore, ListEnvironments, - ObserveRegisteredEnvironment, RegisteredConnectionObservation, RegisteredIdentityMode, - RegistrationKeyPolicy, RevokeEnvironmentRegistrationKey, mint_registration_key, + EnvironmentRegistrationKeyStore, EnvironmentSource, EnvironmentStatus, EnvironmentStore, + ListEnvironments, ObserveRegisteredEnvironment, RegisteredConnectionObservation, + RegisteredIdentityMode, RegistrationKeyPolicy, RevokeEnvironmentRegistrationKey, + mint_registration_key, }; /// A registered environment whose heartbeat stamp is older than this is /// treated as disconnected even if its row still says `Ready`: the gateway /// that held its control connection stopped without recording the -/// disconnect. Three missed heartbeats leaves room for one slow pong. +/// disconnect. The registration channel's liveness rule: the same number +/// of missed heartbeats after which either peer declares the connection +/// dead. pub(crate) const REGISTERED_STALE_AFTER_MS: i64 = - 3 * crate::gateway::registration::HEARTBEAT_INTERVAL.as_millis() as i64; + environment_protocol::limits::HEARTBEAT_MISSED_INTERVALS as i64 + * crate::gateway::registration::HEARTBEAT_INTERVAL.as_millis() as i64; /// What the reconciler does with one open registered environment. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -53,6 +57,21 @@ pub(crate) fn registered_sweep_action( None } +/// Whether a daemon's `registration/goodbye` ends its environment. An +/// ephemeral environment closes at once: its daemon stopped on purpose, +/// so waiting out the disconnect grace would only keep a finished sandbox +/// `Offline` and counted against its key's limit. A persistent one stays +/// `Offline` until its daemon returns or someone closes it. +pub(crate) fn departure_closes(environment: &EnvironmentRecord) -> bool { + matches!( + environment.source, + EnvironmentSource::Registered { + identity_mode: RegisteredIdentityMode::Ephemeral, + .. + } + ) +} + use super::*; use super::{ lifecycle::parse_registration_key_id, @@ -190,6 +209,7 @@ impl EnvironmentService { /// One pass over open registered environments: repair stale `Ready` /// rows and close ephemeral environments past their disconnect grace. + /// One environment's failure is logged and never stops the pass. /// Returns how many rows changed. pub(crate) async fn reconcile_registered_once(&self) -> Result { let environments = EnvironmentStore::list_open_registered_environments(self.store.as_ref()) @@ -202,66 +222,89 @@ impl EnvironmentService { let mut keys: BTreeMap = BTreeMap::new(); let mut changed = 0; for environment in environments { - let Some(key_id) = environment.registration_key_id() else { - continue; - }; - let key = match keys.get(key_id.as_str()) { - Some(key) => key.clone(), - None => { - let key = EnvironmentRegistrationKeyStore::read_registration_key( - self.store.as_ref(), - key_id, - ) - .await - .map_err(map_environments_error)?; - keys.insert(key_id.to_string(), key.clone()); - key - } - }; - match registered_sweep_action(&environment, &key, now) { - None => {} - Some(RegisteredSweepAction::MarkOffline) => { - tracing::info!( - target: "temporal_runtime", - environment = %environment.environment_id, - "registered environment heartbeat is stale; marking offline" - ); - EnvironmentStore::observe_registered_environment( - self.store.as_ref(), - ObserveRegisteredEnvironment { - environment_id: environment.environment_id.clone(), - observation: RegisteredConnectionObservation::Disconnected, - observed_at_ms: now, - metadata: BTreeMap::new(), - }, - ) - .await - .map_err(map_environments_error)?; - changed += 1; - } - Some(RegisteredSweepAction::Close) => { - tracing::info!( - target: "temporal_runtime", - environment = %environment.environment_id, - registration_key_id = %key.registration_key_id, - "ephemeral registered environment exceeded its disconnect grace; closing" - ); - EnvironmentStore::begin_close_environment( - self.store.as_ref(), - BeginCloseEnvironment { - environment_id: environment.environment_id.clone(), - updated_at_ms: now, - }, - ) - .await - .map_err(map_environments_error)?; - changed += 1; - } + match self + .reconcile_registered_environment(&environment, &mut keys, now) + .await + { + Ok(true) => changed += 1, + Ok(false) => {} + Err(error) => tracing::warn!( + target: "temporal_runtime", + environment = %environment.environment_id, + error = %error.message, + "registered environment sweep failed; continuing with the next" + ), } } Ok(changed) } + /// Apply the sweep decision to one registered environment; true when + /// its row changed. + async fn reconcile_registered_environment( + &self, + environment: &EnvironmentRecord, + keys: &mut BTreeMap, + now: i64, + ) -> Result { + let Some(key_id) = environment.registration_key_id() else { + return Ok(false); + }; + let key = match keys.get(key_id.as_str()) { + Some(key) => key.clone(), + None => { + let key = EnvironmentRegistrationKeyStore::read_registration_key( + self.store.as_ref(), + key_id, + ) + .await + .map_err(map_environments_error)?; + keys.insert(key_id.to_string(), key.clone()); + key + } + }; + match registered_sweep_action(environment, &key, now) { + None => Ok(false), + Some(RegisteredSweepAction::MarkOffline) => { + tracing::info!( + target: "temporal_runtime", + environment = %environment.environment_id, + "registered environment heartbeat is stale; marking offline" + ); + EnvironmentStore::observe_registered_environment( + self.store.as_ref(), + ObserveRegisteredEnvironment { + environment_id: environment.environment_id.clone(), + observation: RegisteredConnectionObservation::Disconnected, + observed_at_ms: now, + metadata: BTreeMap::new(), + }, + ) + .await + .map_err(map_environments_error)?; + Ok(true) + } + Some(RegisteredSweepAction::Close) => { + tracing::info!( + target: "temporal_runtime", + environment = %environment.environment_id, + registration_key_id = %key.registration_key_id, + "ephemeral registered environment exceeded its disconnect grace; closing" + ); + EnvironmentStore::begin_close_environment( + self.store.as_ref(), + BeginCloseEnvironment { + environment_id: environment.environment_id.clone(), + updated_at_ms: now, + }, + ) + .await + .map_err(map_environments_error)?; + Ok(true) + } + } + } + async fn registration_key_view( &self, record: &EnvironmentRegistrationKeyRecord, @@ -314,7 +357,7 @@ mod tests { use ::environments::{ EnvironmentDaemonId, EnvironmentId, EnvironmentIncarnationId, EnvironmentIncarnationRecord, - EnvironmentProvisionRequestId, EnvironmentRegistrationKeyId, EnvironmentSource, PowerState, + EnvironmentProvisionRequestId, EnvironmentRegistrationKeyId, PowerState, }; use super::*; @@ -337,6 +380,14 @@ mod tests { } fn environment(status: EnvironmentStatus, last_seen_at_ms: Option) -> EnvironmentRecord { + environment_in_mode(status, last_seen_at_ms, RegisteredIdentityMode::Ephemeral) + } + + fn environment_in_mode( + status: EnvironmentStatus, + last_seen_at_ms: Option, + identity_mode: RegisteredIdentityMode, + ) -> EnvironmentRecord { let public_key = "ab".repeat(32); let daemon_id = EnvironmentDaemonId::from_public_key(&[0xab; 32]); EnvironmentRecord { @@ -346,7 +397,7 @@ mod tests { registration_key_id: EnvironmentRegistrationKeyId::new("rk"), daemon_id, daemon_public_key: public_key, - identity_mode: RegisteredIdentityMode::Ephemeral, + identity_mode, }, display_name: None, status, @@ -366,6 +417,7 @@ mod tests { public_endpoint: None, metadata: BTreeMap::new(), last_seen_at_ms, + lifecycle_error: None, created_at_ms: 0, updated_at_ms: 0, } @@ -438,4 +490,16 @@ mod tests { Some(RegisteredSweepAction::Close) ); } + + #[test] + fn only_ephemeral_environments_close_on_departure() { + let ephemeral = environment(EnvironmentStatus::Ready, Some(1_000)); + assert!(departure_closes(&ephemeral)); + let persistent = environment_in_mode( + EnvironmentStatus::Ready, + Some(1_000), + RegisteredIdentityMode::Persistent, + ); + assert!(!departure_closes(&persistent)); + } } diff --git a/crates/temporal-runtime/src/environments/resolver.rs b/crates/temporal-runtime/src/environments/resolver.rs index 9b9f24a18..85d3b6a99 100644 --- a/crates/temporal-runtime/src/environments/resolver.rs +++ b/crates/temporal-runtime/src/environments/resolver.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use environment_client::{ClientError, DataClient}; use environments::{ EnvironmentId, EnvironmentProviderStore, EnvironmentRecord, EnvironmentRegistryError, EnvironmentSource, EnvironmentStatus, EnvironmentStore, PowerState, SetEnvironmentPower, @@ -70,9 +71,9 @@ impl EnvironmentResolver { EnvironmentStatus::Failed => Err(EnvironmentResolveError::Failed { environment_id: environment.environment_id.to_string(), message: environment - .metadata - .get(LIFECYCLE_ERROR_METADATA_KEY) - .cloned() + .lifecycle_error + .as_ref() + .map(|error| error.message.clone()) .unwrap_or_else(|| "environment provisioning failed".to_owned()), }), EnvironmentStatus::Closing | EnvironmentStatus::Closed => { @@ -85,29 +86,43 @@ impl EnvironmentResolver { } /// Check readiness for actual use, requesting wake-up where supported and - /// probing the data route. Selection itself uses only registry validation. + /// running the data-plane handshake over the route. A daemon of another + /// protocol version is a typed blocker, never a Ready environment whose + /// calls all fail. Selection itself uses only registry validation. pub(crate) async fn ready_for_use( &self, environment_id: &EnvironmentId, now_ms: i64, ) -> Result { let environment = self.resolve_for_connection(environment_id, now_ms).await?; - if let Some(gateway) = &self.gateway { - let connection = gateway.connection_for(self.universe_id, &environment); - if let Ok(mut client) = environment_client::EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("lightspeed-environment-readiness"), - ) + let unavailable = |status: String| EnvironmentResolveError::EnvironmentUnavailable { + environment_id: environment.environment_id.as_str().to_owned(), + status, + }; + let Some(gateway) = &self.gateway else { + return Err(unavailable( + "environment endpoint is not reachable".to_owned(), + )); + }; + let endpoint = gateway.endpoint_for(self.universe_id, &environment); + match DataClient::connect_initialized(&endpoint, gateway.connect_options(READINESS_CLIENT)) .await - { + { + Ok((client, _)) => { let _ = client.close().await; - return Ok(environment); + Ok(environment) } + Err(ClientError::VersionMismatch { local, remote, .. }) => { + Err(EnvironmentResolveError::VersionMismatch { + environment_id: environment.environment_id.as_str().to_owned(), + local, + remote, + }) + } + Err(error) => Err(unavailable(format!( + "environment endpoint is not reachable: {error}" + ))), } - Err(EnvironmentResolveError::EnvironmentUnavailable { - environment_id: environment.environment_id.as_str().to_owned(), - status: "environment endpoint is not reachable".to_owned(), - }) } /// Validate lifecycle and policy immediately before opening a real @@ -198,11 +213,21 @@ pub(crate) enum EnvironmentResolveError { #[error("environment is closed: {environment_id}")] Closed { environment_id: String }, + + /// The environment's envd speaks another data-plane protocol version; + /// it cannot serve this runtime until it is replaced or upgraded. + #[error( + "environment {environment_id} runs envd protocol version {remote}; this runtime requires {local}" + )] + VersionMismatch { + environment_id: String, + local: u32, + remote: u32, + }, } -/// Metadata key under which the lifecycle reconciler records the provider's -/// failure message on a `failed` environment. -pub(crate) const LIFECYCLE_ERROR_METADATA_KEY: &str = "lifecycleError"; +/// `clientInfo.name` of readiness handshakes. +const READINESS_CLIENT: &str = "lightspeed-environment-readiness"; /// A powered-down provisioned environment wakes on use when its provider /// reported that it can be moved back to `running`. External environments and @@ -220,9 +245,10 @@ pub(crate) fn wake_on_use_applies(environment: &EnvironmentRecord) -> bool { mod tests { use std::collections::BTreeMap; - use environment_protocol::shared::{EnvironmentTransport, ProviderTargetId}; + use auth::SecretValue; + use environment_protocol::ids::ProviderTargetId; use environments::{ - CreateEnvironment, EnvironmentConnectionSpec, EnvironmentIncarnationId, + CreateEnvironment, EnvironmentEndpoint, EnvironmentIncarnationId, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProvisionRequestId, EnvironmentStatus, EnvironmentTemplateId, ObserveProvisionedEnvironment, @@ -231,6 +257,26 @@ mod tests { use super::*; + fn target_id() -> Option { + Some(ProviderTargetId::new("target-1").unwrap()) + } + + fn observation( + environment_id: &EnvironmentId, + status: EnvironmentStatus, + power_states: Vec, + observed_at_ms: i64, + ) -> ObserveProvisionedEnvironment { + ObserveProvisionedEnvironment { + environment_id: environment_id.clone(), + provider_target_id: target_id(), + status, + power_states, + daemon_build: None, + observed_at_ms, + } + } + async fn resolver() -> (EnvironmentResolver, EnvironmentId) { let store = Arc::new(environments::InMemoryEnvironmentRegistryStore::new()); let provider_id = EnvironmentProviderId::new("allowed"); @@ -238,11 +284,10 @@ mod tests { .put_provider(PutEnvironmentProvider { provider_id: provider_id.clone(), display_name: None, - controller_connection: EnvironmentConnectionSpec::new( - "http://controller.test", - EnvironmentTransport::Http, - ), + controller_endpoint: EnvironmentEndpoint::parse("ws://controller.test/control") + .unwrap(), metadata: BTreeMap::new(), + credential: Some(SecretValue::new("provider-credential")), updated_at_ms: 10, }) .await @@ -260,7 +305,6 @@ mod tests { .await .expect("binding"); let environment_id = EnvironmentId::new("environment-1"); - let target_id = ProviderTargetId::new("target-1"); store .create_environment(CreateEnvironment { request_id: EnvironmentProvisionRequestId::new("request-1"), @@ -270,20 +314,18 @@ mod tests { template_id: EnvironmentTemplateId::new("test-template"), display_name: None, metadata: BTreeMap::new(), - idle_policy: None, created_at_ms: 10, }) .await .expect("create environment"); store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: target_id.clone(), - status: EnvironmentStatus::Offline, - power_states: Vec::new(), - observed_at_ms: 10, - }) + .observe_provisioned_environment(observation( + &environment_id, + EnvironmentStatus::Offline, + Vec::new(), + 10, + )) .await .expect("environment"); ( @@ -292,146 +334,161 @@ mod tests { ) } + /// Counters and switches of the fake gateway route below. + #[derive(Clone, Default)] + struct Route { + connections: Arc, + initializations: Arc, + scans: Arc, + unchanged_roots: Arc, + stall_skills: Arc, + stall: Arc, + unsupported: Arc, + } + + impl Route { + fn counts(&self) -> (usize, usize, usize) { + use std::sync::atomic::Ordering::SeqCst; + ( + self.connections.load(SeqCst), + self.initializations.load(SeqCst), + self.scans.load(SeqCst), + ) + } + } + + /// Serve a gateway data route that answers with a real envd's handlers + /// over `root`, counting connections, handshakes and scans. + async fn serve_route( + root: std::path::PathBuf, + state_dir: std::path::PathBuf, + route: Route, + ) -> (String, tokio::task::JoinHandle<()>) { + use environment_protocol::{data::scan::ScanResponse, jsonrpc, paths::EnvironmentPath}; + use futures_util::{SinkExt, StreamExt}; + use std::sync::atomic::Ordering::SeqCst; + let runtime = + environment_daemon::DaemonRuntime::new(environment_daemon::config::DaemonConfig { + listen: None, + cwd: root.clone(), + fs_root: root.clone(), + state_dir, + read_only_fs: true, + registration: None, + scrubbed_env: Vec::new(), + }) + .unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let mut clients = tokio::task::JoinSet::new(); + loop { + let (socket, _) = listener.accept().await.unwrap(); + route.connections.fetch_add(1, SeqCst); + while let Some(result) = clients.try_join_next() { + result.unwrap(); + } + let route = route.clone(); + let runtime = runtime.clone(); + let root = root.clone(); + clients.spawn(async move { + let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap(); + let context = environment_daemon::dispatch::RequestContext::detached(); + while let Some(Ok(message)) = socket.next().await { + let Ok(text) = message.to_text() else { + continue; + }; + let Ok(jsonrpc::Message::Request(request)) = jsonrpc::parse(text) else { + continue; + }; + if route.stall.load(SeqCst) { + std::future::pending::<()>().await; + } + let response = if request.method == "initialize" { + route.initializations.fetch_add(1, SeqCst); + let mut initialized = runtime.initialize_response(); + initialized.home_directory = + Some(EnvironmentPath::new(root.to_str().unwrap()).unwrap()); + if route.unsupported.load(SeqCst) { + initialized.capabilities.filesystem = None; + } + jsonrpc::Response::success(request.id, &initialized) + } else { + if request.method == "fs/scan" { + route.scans.fetch_add(1, SeqCst); + let skills = request.params["filter"]["include"] + .as_array() + .is_some_and(|include| include.iter().any(|p| p == "SKILL.md")); + if skills && route.stall_skills.load(SeqCst) { + std::future::pending::<()>().await; + } + } + let result = environment_daemon::dispatch::handle_request( + &runtime, + &context, + &request.method, + request.params.clone(), + ) + .await + .map(|reply| reply.value); + if let Ok(value) = &result + && let Ok(scan) = serde_json::from_value::(value.clone()) + { + let unchanged = scan + .roots + .iter() + .filter(|root| { + matches!( + root, + environment_protocol::data::scan::ScanRootResult::Unchanged { .. } + ) + }) + .count(); + route.unchanged_roots.fetch_add(unchanged, SeqCst); + } + jsonrpc::Response::from_result(request.id, result) + }; + if socket + .send(tokio_tungstenite::tungstenite::Message::Text( + response.to_text().into(), + )) + .await + .is_err() + { + break; + } + } + }); + } + }); + (base_url, task) + } + #[tokio::test(flavor = "current_thread")] async fn environment_discovery_shares_connection_preserves_freshness_and_never_wakes() { - use futures_util::{SinkExt, StreamExt}; use harness::{ CoreAgentCommand, storage::{BlobStore, InMemoryBlobStore}, }; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::atomic::Ordering::SeqCst; use tools::skills::environment::{EnvironmentSkillAvailability, EnvironmentSkillCatalog}; let (resolver, environment_id) = resolver().await; let store = resolver.environments.clone(); let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); + let root = temp.path().canonicalize().unwrap().join("root"); std::fs::create_dir_all(root.join(".agents/skills/review")).unwrap(); let doc = "---\nname: review\ndescription: Review code.\n---\nbody"; let skill_path = root.join(".agents/skills/review/SKILL.md"); std::fs::write(&skill_path, doc).unwrap(); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let gateway = crate::environments::gateway::EnvironmentGatewayClientConfig::new( - format!("http://{}", listener.local_addr().unwrap()), - "test", - ); - let connections = Arc::new(AtomicUsize::new(0)); - let initializations = Arc::new(AtomicUsize::new(0)); - let metadata_reads = Arc::new(AtomicUsize::new(0)); - let stall_skills = Arc::new(AtomicBool::new(false)); - let scans = Arc::new(AtomicUsize::new(0)); - let unchanged = Arc::new(AtomicUsize::new(0)); - let supported = Arc::new(AtomicBool::new(true)); - let stall = Arc::new(AtomicBool::new(false)); - let task = { - let connections = connections.clone(); - let initializations = initializations.clone(); - let metadata_reads = metadata_reads.clone(); - let stall_skills = stall_skills.clone(); - let scans = scans.clone(); - let unchanged = unchanged.clone(); - let supported = supported.clone(); - let stall = stall.clone(); - let root = root.clone(); - tokio::spawn(async move { - let mut clients = tokio::task::JoinSet::new(); - loop { - let (socket, _) = listener.accept().await.unwrap(); - connections.fetch_add(1, Ordering::SeqCst); - while let Some(result) = clients.try_join_next() { - result.unwrap(); - } - let initializations = initializations.clone(); - let metadata_reads = metadata_reads.clone(); - let stall_skills = stall_skills.clone(); - let scans = scans.clone(); - let unchanged = unchanged.clone(); - let supported = supported.clone(); - let stall = stall.clone(); - let root = root.clone(); - clients.spawn(async move { - let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap(); - while let Some(Ok(message)) = socket.next().await { - let Ok(text) = message.to_text() else { - continue; - }; - let Ok(request) = serde_json::from_str::(text) else { - continue; - }; - let Some(id) = request.get("id") else { - continue; - }; - if stall.load(Ordering::SeqCst) { - std::future::pending::<()>().await; - } - let result = match request["method"].as_str().unwrap() { - "initialize" => { - initializations.fetch_add(1, Ordering::SeqCst); - serde_json::json!({ "protocolVersion": environment_protocol::shared::CURRENT_PROTOCOL_VERSION, "connectionId": "test", "capabilities": {"filesystemRead": true, "filesystemScan": supported.load(Ordering::SeqCst)}, "defaultCwd": root, "homeDirectory": root, "implementation": {"name": "test", "version": "1"} }) - } - "fs/getMetadata" => { - metadata_reads.fetch_add(1, Ordering::SeqCst); - let fs = environment_daemon::filesystem::LocalFileSystem::new( - root.clone(), - root.clone(), - false, - ); - serde_json::to_value( - fs.get_metadata( - serde_json::from_value(request["params"].clone()).unwrap(), - ) - .await - .unwrap(), - ) - .unwrap() - } - "fs/scan" => { - scans.fetch_add(1, Ordering::SeqCst); - if stall_skills.load(Ordering::SeqCst) - && request["params"]["includePatterns"].as_array().unwrap().iter().any(|p| p == "SKILL.md") { - std::future::pending::<()>().await; - } - let fs = environment_daemon::filesystem::LocalFileSystem::new( - root.clone(), - root.clone(), - false, - ); - let result = fs - .scan( - serde_json::from_value(request["params"].clone()).unwrap(), - ) - .await - .unwrap(); - if result.unchanged { - unchanged.fetch_add(1, Ordering::SeqCst); - } - serde_json::to_value(result).unwrap() - } - other => panic!("unexpected discovery RPC: {other}"), - }; - if socket - .send(tokio_tungstenite::tungstenite::Message::Text( - serde_json::json!({"jsonrpc":"2.0", "id":id, "result":result}) - .to_string() - .into(), - )) - .await - .is_err() - { - break; - } - } - }); - } - }) - }; - let counts = || { - ( - connections.load(Ordering::SeqCst), - initializations.load(Ordering::SeqCst), - metadata_reads.load(Ordering::SeqCst), - scans.load(Ordering::SeqCst), - ) - }; + let route = Route::default(); + let (base_url, task) = serve_route( + root.clone(), + temp.path().canonicalize().unwrap().join("state"), + route.clone(), + ) + .await; + let gateway = + crate::environments::gateway::EnvironmentGatewayClientConfig::new(base_url, "test"); let blobs = InMemoryBlobStore::new(); let session_id = harness::SessionId::new(uuid::Uuid::new_v4().to_string()); let feature = harness::EnvironmentsFeature { @@ -461,40 +518,46 @@ mod tests { Some(CoreAgentCommand::UpsertContext { entry, .. }) => entry, other => panic!("expected publication: {other:?}"), }; + let catalog_of = async |entry: &harness::ContextEntryInput| -> EnvironmentSkillCatalog { + serde_json::from_slice( + &blobs + .read_bytes(entry.provenance_ref.as_ref().unwrap()) + .await + .unwrap(), + ) + .unwrap() + }; // Offline inspection never changes desired power and never opens a data route. let before = store.read_environment(&environment_id).await.unwrap(); let unavailable = entry(refresh(None).await.unwrap()); - assert_eq!(scans.load(Ordering::SeqCst), 0); + assert_eq!(route.counts(), (0, 0, 0)); assert_eq!( store.read_environment(&environment_id).await.unwrap(), before ); assert!(refresh(Some(&unavailable)).await.unwrap().is_none()); store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 20, - }) + .observe_provisioned_environment(observation( + &environment_id, + EnvironmentStatus::Ready, + vec![PowerState::Running, PowerState::Paused], + 20, + )) .await .unwrap(); let available = entry(refresh(Some(&unavailable)).await.unwrap()); - let catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(available.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + let catalog = catalog_of(&available).await; assert_eq!( catalog.availability, EnvironmentSkillAvailability::Available ); assert_eq!(catalog.skills.len(), 1); + // The working directory is proven by the scan's probe root, not by + // a separate metadata call: one connection, one handshake, one scan. + assert_eq!(route.counts(), (1, 1, 1)); + // An unchanged root answers by fingerprint and reuses the cached skills. assert!(refresh(Some(&available)).await.unwrap().is_none()); - assert_eq!(unchanged.load(Ordering::SeqCst), 1); + assert_eq!(route.unchanged_roots.load(SeqCst), 1); std::fs::write(&skill_path, doc.replace("body", "edit")).unwrap(); assert!( refresh(Some(&available)).await.unwrap().is_none(), @@ -526,13 +589,10 @@ mod tests { }; for text in ["First instructions", "Updated instructions"] { std::fs::write(&prompt_path, text).unwrap(); - let before = counts(); + let before = route.counts(); let result = refresh_sources(&both, Some(&edited)).await; assert!(result.skill_command.is_none()); - assert_eq!( - counts(), - (before.0 + 1, before.1 + 1, before.2 + 1, before.3 + 2) - ); + assert_eq!(route.counts(), (before.0 + 1, before.1 + 1, before.2 + 2)); assert_eq!( blobs .read_bytes(&result.prompt_entries[&prompt_key].content.content_ref) @@ -541,7 +601,7 @@ mod tests { text.as_bytes() ); } - let before = counts(); + let before = route.counts(); let prompts_only = harness::EnvironmentsFeature { prompts: Some(Default::default()), environments: feature.environments.clone(), @@ -550,12 +610,9 @@ mod tests { let result = refresh_sources(&prompts_only, None).await; assert!(result.skill_command.is_none()); assert!(result.prompt_entries.contains_key(&prompt_key)); - assert_eq!( - counts(), - (before.0 + 1, before.1 + 1, before.2 + 1, before.3 + 1) - ); + assert_eq!(route.counts(), (before.0 + 1, before.1 + 1, before.2 + 1)); - let before = counts(); + let before = route.counts(); let disabled_feature = harness::EnvironmentsFeature::default(); let disabled = refresh_sources(&disabled_feature, None).await; assert!(disabled.skill_command.is_none()); @@ -569,30 +626,22 @@ mod tests { .is_none() ); assert_eq!( - counts(), + route.counts(), before, "disabled and controller-owned sources need no connection" ); let result = refresh_sources(&both, Some(&controller_owned)).await; assert!(result.skill_command.is_none()); assert!(result.prompt_entries.contains_key(&prompt_key)); - assert_eq!( - counts(), - (before.0 + 1, before.1 + 1, before.2 + 1, before.3 + 1) - ); + assert_eq!(route.counts(), (before.0 + 1, before.1 + 1, before.2 + 1)); - // A timed-out skill RPC must not poison the prompt scan with an unread response. - stall_skills.store(true, Ordering::SeqCst); - let before = counts(); + // A timed-out skill scan does not delay the prompt scan, which runs + // on a fresh connection. + route.stall_skills.store(true, SeqCst); + let before = route.counts(); let result = refresh_sources(&both, Some(&edited)).await; let failed_skills = entry(result.skill_command); - let failed_catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(failed_skills.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + let failed_catalog = catalog_of(&failed_skills).await; assert_eq!( failed_catalog.availability, EnvironmentSkillAvailability::Stale @@ -606,79 +655,58 @@ mod tests { .unwrap(), b"Updated instructions" ); - assert_eq!( - counts(), - (before.0 + 2, before.1 + 2, before.2 + 2, before.3 + 2) - ); - stall_skills.store(false, Ordering::SeqCst); + assert_eq!(route.counts(), (before.0 + 2, before.1 + 2, before.2 + 2)); + route.stall_skills.store(false, SeqCst); - // An incomplete scan retains the last observation without appending a menu. - std::fs::write(&skill_path, vec![b'x'; 65537]).unwrap(); - let stale = entry(refresh(Some(&edited)).await.unwrap()); - let catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(stale.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), + // One oversized skill is skipped and reported under its root; the + // root's other skills are still listed. + std::fs::create_dir_all(root.join(".agents/skills/other")).unwrap(); + std::fs::write( + root.join(".agents/skills/other/SKILL.md"), + "---\nname: other\ndescription: Other skill.\n---\nbody", ) .unwrap(); - assert_eq!(catalog.availability, EnvironmentSkillAvailability::Stale); - assert_eq!(catalog.skills.len(), 1); - assert_eq!(stale.content, edited.content); - assert!(refresh(Some(&stale)).await.unwrap().is_none()); - let result = refresh_sources(&both, Some(&edited)).await; - assert!(result.skill_command.is_some()); + std::fs::write(&skill_path, vec![b'x'; 65537]).unwrap(); + let degraded = entry(refresh(Some(&edited)).await.unwrap()); + let catalog = catalog_of(°raded).await; assert_eq!( - blobs - .read_bytes(&result.prompt_entries[&prompt_key].content.content_ref) - .await - .unwrap(), - b"Updated instructions" + catalog.availability, + EnvironmentSkillAvailability::Available ); + assert_eq!( + catalog + .skills + .iter() + .map(|skill| skill.name.as_str()) + .collect::>(), + vec!["other"] + ); + let skills_root = root.join(".agents/skills"); + assert!( + catalog + .warnings + .iter() + .any(|warning| warning.starts_with(skills_root.to_str().unwrap())), + "{:?}", + catalog.warnings + ); + std::fs::remove_dir_all(root.join(".agents/skills/other")).unwrap(); std::fs::write(&skill_path, doc).unwrap(); - std::fs::write(&prompt_path, vec![b'x'; 65537]).unwrap(); - let result = refresh_sources(&both, Some(&stale)).await; - let recovered = entry(result.skill_command); - let recovered_catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(recovered.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + + // Without the filesystem family discovery is explicitly + // unavailable, with no scan. + route.unsupported.store(true, SeqCst); + let before = route.counts().2; + let unsupported = entry(refresh(Some(°raded)).await.unwrap()); + assert_eq!(route.counts().2, before); assert_eq!( - recovered_catalog.availability, - EnvironmentSkillAvailability::Available + catalog_of(&unsupported).await.availability, + EnvironmentSkillAvailability::Stale ); - let prompt_report: tools::prompts::environment::EnvironmentPromptReport = - serde_json::from_slice( - &blobs - .read_bytes( - result.prompt_entries[&prompt_key] - .provenance_ref - .as_ref() - .unwrap(), - ) - .await - .unwrap(), - ) - .unwrap(); - assert!(!prompt_report.available); - // Missing fs/scan is explicit unavailable discovery, with no RPC fallback. - supported.store(false, Ordering::SeqCst); - let before = scans.load(Ordering::SeqCst); - assert!(refresh(Some(&stale)).await.unwrap().is_none()); - assert_eq!(scans.load(Ordering::SeqCst), before); - supported.store(true, Ordering::SeqCst); + route.unsupported.store(false, SeqCst); std::fs::remove_file(&skill_path).unwrap(); - let empty = entry(refresh(Some(&stale)).await.unwrap()); - let catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(empty.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + let empty = entry(refresh(Some(&unsupported)).await.unwrap()); + let catalog = catalog_of(&empty).await; assert!(catalog.skills.is_empty()); assert_eq!( catalog.availability, @@ -715,13 +743,7 @@ mod tests { .unwrap() .skill_command, ); - let denied_catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(denied_entry.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + let denied_catalog = catalog_of(&denied_entry).await; assert_eq!( denied_catalog.availability, EnvironmentSkillAvailability::Unavailable @@ -730,22 +752,15 @@ mod tests { denied_catalog.skills.is_empty(), "a revoked domain grant cannot retain advertised paths" ); - stall.store(true, Ordering::SeqCst); + route.stall.store(true, SeqCst); let start = std::time::Instant::now(); let timed_out = entry(refresh(Some(&available)).await.unwrap()); assert!( start.elapsed() < std::time::Duration::from_secs(6), "unresponsive endpoint must remain bounded" ); - let timed_out_catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(timed_out.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); assert_eq!( - timed_out_catalog.availability, + catalog_of(&timed_out).await.availability, EnvironmentSkillAvailability::Stale ); assert_eq!(timed_out.content, available.content); @@ -769,6 +784,65 @@ mod tests { )); } + #[tokio::test(flavor = "current_thread")] + async fn readiness_reports_a_daemon_of_another_protocol_version() { + use environment_protocol::{error::ProtocolError, jsonrpc}; + use futures_util::{SinkExt, StreamExt}; + let (resolver, environment_id) = resolver().await; + resolver + .environments + .observe_provisioned_environment(observation( + &environment_id, + EnvironmentStatus::Ready, + vec![PowerState::Running], + 20, + )) + .await + .unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let remote = environment_protocol::ENVD_PROTOCOL_VERSION + 1; + let task = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap(); + while let Some(Ok(message)) = socket.next().await { + let Ok(text) = message.to_text() else { + continue; + }; + let Ok(jsonrpc::Message::Request(request)) = jsonrpc::parse(text) else { + continue; + }; + let response = jsonrpc::Response::failure( + Some(request.id), + ProtocolError::version_mismatch( + remote, + environment_protocol::ENVD_PROTOCOL_VERSION, + ), + ); + let _ = socket + .send(tokio_tungstenite::tungstenite::Message::Text( + response.to_text().into(), + )) + .await; + } + }); + let resolver = resolver.with_gateway( + crate::environments::gateway::EnvironmentGatewayClientConfig::new(base_url, "test"), + ); + match resolver.ready_for_use(&environment_id, 30).await { + Err(EnvironmentResolveError::VersionMismatch { + local, + remote: reported, + .. + }) => { + assert_eq!(local, environment_protocol::ENVD_PROTOCOL_VERSION); + assert_eq!(reported, remote); + } + other => panic!("expected a version mismatch, got {other:?}"), + } + task.abort(); + } + #[tokio::test(flavor = "current_thread")] async fn selection_preserves_power_and_needs_no_gateway_in_any_nonterminal_state() { let (resolver, environment_id) = resolver().await; @@ -791,13 +865,12 @@ mod tests { EnvironmentStatus::Unknown, ] { store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), + .observe_provisioned_environment(observation( + &environment_id, status, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 30, - }) + vec![PowerState::Running, PowerState::Paused], + 30, + )) .await .unwrap(); let before = store.read_environment(&environment_id).await.unwrap(); @@ -830,6 +903,7 @@ mod tests { .fail_environment_lifecycle(environments::FailEnvironmentLifecycle { environment_id: environment_id.clone(), message: "no capacity".into(), + daemon_build: None, observed_at_ms: 40, }) .await @@ -856,13 +930,12 @@ mod tests { let (resolver, environment_id) = resolver().await; let store = resolver.environments.clone(); store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 20, - }) + .observe_provisioned_environment(observation( + &environment_id, + EnvironmentStatus::Ready, + vec![PowerState::Running, PowerState::Paused], + 20, + )) .await .expect("observe"); // The reaper decided to pause; the reconciler has not acted yet. @@ -896,13 +969,12 @@ mod tests { let environment_id = environment_id.clone(); async move { store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id, - provider_target_id: ProviderTargetId::new("target-1"), + .observe_provisioned_environment(observation( + &environment_id, status, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: at, - }) + vec![PowerState::Running, PowerState::Paused], + at, + )) .await .expect("observe"); } @@ -948,13 +1020,12 @@ mod tests { // A stopped environment whose provider offers no power control keeps // the old behaviour: no wake, plain unavailability. store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Offline, - power_states: Vec::new(), - observed_at_ms: 60, - }) + .observe_provisioned_environment(observation( + &environment_id, + EnvironmentStatus::Offline, + Vec::new(), + 60, + )) .await .expect("observe offline without power control"); assert!(matches!( @@ -980,13 +1051,12 @@ mod tests { let environment_id = environment_id.clone(); async move { store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id, - provider_target_id: ProviderTargetId::new("target-1"), + .observe_provisioned_environment(observation( + &environment_id, status, - power_states: Vec::new(), - observed_at_ms: 20, - }) + Vec::new(), + 20, + )) .await .expect("observe"); } @@ -1020,6 +1090,7 @@ mod tests { .fail_environment_lifecycle(environments::FailEnvironmentLifecycle { environment_id: environment_id.clone(), message: "no capacity".to_owned(), + daemon_build: None, observed_at_ms: 40, }) .await diff --git a/crates/temporal-runtime/src/environments/runtime.rs b/crates/temporal-runtime/src/environments/runtime.rs index cd453e6de..db785bbf1 100644 --- a/crates/temporal-runtime/src/environments/runtime.rs +++ b/crates/temporal-runtime/src/environments/runtime.rs @@ -2,7 +2,6 @@ use std::collections::BTreeMap; -use environment_client::WebSocketTransport; use environments::EnvironmentRecord; use tools::{ environment::EnvironmentToolContext, environment_protocol::RemoteEnvironmentConnection, @@ -12,7 +11,7 @@ use tools::{ pub struct RuntimeEnvironment { resource: EnvironmentRecord, tool_context: EnvironmentToolContext, - remote_connection: Option>, + remote_connection: Option, } impl RuntimeEnvironment { @@ -34,10 +33,7 @@ impl RuntimeEnvironment { } } - pub fn with_remote_connection( - mut self, - connection: RemoteEnvironmentConnection, - ) -> Self { + pub fn with_remote_connection(mut self, connection: RemoteEnvironmentConnection) -> Self { self.remote_connection = Some(connection); self } @@ -143,7 +139,7 @@ impl SessionEnvironmentManager { mod tests { use std::{collections::BTreeMap, sync::Arc}; - use environment_protocol::shared::ProviderTargetId; + use environment_protocol::ids::ProviderTargetId; use environments::{ EnvironmentIncarnationId, EnvironmentIncarnationRecord, EnvironmentProviderBindingId, EnvironmentProviderId, EnvironmentProvisionRequestId, EnvironmentSource, EnvironmentStatus, @@ -155,7 +151,7 @@ mod tests { use super::*; fn resource() -> EnvironmentRecord { - let target_id = ProviderTargetId::new("target-a"); + let target_id = ProviderTargetId::new("target-a").unwrap(); EnvironmentRecord { environment_id: harness::EnvironmentId::new("environment-a"), request_id: EnvironmentProvisionRequestId::new("request-a"), @@ -181,6 +177,7 @@ mod tests { public_endpoint: None, metadata: BTreeMap::from([("fsRoot".to_owned(), "/sandbox".to_owned())]), last_seen_at_ms: None, + lifecycle_error: None, created_at_ms: 1, updated_at_ms: 1, } diff --git a/crates/temporal-runtime/src/environments/skills.rs b/crates/temporal-runtime/src/environments/skills.rs index a9037fa40..69f0303ee 100644 --- a/crates/temporal-runtime/src/environments/skills.rs +++ b/crates/temporal-runtime/src/environments/skills.rs @@ -11,15 +11,11 @@ use std::{ }; use tools::skills::environment::*; -#[derive(Clone)] -struct CachedObservation { - fingerprint: String, - catalog: EnvironmentSkillCatalog, -} -// An optimization only: losing/evicting this cache forces a complete scan. The -// cached catalog is reused only after the environment confirms its fingerprint. -static OBSERVATIONS: OnceLock>> = OnceLock::new(); -fn observations() -> &'static Mutex> { +// An optimization only: losing/evicting this cache forces a full scan. A +// cached catalog contributes only the roots the environment confirms +// unchanged by their fingerprints. +static OBSERVATIONS: OnceLock>> = OnceLock::new(); +fn observations() -> &'static Mutex> { OBSERVATIONS.get_or_init(Default::default) } @@ -56,10 +52,14 @@ pub(crate) async fn refresh( .expect("serialize discovery scope"); let attempt = async { let connection = discovery.connection().await?; - let mut query = environment_skill_scan_query( + let query = environment_skill_scan_query( config, - Some(&connection.cwd), - connection.initialized.home_directory.as_deref(), + Some(connection.cwd.as_str()), + connection + .initialized + .home_directory + .as_ref() + .map(|home| home.as_str()), )?; let cache_key = serde_json::to_string(&( session_id, @@ -74,42 +74,30 @@ pub(crate) async fn refresh( .expect("observation lock") .get(&cache_key) .cloned(); - query.if_none_match = cached.as_ref().map(|c| c.fingerprint.clone()); - let scan = { + let catalog = { let _timer = PhaseTimer::new("skills_scan"); - connection - .client - .scan(&query) - .await - .map_err(|e| e.to_string())? - }; - if !scan.complete { - return Err(format!( - "incomplete environment discovery: {:?}", - scan.diagnostics - )); - } - if scan.unchanged { - return cached - .filter(|cached| scan.fingerprint.as_ref() == Some(&cached.fingerprint)) - .map(|cached| cached.catalog) - .ok_or("unexpected unchanged scan without matching observation".to_owned()); - } - let catalog = environment_skill_catalog(environment_id.as_str(), &scan)?; - if let Some(fingerprint) = scan.fingerprint { - let mut cache = observations().lock().expect("observation lock"); - if cache.len() >= 128 { - cache.clear(); + let mut cached_query = query.clone(); + if let Some(previous) = &cached { + apply_cached_fingerprints(&mut cached_query, previous); } - cache.insert( - cache_key, - CachedObservation { - fingerprint, - catalog: catalog.clone(), - }, - ); + let scan = connection.scan(cached_query).await?; + match environment_skill_catalog(environment_id.as_str(), &scan, cached.as_ref()) { + Ok(catalog) => catalog, + // A root answered unchanged against an observation the + // catalog cannot reuse: observe every root in full. + Err(_) if cached.is_some() => { + let scan = connection.scan(query).await?; + environment_skill_catalog(environment_id.as_str(), &scan, None)? + } + Err(error) => return Err(error), + } + }; + let mut cache = observations().lock().expect("observation lock"); + if cache.len() >= 128 { + cache.clear(); } - Ok(catalog) + cache.insert(cache_key, catalog.clone()); + Ok::<_, String>(catalog) }; let mut catalog = match tokio::time::timeout(Duration::from_secs(4), attempt).await { Ok(Ok(catalog)) => catalog, @@ -166,6 +154,7 @@ mod tests { catalog.availability = EnvironmentSkillAvailability::Available; catalog.discovery_scope = Some("configured roots".into()); catalog.skills.push(EnvironmentSkill { + root: "/skills".into(), skill_id: tools::skills::SkillId::new("internal-id"), name: "review".into(), description: "Review changes".into(), diff --git a/crates/temporal-runtime/src/environments/sources.rs b/crates/temporal-runtime/src/environments/sources.rs index 8de5fd1e9..9f66bdbaf 100644 --- a/crates/temporal-runtime/src/environments/sources.rs +++ b/crates/temporal-runtime/src/environments/sources.rs @@ -3,13 +3,13 @@ use crate::{ environments::gateway::EnvironmentGatewayClientConfig, environments::resolver::EnvironmentResolver, }; -use environment_client::{EnvironmentDataClient, JsonRpcTransport, WebSocketTransport}; +use environment_client::DataClient; use environment_protocol::{ data::{ - fs::GetMetadataParams, - handshake::{InitializeParams, InitializeResponse, InitializedParams}, + handshake::InitializeResponse, + scan::{ScanParams, ScanResponse, ScanRootResult}, }, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentPath}, + paths::EnvironmentPath, }; use harness::{ ContextEntryInput, ContextEntryKey, CoreAgentCommand, EnvironmentId, EnvironmentsFeature, @@ -21,30 +21,25 @@ use std::{ time::{Duration, Instant}, }; -pub(crate) async fn working_directory( - client: &mut EnvironmentDataClient, +/// `clientInfo.name` of discovery connections. +const SOURCE_DISCOVERY_CLIENT: &str = "lightspeed-source-discovery"; + +/// The working directory discovery resolves sources against: the +/// attachment's configured directory, else the daemon's `defaultCwd`. It is +/// proven to exist by the probe root of each discovery scan, not by a +/// separate call. +pub(crate) fn working_directory( configured: Option<&str>, - default: Option<&str>, -) -> Result { - let value = configured - .or(default) - .ok_or("environment does not advertise a default working directory")?; + default: &EnvironmentPath, +) -> Result { + let Some(value) = configured else { + return Ok(default.clone()); + }; if !value.starts_with('/') { return Err("environment working directory must be absolute".into()); } - let cwd = tools::environment::sources::absolute(std::path::Path::new("/"), value)? - .to_string_lossy() - .into_owned(); - let metadata = client - .get_metadata(&GetMetadataParams { - path: EnvironmentPath::new(&cwd).map_err(|e| e.to_string())?, - }) - .await - .map_err(|e| format!("working directory {cwd}: {e}"))?; - if !metadata.is_directory { - return Err(format!("working directory is not a directory: {cwd}")); - } - Ok(cwd) + let cwd = tools::environment::sources::absolute(std::path::Path::new("/"), value)?; + EnvironmentPath::new(cwd.to_string_lossy()).map_err(|error| error.to_string()) } pub(crate) struct Discovery<'a> { @@ -56,13 +51,37 @@ pub(crate) struct Discovery<'a> { } pub(crate) struct SourceConnection { - pub client: EnvironmentDataClient, + pub client: DataClient, pub initialized: InitializeResponse, - pub cwd: String, + pub cwd: EnvironmentPath, // Includes universe and incarnation so cached observations cannot cross routes. pub identity: String, } +impl SourceConnection { + /// Run a discovery scan with the working-directory probe appended, check + /// that probe, and return the response without it. `fs/scan` is an + /// observation, so discovery never resets the idle clock. + pub async fn scan(&self, mut query: ScanParams) -> Result { + tools::environment::sources::with_working_directory_probe(&mut query, &self.cwd); + let mut scan = self + .client + .scan(&query) + .await + .map_err(|error| error.to_string())?; + match scan.roots.pop() { + Some(ScanRootResult::Complete { .. } | ScanRootResult::Unchanged { .. }) => Ok(scan), + Some(ScanRootResult::Missing { .. }) => { + Err(format!("working directory {} does not exist", self.cwd)) + } + Some(ScanRootResult::Failed { error, .. }) => { + Err(format!("working directory {}: {error}", self.cwd)) + } + None => Err("discovery scan answered without its working directory probe".into()), + } + } +} + impl Discovery<'_> { pub async fn connection(&mut self) -> Result<&mut SourceConnection, String> { if self.connection.is_none() { @@ -84,8 +103,13 @@ impl Discovery<'_> { } pub fn discard_connection(&mut self) { - // A cancelled RPC can leave an unread response. Never reuse that stream. - self.connection = None; + // Abandoned scans were cancelled by the multiplexed client; a fresh + // connection keeps a failed attempt from affecting the next source. + if let Some(connection) = self.connection.take() { + tokio::spawn(async move { + let _ = connection.client.close().await; + }); + } } } @@ -123,7 +147,7 @@ pub(crate) async fn refresh( }) } .await; - if let Some(mut connection) = discovery.connection.take() { + if let Some(connection) = discovery.connection.take() { let _close = PhaseTimer::new("close"); let _ = tokio::time::timeout(Duration::from_secs(1), connection.client.close()).await; } @@ -171,49 +195,32 @@ async fn connect( { return Err("environment is not accessible; discovery does not wake it".into()); } - let connection = gateway.connection_for(resolver.universe_id(), &environment); - let identity = - serde_json::to_string(&(resolver.universe_id(), &connection)).map_err(|e| e.to_string())?; - let mut client = { + let endpoint = gateway.endpoint_for(resolver.universe_id(), &environment); + let identity = serde_json::to_string(&( + resolver.universe_id(), + environment.environment_id.as_str(), + environment.incarnation.incarnation_id.as_str(), + )) + .map_err(|e| e.to_string())?; + let (client, initialized) = { let _timer = PhaseTimer::new("connect"); - EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("lightspeed-source-discovery"), - ) - .await - .map_err(|e| e.to_string())? - }; - let initialized = { - let _timer = PhaseTimer::new("initialize"); - let initialized = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-source-discovery".into(), - scope: connection.scope, - resume_connection_id: None, - }) - .await - .map_err(|e| e.to_string())?; - client - .initialized(&InitializedParams {}) + DataClient::connect_initialized(&endpoint, gateway.connect_options(SOURCE_DISCOVERY_CLIENT)) .await - .map_err(|e| e.to_string())?; - initialized + .map_err(|e| e.to_string())? }; - if initialized.protocol_version != CURRENT_PROTOCOL_VERSION - || !initialized.capabilities.filesystem_read - || !initialized.capabilities.filesystem_scan - { + if initialized.capabilities.filesystem.is_none() { + let _ = client.close().await; return Err("endpoint does not support filesystem source discovery".into()); } - let cwd = { - let _timer = PhaseTimer::new("working_directory"); - working_directory( - &mut client, - attachment.working_directory.as_deref(), - initialized.default_cwd.as_deref(), - ) - .await? + let cwd = match working_directory( + attachment.working_directory.as_deref(), + &initialized.default_cwd, + ) { + Ok(cwd) => cwd, + Err(error) => { + let _ = client.close().await; + return Err(error); + } }; Ok(SourceConnection { client, @@ -222,3 +229,22 @@ async fn connect( identity, }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_working_directory_is_configured_or_the_daemon_default() { + let default = EnvironmentPath::new("/workspace").unwrap(); + assert_eq!(working_directory(None, &default).unwrap(), default); + assert_eq!( + working_directory(Some("/srv/app/./src/.."), &default) + .unwrap() + .as_str(), + "/srv/app" + ); + assert!(working_directory(Some("relative"), &default).is_err()); + assert!(working_directory(Some("/../escape"), &default).is_err()); + } +} diff --git a/crates/temporal-runtime/src/gateway/deployment.rs b/crates/temporal-runtime/src/gateway/deployment.rs index eda0a394c..ef63f75b0 100644 --- a/crates/temporal-runtime/src/gateway/deployment.rs +++ b/crates/temporal-runtime/src/gateway/deployment.rs @@ -19,26 +19,27 @@ use api::{ DeploymentApiKeyListParams, DeploymentApiKeyListResponse, DeploymentApiKeyRevokeParams, DeploymentApiKeyRevokeResponse, DeploymentApiKeyRotateParams, DeploymentApiKeyView, DeploymentApiService, DeploymentEnvironmentAdoptParams, DeploymentEnvironmentAdoptResponse, - DeploymentEnvironmentProviderConnection, DeploymentEnvironmentProviderDeleteParams, - DeploymentEnvironmentProviderDeleteResponse, DeploymentEnvironmentProviderListParams, + DeploymentEnvironmentProviderDeleteParams, DeploymentEnvironmentProviderDeleteResponse, + DeploymentEnvironmentProviderFeaturesView, DeploymentEnvironmentProviderListParams, DeploymentEnvironmentProviderListResponse, DeploymentEnvironmentProviderPutParams, DeploymentEnvironmentProviderPutResponse, DeploymentEnvironmentProviderReadParams, - DeploymentEnvironmentProviderReadResponse, DeploymentEnvironmentProviderTransport, - DeploymentEnvironmentProviderView, DeploymentProviderBindingDeleteParams, - DeploymentProviderBindingDeleteResponse, DeploymentProviderBindingPutParams, - DeploymentProviderBindingPutResponse, DeploymentUniverseCreateParams, - DeploymentUniverseCreateResponse, DeploymentUniverseDeleteParams, - DeploymentUniverseDeleteResponse, DeploymentUniverseListParams, DeploymentUniverseListResponse, - DeploymentUniverseReadParams, DeploymentUniverseReadResponse, DeploymentUniverseView, + DeploymentEnvironmentProviderReadResponse, DeploymentEnvironmentProviderView, + DeploymentProviderBindingDeleteParams, DeploymentProviderBindingDeleteResponse, + DeploymentProviderBindingPutParams, DeploymentProviderBindingPutResponse, + DeploymentUniverseCreateParams, DeploymentUniverseCreateResponse, + DeploymentUniverseDeleteParams, DeploymentUniverseDeleteResponse, DeploymentUniverseListParams, + DeploymentUniverseListResponse, DeploymentUniverseReadParams, DeploymentUniverseReadResponse, + DeploymentUniverseView, }; use async_trait::async_trait; -use environment_protocol::shared::EnvironmentTransport; +use auth::SecretValue; use environments::{ - AdoptEnvironment, EnvironmentConnectionSpec, EnvironmentId, EnvironmentIncarnationId, + AdoptEnvironment, EnvironmentEndpoint, EnvironmentId, EnvironmentIncarnationId, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderRecord, - EnvironmentProviderStore, EnvironmentProvisionRequestId, EnvironmentStore, - ListEnvironmentProviders, PutEnvironmentProvider, PutEnvironmentProviderBinding, + EnvironmentProviderStore, EnvironmentProvisionRequestId, EnvironmentRegistryError, + EnvironmentStore, ListEnvironmentProviders, PutEnvironmentProvider, + PutEnvironmentProviderBinding, RecordProviderController, }; use harness::SessionId; use object_store::ObjectStoreExt as _; @@ -47,7 +48,12 @@ use temporal_workflow::{AgentSessionWorkflow, compose_workflow_id}; use temporalio_client::{Client, WorkflowTerminateOptions, errors::WorkflowInteractionError}; use uuid::Uuid; -use crate::universe::UniverseRuntime; +use crate::{ + environments::provider_controllers::{ + ControllerFeature, WebSocketProviderControllerConnector, probe_controller, + }, + universe::UniverseRuntime, +}; pub struct GatewayDeploymentApi { runtime: Arc, @@ -262,19 +268,65 @@ impl DeploymentApiService for GatewayDeploymentApi { ) -> Result, AgentApiError> { self.admitted(api::METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_PUT, async { let store = self.runtime.stores().store_for(Uuid::nil()); + let provider_id = parse_environment_provider_id(params.provider_id)?; + let controller_endpoint = EnvironmentEndpoint::parse(¶ms.controller_endpoint) + .map_err(|error| { + AgentApiError::invalid_request(format!("invalid controller endpoint: {error}")) + })?; + let credential = params.credential.map(SecretValue::new); + if let Some(credential) = &credential { + environments::validate_provider_credential(credential) + .map_err(super::service::environment_providers::map_environments_error)?; + } + // The handshake presents the credential the provider will hold: + // the new one, else the stored one. + let presented = match &credential { + Some(credential) => Some(credential.clone()), + None => match store.read_provider_credential(&provider_id).await { + Ok(stored) => Some(stored), + Err(EnvironmentRegistryError::NotFound { .. }) => None, + Err(error) => { + return Err( + super::service::environment_providers::map_environments_error(error), + ); + } + }, + }; + let facts = match &presented { + Some(credential) => { + probe_controller( + &WebSocketProviderControllerConnector, + &controller_endpoint, + credential, + ) + .await? + } + None => None, + }; let provider = store .put_provider(PutEnvironmentProvider { - provider_id: parse_environment_provider_id(params.provider_id)?, + provider_id: provider_id.clone(), display_name: params.display_name, - controller_connection: provider_connection_from_api( - params.controller_connection, - ), + controller_endpoint: controller_endpoint.clone(), metadata: params.metadata, + credential, updated_at_ms: i64::try_from(current_time_ms()?) .map_err(|_| AgentApiError::internal("current timestamp exceeds i64"))?, }) .await .map_err(super::service::environment_providers::map_environments_error)?; + let provider = match facts { + Some((features, data_route_base)) => store + .record_provider_controller(RecordProviderController { + provider_id, + controller_endpoint, + features, + data_route_base, + }) + .await + .map_err(super::service::environment_providers::map_environments_error)?, + None => provider, + }; Ok(AgentApiOutcome::new( DeploymentEnvironmentProviderPutResponse { provider: environment_provider_view(provider), @@ -441,6 +493,20 @@ impl DeploymentApiService for GatewayDeploymentApi { })?; validate_adoption_source(¶ms.source_target)?; let store = self.runtime.stores().store_for(universe_id); + // Refuse up front when the binding's provider cannot adopt, instead + // of failing the environment after its record exists. + let binding = store + .read_provider_binding(universe_id, &binding_id) + .await + .map_err(super::service::environment_providers::map_environments_error)?; + self.runtime + .state_for(universe_id, false) + .await + .map_err(|error| AgentApiError::internal(error.to_string()))? + .api + .environment_service() + .require_controller_feature(&binding.provider_id, ControllerFeature::AdoptTarget) + .await?; let environment = EnvironmentStore::adopt_environment( store.as_ref(), AdoptEnvironment { @@ -784,44 +850,21 @@ fn parse_environment_provider_id(value: String) -> Result EnvironmentConnectionSpec { - EnvironmentConnectionSpec { - endpoint: connection.endpoint, - transport: match connection.transport { - DeploymentEnvironmentProviderTransport::WebSocket => EnvironmentTransport::WebSocket, - DeploymentEnvironmentProviderTransport::Http => EnvironmentTransport::Http, - DeploymentEnvironmentProviderTransport::Stdio => EnvironmentTransport::Stdio, - DeploymentEnvironmentProviderTransport::Ssh => EnvironmentTransport::Ssh, - DeploymentEnvironmentProviderTransport::Provider { provider_type } => { - EnvironmentTransport::Provider { provider_type } - } - }, - } -} - fn environment_provider_view( record: EnvironmentProviderRecord, ) -> DeploymentEnvironmentProviderView { DeploymentEnvironmentProviderView { provider_id: record.provider_id.to_string(), display_name: record.display_name, - controller_connection: DeploymentEnvironmentProviderConnection { - endpoint: record.controller_connection.endpoint, - transport: match record.controller_connection.transport { - EnvironmentTransport::WebSocket => { - DeploymentEnvironmentProviderTransport::WebSocket - } - EnvironmentTransport::Http => DeploymentEnvironmentProviderTransport::Http, - EnvironmentTransport::Stdio => DeploymentEnvironmentProviderTransport::Stdio, - EnvironmentTransport::Ssh => DeploymentEnvironmentProviderTransport::Ssh, - EnvironmentTransport::Provider { provider_type } => { - DeploymentEnvironmentProviderTransport::Provider { provider_type } - } - }, - }, + controller_endpoint: record.controller_endpoint.as_str().to_owned(), metadata: record.metadata, + controller_features: record.controller_features.map(|features| { + DeploymentEnvironmentProviderFeaturesView { + adopt_target: features.adopt_target, + ingress: features.ingress, + } + }), + data_route_base: record.data_route_base.map(|base| base.as_str().to_owned()), created_at_ms: record.created_at_ms, updated_at_ms: record.updated_at_ms, } diff --git a/crates/temporal-runtime/src/gateway/http.rs b/crates/temporal-runtime/src/gateway/http.rs index 839cf94f0..aa6ff1b5f 100644 --- a/crates/temporal-runtime/src/gateway/http.rs +++ b/crates/temporal-runtime/src/gateway/http.rs @@ -18,18 +18,28 @@ use axum::{ routing::{get, post}, }; use environment_protocol::{ - gateway::{PROVIDER_DATA_PATH_PREFIX, ROUTE_PATH_PREFIX}, - registration::{CONNECT_PATH, DATA_PATH}, + control::target::TargetOwner, + ids::{BindingId, EnvironmentId, IncarnationId, ProviderTargetId, UniverseId}, + limits::MAX_MESSAGE_BYTES, + routes::{CONNECT_PATH, CanonicalOrigin, DATA_PATH, canonical_origin, provider_data_route}, }; +use environments::{EndpointScheme, EnvironmentEndpoint}; use futures_util::{SinkExt as _, StreamExt as _}; use serde::Deserialize; use store_pg::PgApiKeyStore; -use tokio_tungstenite::{connect_async, tungstenite::Message as ProviderMessage}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio_tungstenite::{ + WebSocketStream, client_async_with_config, connect_async_with_config, + tungstenite::{ + Message as UpstreamMessage, client::IntoClientRequest as _, + http::HeaderValue as UpstreamHeaderValue, protocol::WebSocketConfig, + }, +}; use uuid::Uuid; use crate::{ config::{DeploymentStores, GatewayAuthMode, gateway_auth_mode_from_env}, - environments::gateway::{RouteKey, bearer_matches, close_message}, + environments::gateway::{ROUTE_PATH_PREFIX, RouteKey, bearer_matches, close_message}, universe::{UniverseError, UniverseRuntime}, }; @@ -51,8 +61,9 @@ pub struct GatewayServerConfig { pub temporal_target: String, pub namespace: String, pub max_request_body_bytes: usize, - /// Externally reachable base URL for the OAuth callback - /// (`{base}/auth/callback`). Defaults to `http://{bind}`. + /// Externally reachable base URL: the OAuth callback + /// (`{base}/auth/callback`) and the origin registered daemons sign. + /// Defaults to `http://{bind}`, which only a loopback bind may rely on. pub public_base_url: Option, } @@ -80,8 +91,10 @@ pub struct GatewayState { resolution: UniverseResolution, /// Live registered-daemon connections on this replica. registrations: Arc, - /// Public base URL daemons are told to dial for data connections. - public_base_url: String, + /// The origin daemons sign registrations for: the canonical origin of + /// the public base URL. Absent when that URL has no WebSocket origin; + /// the connect route then refuses every daemon. + registration_origin: Option, } /// Which route families one HTTP listener serves, derived from the @@ -106,11 +119,11 @@ impl GatewayRoutes { impl GatewayState { /// Route every request to one existing service instance. pub fn for_api(api: Arc) -> Self { - let public_base_url = api.public_base_url().to_owned(); + let registration_origin = registration_origin_of(api.public_base_url()); Self { resolution: UniverseResolution::FixedApi { api }, registrations: Arc::new(RegisteredConnections::new()), - public_base_url, + registration_origin, } } @@ -118,17 +131,8 @@ impl GatewayState { &self.registrations } - /// Tell daemons to dial a different public base than the general one. - pub fn with_environment_public_url(mut self, url: Option) -> Self { - if let Some(url) = url { - self.public_base_url = url; - } - self - } - - /// Public data route for reverse-dialed daemon sockets. - pub(super) fn registration_data_url(&self) -> String { - registration::data_url(&self.public_base_url) + pub(super) fn registration_origin(&self) -> Option<&CanonicalOrigin> { + self.registration_origin.as_ref() } /// The universe a register request belongs to: the environment's @@ -148,7 +152,7 @@ impl GatewayState { let universe_id = store.config().universe_id; let known = environments::EnvironmentStore::read_environment_by_daemon_public_key( store.as_ref(), - ¶ms.daemon_public_key, + ¶ms.daemon_public_key.to_hex(), ) .await .map_err(|error| AgentApiError::internal(error.to_string()))?; @@ -168,10 +172,12 @@ impl GatewayState { } UniverseResolution::Multi { runtime, .. } => { let pool = runtime.stores().pool(); - if let Some(universe_id) = - store_pg::find_registered_environment_universe(pool, ¶ms.daemon_public_key) - .await - .map_err(|error| AgentApiError::internal(error.to_string()))? + if let Some(universe_id) = store_pg::find_registered_environment_universe( + pool, + ¶ms.daemon_public_key.to_hex(), + ) + .await + .map_err(|error| AgentApiError::internal(error.to_string()))? { return Ok(Some(universe_id)); } @@ -192,16 +198,17 @@ impl GatewayState { ) -> Self { let api_keys = PgApiKeyStore::new(runtime.stores().pool().clone()); let deployment = Arc::new(GatewayDeploymentApi::new(runtime.clone())); + let registration_origin = registration_origin_of(&public_base_url); Self { resolution: UniverseResolution::Multi { mode, runtime, - public_base_url: public_base_url.clone(), + public_base_url, api_keys, deployment, }, registrations: Arc::new(RegisteredConnections::new()), - public_base_url, + registration_origin, } } @@ -321,20 +328,76 @@ impl GatewayState { } } + fn deployment_store(&self) -> Arc { + match &self.resolution { + UniverseResolution::FixedApi { api } => api.store().clone(), + UniverseResolution::Multi { runtime, .. } => runtime.stores().store_for(Uuid::nil()), + } + } + async fn environment_provider( &self, provider_id: &environments::EnvironmentProviderId, - ) -> Result { - let store = match &self.resolution { - UniverseResolution::FixedApi { api } => api.store().clone(), - UniverseResolution::Multi { runtime, .. } => runtime.stores().store_for(Uuid::nil()), - }; - environments::EnvironmentProviderStore::read_provider(store.as_ref(), provider_id) - .await - .map_err(|error| AgentApiError::rejected(error.to_string())) + ) -> Result + { + environments::EnvironmentProviderStore::read_provider( + self.deployment_store().as_ref(), + provider_id, + ) + .await + } + + async fn environment_provider_credential( + &self, + provider_id: &environments::EnvironmentProviderId, + ) -> Result { + environments::EnvironmentProviderStore::read_provider_credential( + self.deployment_store().as_ref(), + provider_id, + ) + .await } } +/// The registration signature origin of a public base URL, if it has one. +fn registration_origin_of(public_base_url: &str) -> Option { + match canonical_origin(public_base_url) { + Ok(origin) => Some(origin), + Err(error) => { + tracing::error!( + target: "temporal_runtime", + %error, + public_base_url, + "the public base URL has no WebSocket origin; daemon registration is refused" + ); + None + } + } +} + +/// The origin registered daemons must sign for, checked once at startup by +/// a process that serves the registration route: the canonical origin of +/// `LIGHTSPEED_PUBLIC_BASE_URL`. Without that variable the default +/// `http://{bind}` is the true origin only on a loopback bind, so any other +/// bind refuses to start rather than reject every remote daemon. +pub fn registration_origin( + public_base_url: Option<&str>, + bind: SocketAddr, +) -> anyhow::Result { + let Some(public_base_url) = public_base_url else { + if !bind.ip().is_loopback() { + anyhow::bail!( + "LIGHTSPEED_PUBLIC_BASE_URL is required to serve environment registration on {bind}: \ + daemons sign the gateway origin they dial, and only that URL names it" + ); + } + return Ok(canonical_origin(&format!("http://{bind}"))?); + }; + canonical_origin(public_base_url).map_err(|error| { + anyhow::anyhow!("LIGHTSPEED_PUBLIC_BASE_URL has no WebSocket origin: {error}") + }) +} + fn map_universe_error(error: UniverseError) -> AgentApiError { match error { UniverseError::Unknown { universe_id } => { @@ -350,6 +413,7 @@ pub async fn serve_gateway(config: GatewayServerConfig) -> anyhow::Result<()> { let stores = DeploymentStores::from_env() .await? .with_blob_cache(crate::config::blob_cache_from_env()?); + registration_origin(config.public_base_url.as_deref(), config.bind)?; let public_base_url = public_base_url_or_default(&config); let runtime = Arc::new(UniverseRuntime::new( client, @@ -535,89 +599,58 @@ async fn environment_route_upgrade( if environment.incarnation.incarnation_id.as_str() != key.incarnation_id { return StatusCode::CONFLICT.into_response(); } + let upgrade = upgrade + .max_message_size(MAX_MESSAGE_BYTES as usize) + .max_frame_size(MAX_MESSAGE_BYTES as usize); match &environment.source { - environments::EnvironmentSource::External { connection } => { - if connection.transport != environment_protocol::shared::EnvironmentTransport::WebSocket - { - return StatusCode::BAD_GATEWAY.into_response(); - } - let endpoint = daemon_data_endpoint(&connection.endpoint); - let daemon_socket = match connect_async(&endpoint).await { - Ok((socket, _)) => socket, - Err(error) => { - tracing::warn!(target: "temporal_runtime", %error, %endpoint, "external environment connection failed"); - return StatusCode::SERVICE_UNAVAILABLE.into_response(); - } + environments::EnvironmentSource::External { endpoint } => { + let upstream = + match dial_upstream(endpoint.as_str(), external_target(endpoint), None).await { + Ok(upstream) => upstream, + Err(failure) => return failure.respond(endpoint.as_str()), + }; + let route = UpstreamRoute::External { + endpoint: endpoint.clone(), }; upgrade - .max_message_size(64 * 1024 * 1024) - .on_upgrade(move |socket| { - proxy_external_route(state, key, endpoint, socket, daemon_socket) - }) + .on_upgrade(move |socket| proxy_upstream(state, key, route, socket, upstream)) .into_response() } environments::EnvironmentSource::Provisioned { provider_id, binding_id, } => { - let Some(target_id) = environment.incarnation.provider_target_id.as_ref() else { + let Some(target_id) = environment.incarnation.provider_target_id.clone() else { return StatusCode::CONFLICT.into_response(); }; - if authorize_provisioned_route( - &state, - &key, - provider_id, - binding_id, - target_id.as_str(), - ) - .await - .is_err() - { + let route = UpstreamRoute::Provider { + provider_id: provider_id.clone(), + binding_id: binding_id.clone(), + target_id, + }; + if authorize_upstream(&state, &key, &route).await.is_err() { return StatusCode::CONFLICT.into_response(); } - let provider = match state.environment_provider(provider_id).await { - Ok(provider) => provider, - Err(_) => return StatusCode::BAD_GATEWAY.into_response(), - }; - let endpoint = match provider_data_endpoint( - &provider.controller_connection.endpoint, - &key, - binding_id.as_str(), - target_id.as_str(), - ) { - Ok(endpoint) => endpoint, - Err(_) => return StatusCode::BAD_GATEWAY.into_response(), + let (endpoint, credential) = match provider_route(&state, &key, &route).await { + Ok(route) => route, + Err(refusal) => return refusal.respond(), }; - let provider_socket = match connect_async(&endpoint).await { - Ok((socket, _)) => socket, - Err(error) => { - tracing::warn!(target: "temporal_runtime", %error, %endpoint, "provider data connection failed"); - return StatusCode::SERVICE_UNAVAILABLE.into_response(); - } + let upstream = match dial_upstream( + &endpoint, + UpstreamTarget::Url, + Some(credential.expose()), + ) + .await + { + Ok(upstream) => upstream, + Err(failure) => return failure.respond(&endpoint), }; - let provider_id = provider_id.clone(); - let binding_id = binding_id.clone(); - let target_id = target_id.to_string(); upgrade - .max_message_size(64 * 1024 * 1024) - .on_upgrade(move |socket| { - proxy_provider_route( - state, - key, - provider_id, - binding_id, - target_id, - socket, - provider_socket, - ) - }) + .on_upgrade(move |socket| proxy_upstream(state, key, route, socket, upstream)) .into_response() } environments::EnvironmentSource::Registered { .. } => { - if matches!( - environment.status, - environments::EnvironmentStatus::Closing | environments::EnvironmentStatus::Closed - ) { + if environment.status.is_closing_or_closed() { return StatusCode::CONFLICT.into_response(); } let (daemon_socket, control_connection_id) = match registration::open_registered_route( @@ -632,7 +665,6 @@ async fn environment_route_upgrade( } }; upgrade - .max_message_size(64 * 1024 * 1024) .on_upgrade(move |socket| { registration::proxy_registered_route( state, @@ -647,92 +679,231 @@ async fn environment_route_upgrade( } } -fn provider_data_endpoint( - controller_endpoint: &str, - key: &RouteKey, - binding_id: &str, - target_id: &str, -) -> anyhow::Result { - let mut endpoint = url::Url::parse(controller_endpoint)?; - let scheme = match endpoint.scheme() { - "http" => "ws", - "https" => "wss", - "ws" => "ws", - "wss" => "wss", - other => anyhow::bail!("unsupported provider controller URL scheme: {other}"), - }; - endpoint - .set_scheme(scheme) - .map_err(|_| anyhow::anyhow!("invalid provider controller URL scheme"))?; - endpoint.set_query(None); - endpoint.set_fragment(None); - let base_path = endpoint - .path() - .trim_end_matches('/') - .strip_suffix("/control") - .ok_or_else(|| anyhow::anyhow!("provider controller endpoint must end in /control"))?; - endpoint.set_path(&format!("{base_path}{PROVIDER_DATA_PATH_PREFIX}")); - endpoint - .path_segments_mut() - .map_err(|_| anyhow::anyhow!("provider controller URL cannot be a base URL"))? - .pop_if_empty() - .push(&key.universe_id.to_string()) - .push(binding_id) - .push(&key.environment_id) - .push(&key.incarnation_id) - .push(target_id); - Ok(endpoint.into()) +/// How long the gateway waits for a provider route or external envd to +/// accept the upstream WebSocket. +const UPSTREAM_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// A provider that has not yet advertised its data routes is asked again +/// after this long. +const PROVIDER_ROUTES_RETRY_AFTER_SECS: u64 = 5; + +/// The upstream a worker route is proxied to, and what keeps it +/// authorized. +enum UpstreamRoute { + External { + endpoint: EnvironmentEndpoint, + }, + Provider { + provider_id: environments::EnvironmentProviderId, + binding_id: environments::EnvironmentProviderBindingId, + target_id: ProviderTargetId, + }, } -fn daemon_data_endpoint(endpoint: &str) -> String { - if let Some(rest) = endpoint.strip_prefix("http://") { - format!("ws://{rest}") - } else if let Some(rest) = endpoint.strip_prefix("https://") { - format!("wss://{rest}") - } else { - endpoint.to_owned() +/// Why a provisioned target's data route cannot be dialed now. +enum ProviderRouteRefusal { + /// The provider's controller has not advertised its data routes yet. + NotAdvertised, + Status(StatusCode), +} + +impl ProviderRouteRefusal { + fn respond(self) -> Response { + match self { + Self::NotAdvertised => ( + StatusCode::SERVICE_UNAVAILABLE, + [( + header::RETRY_AFTER, + HeaderValue::from(PROVIDER_ROUTES_RETRY_AFTER_SECS), + )], + ) + .into_response(), + Self::Status(status) => status.into_response(), + } } } -async fn authorize_external_route( +/// The provider data route for a provisioned target and the credential it +/// is dialed with. The route comes only from the base the controller +/// advertised; a provider that has not advertised one yet is asked to +/// retry, one without a credential is a configuration error. +async fn provider_route( state: &GatewayState, key: &RouteKey, - endpoint: &str, -) -> anyhow::Result<()> { - let api = state - .api_for_daemon(key.universe_id) - .await - .map_err(|error| anyhow::anyhow!(error.message))?; - let environment_id = environments::EnvironmentId::try_new(key.environment_id.clone())?; - let environment = - environments::EnvironmentStore::read_environment(api.store().as_ref(), &environment_id) - .await?; - let environments::EnvironmentSource::External { - connection: assigned_connection, - } = &environment.source + route: &UpstreamRoute, +) -> Result<(String, auth::SecretValue), ProviderRouteRefusal> { + let UpstreamRoute::Provider { + provider_id, + binding_id, + target_id, + } = route else { - anyhow::bail!("external route no longer belongs to an external environment") + return Err(ProviderRouteRefusal::Status(StatusCode::BAD_GATEWAY)); }; - if assigned_connection.transport - != environment_protocol::shared::EnvironmentTransport::WebSocket - || daemon_data_endpoint(&assigned_connection.endpoint) != endpoint - || environment.incarnation.incarnation_id.as_str() != key.incarnation_id - || matches!( - environment.status, - environments::EnvironmentStatus::Closing | environments::EnvironmentStatus::Closed - ) - { - anyhow::bail!("external environment route is stale or no longer authorized") + let unavailable = |error: environments::EnvironmentRegistryError| match error { + environments::EnvironmentRegistryError::Store { .. } => { + tracing::warn!(target: "temporal_runtime", %error, provider = %provider_id, "provider record unavailable"); + ProviderRouteRefusal::Status(StatusCode::SERVICE_UNAVAILABLE) + } + error => { + tracing::error!(target: "temporal_runtime", %error, provider = %provider_id, "provider data route is misconfigured"); + ProviderRouteRefusal::Status(StatusCode::BAD_GATEWAY) + } + }; + let provider = state + .environment_provider(provider_id) + .await + .map_err(unavailable)?; + let Some(base) = provider.data_route_base.as_ref() else { + return Err(ProviderRouteRefusal::NotAdvertised); + }; + let credential = state + .environment_provider_credential(provider_id) + .await + .map_err(unavailable)?; + let owner = target_owner(key, binding_id).map_err(|error| { + tracing::error!(target: "temporal_runtime", %error, environment = %key.environment_id, "route identity is not a valid target owner"); + ProviderRouteRefusal::Status(StatusCode::BAD_REQUEST) + })?; + Ok((provider_data_route(base, &owner, target_id), credential)) +} + +fn target_owner( + key: &RouteKey, + binding_id: &environments::EnvironmentProviderBindingId, +) -> Result { + Ok(TargetOwner { + universe_id: UniverseId::new(key.universe_id.to_string())?, + binding_id: BindingId::new(binding_id.as_str())?, + environment_id: EnvironmentId::new(key.environment_id.as_str())?, + incarnation_id: IncarnationId::new(key.incarnation_id.as_str())?, + }) +} + +/// Where an upstream socket is dialed. +#[derive(Clone, Copy)] +enum UpstreamTarget<'a> { + /// `ws://` or `wss://`, dialed as written. + Url, + /// A Unix socket path; the upgrade targets `/` with `Host: localhost`. + Unix(&'a str), +} + +fn external_target(endpoint: &EnvironmentEndpoint) -> UpstreamTarget<'_> { + match (endpoint.scheme(), endpoint.unix_socket_path()) { + (EndpointScheme::Unix, Some(path)) => UpstreamTarget::Unix(path), + _ => UpstreamTarget::Url, } - Ok(()) } -async fn authorize_provisioned_route( +enum Upstream { + Tcp(Box>>), + Unix(Box>), +} + +/// Why an upstream dial failed, and what the worker is told. +enum UpstreamFailure { + /// Unreachable, slow or answering 5xx/429: the worker may retry. + Unavailable(String), + /// The upstream refused the upgrade for good (wrong credential, no such + /// route) or the URL is unusable. + Refused(String), +} + +impl UpstreamFailure { + fn respond(self, endpoint: &str) -> Response { + match self { + Self::Unavailable(error) => { + tracing::warn!(target: "temporal_runtime", %error, %endpoint, "environment upstream unavailable"); + StatusCode::SERVICE_UNAVAILABLE.into_response() + } + Self::Refused(error) => { + tracing::error!(target: "temporal_runtime", %error, %endpoint, "environment upstream refused the connection"); + StatusCode::BAD_GATEWAY.into_response() + } + } + } +} + +fn upstream_config() -> WebSocketConfig { + WebSocketConfig::default() + .max_message_size(Some(MAX_MESSAGE_BYTES as usize)) + .max_frame_size(Some(MAX_MESSAGE_BYTES as usize)) +} + +/// Dial an upstream data socket with the protocol's message limits, an +/// optional bearer credential and a connect bound. +async fn dial_upstream( + endpoint: &str, + target: UpstreamTarget<'_>, + bearer: Option<&str>, +) -> Result { + let url = match target { + UpstreamTarget::Url => endpoint, + UpstreamTarget::Unix(_) => "ws://localhost/", + }; + let mut request = url + .into_client_request() + .map_err(|error| UpstreamFailure::Refused(error.to_string()))?; + if let Some(bearer) = bearer { + let mut value = + UpstreamHeaderValue::from_str(&format!("Bearer {bearer}")).map_err(|_| { + UpstreamFailure::Refused("the credential is not a valid header value".to_owned()) + })?; + value.set_sensitive(true); + request.headers_mut().insert( + tokio_tungstenite::tungstenite::http::header::AUTHORIZATION, + value, + ); + } + let dialed = tokio::time::timeout(UPSTREAM_CONNECT_TIMEOUT, async { + match target { + UpstreamTarget::Url => { + connect_async_with_config(request, Some(upstream_config()), true) + .await + .map(|(socket, _)| Upstream::Tcp(Box::new(socket))) + .map_err(upstream_failure) + } + UpstreamTarget::Unix(path) => { + let stream = tokio::net::UnixStream::connect(path) + .await + .map_err(|error| UpstreamFailure::Unavailable(error.to_string()))?; + client_async_with_config(request, stream, Some(upstream_config())) + .await + .map(|(socket, _)| Upstream::Unix(Box::new(socket))) + .map_err(upstream_failure) + } + } + }) + .await; + dialed.unwrap_or_else(|_| { + Err(UpstreamFailure::Unavailable(format!( + "no upgrade within {} s", + UPSTREAM_CONNECT_TIMEOUT.as_secs() + ))) + }) +} + +fn upstream_failure(error: tokio_tungstenite::tungstenite::Error) -> UpstreamFailure { + use tokio_tungstenite::tungstenite::Error; + match error { + Error::Http(response) => { + let status = response.status(); + let message = format!("upgrade refused with HTTP {status}"); + if status.is_server_error() || status.as_u16() == 429 { + UpstreamFailure::Unavailable(message) + } else { + UpstreamFailure::Refused(message) + } + } + Error::Url(error) => UpstreamFailure::Refused(error.to_string()), + error => UpstreamFailure::Unavailable(error.to_string()), + } +} + +async fn authorize_upstream( state: &GatewayState, key: &RouteKey, - provider_id: &environments::EnvironmentProviderId, - binding_id: &environments::EnvironmentProviderBindingId, - target_id: &str, + route: &UpstreamRoute, ) -> anyhow::Result<()> { let api = state .api_for_daemon(key.universe_id) @@ -742,118 +913,78 @@ async fn authorize_provisioned_route( let environment = environments::EnvironmentStore::read_environment(api.store().as_ref(), &environment_id) .await?; - let environments::EnvironmentSource::Provisioned { - provider_id: assigned_provider, - binding_id: assigned_binding, - } = &environment.source - else { - anyhow::bail!("provider data routes require a provisioned environment") - }; - if assigned_provider != provider_id - || assigned_binding != binding_id - || environment.incarnation.incarnation_id.as_str() != key.incarnation_id - || environment - .incarnation - .provider_target_id - .as_ref() - .map(|id| id.as_str()) - != Some(target_id) - || matches!( - environment.status, - environments::EnvironmentStatus::Closing | environments::EnvironmentStatus::Closed - ) + if environment.incarnation.incarnation_id.as_str() != key.incarnation_id + || environment.status.is_closing_or_closed() { - anyhow::bail!("provider data route is stale or no longer owned") + anyhow::bail!("environment route is stale or the environment is closing") } - let binding = environments::EnvironmentProviderBindingStore::read_provider_binding( - api.store().as_ref(), - key.universe_id, - binding_id, - ) - .await?; - if binding.status != environments::EnvironmentProviderBindingStatus::Enabled - || binding.provider_id != *provider_id - { - anyhow::bail!("provider binding is disabled or changed") + match (route, &environment.source) { + ( + UpstreamRoute::External { endpoint }, + environments::EnvironmentSource::External { endpoint: assigned }, + ) if assigned == endpoint => Ok(()), + ( + UpstreamRoute::Provider { + provider_id, + binding_id, + target_id, + }, + environments::EnvironmentSource::Provisioned { + provider_id: assigned_provider, + binding_id: assigned_binding, + }, + ) if assigned_provider == provider_id + && assigned_binding == binding_id + && environment.incarnation.provider_target_id.as_ref() == Some(target_id) => + { + let binding = environments::EnvironmentProviderBindingStore::read_provider_binding( + api.store().as_ref(), + key.universe_id, + binding_id, + ) + .await?; + if binding.status != environments::EnvironmentProviderBindingStatus::Enabled + || binding.provider_id != *provider_id + { + anyhow::bail!("provider binding is disabled or changed") + } + Ok(()) + } + _ => anyhow::bail!("environment route no longer matches its environment"), } - Ok(()) } -async fn proxy_provider_route( +async fn proxy_upstream( state: Arc, key: RouteKey, - provider_id: environments::EnvironmentProviderId, - binding_id: environments::EnvironmentProviderBindingId, - target_id: String, - mut worker: WebSocket, - provider: tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream, - >, + route: UpstreamRoute, + worker: WebSocket, + upstream: Upstream, ) { - let (mut provider_writer, mut provider_reader) = provider.split(); - let mut reauthorize = tokio::time::interval(Duration::from_secs(5)); - loop { - tokio::select! { - _ = reauthorize.tick() => { - if authorize_provisioned_route( - &state, &key, &provider_id, &binding_id, &target_id, - ).await.is_err() { - let _ = worker.send(close_message("provider route authorization changed")).await; - break; - } - } - message = worker.recv() => { - let Some(Ok(message)) = message else { break }; - let close = matches!(message, Message::Close(_)); - let message = match message { - Message::Text(value) => ProviderMessage::Text(value.to_string().into()), - Message::Binary(value) => ProviderMessage::Binary(value), - Message::Ping(value) => ProviderMessage::Ping(value), - Message::Pong(value) => ProviderMessage::Pong(value), - Message::Close(_) => ProviderMessage::Close(None), - }; - if provider_writer.send(message).await.is_err() || close { break } - } - message = provider_reader.next() => { - let Some(Ok(message)) = message else { break }; - let close = matches!(message, ProviderMessage::Close(_)); - let message = match message { - ProviderMessage::Text(value) => Some(Message::Text(value.to_string().into())), - ProviderMessage::Binary(value) => Some(Message::Binary(value)), - ProviderMessage::Ping(value) => Some(Message::Ping(value)), - ProviderMessage::Pong(value) => Some(Message::Pong(value)), - ProviderMessage::Close(_) => Some(Message::Close(None)), - ProviderMessage::Frame(_) => None, - }; - if let Some(message) = message - && worker.send(message).await.is_err() - { break } - if close { break } - } - } + match upstream { + Upstream::Tcp(socket) => proxy_frames(state, key, route, worker, *socket).await, + Upstream::Unix(socket) => proxy_frames(state, key, route, worker, *socket).await, } - // Whichever leg ended first, complete the WebSocket closing handshake on - // both sides instead of dropping the opposite TCP stream abruptly. - let _ = provider_writer.send(ProviderMessage::Close(None)).await; - let _ = worker.send(Message::Close(None)).await; } -async fn proxy_external_route( +/// Pipe one worker route and its upstream frame for frame, pings and pongs +/// included, re-checking the route's authorization on an interval. +async fn proxy_frames( state: Arc, key: RouteKey, - endpoint: String, + route: UpstreamRoute, mut worker: WebSocket, - daemon: tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream, - >, -) { - let (mut daemon_writer, mut daemon_reader) = daemon.split(); + upstream: WebSocketStream, +) where + S: AsyncRead + AsyncWrite + Unpin, +{ + let (mut upstream_writer, mut upstream_reader) = upstream.split(); let mut reauthorize = tokio::time::interval(Duration::from_secs(5)); loop { tokio::select! { _ = reauthorize.tick() => { - if authorize_external_route(&state, &key, &endpoint).await.is_err() { - let _ = worker.send(close_message("external route authorization changed")).await; + if authorize_upstream(&state, &key, &route).await.is_err() { + let _ = worker.send(close_message("environment route authorization changed")).await; break; } } @@ -861,24 +992,24 @@ async fn proxy_external_route( let Some(Ok(message)) = message else { break }; let close = matches!(message, Message::Close(_)); let message = match message { - Message::Text(value) => ProviderMessage::Text(value.to_string().into()), - Message::Binary(value) => ProviderMessage::Binary(value), - Message::Ping(value) => ProviderMessage::Ping(value), - Message::Pong(value) => ProviderMessage::Pong(value), - Message::Close(_) => ProviderMessage::Close(None), + Message::Text(value) => UpstreamMessage::Text(value.to_string().into()), + Message::Binary(value) => UpstreamMessage::Binary(value), + Message::Ping(value) => UpstreamMessage::Ping(value), + Message::Pong(value) => UpstreamMessage::Pong(value), + Message::Close(_) => UpstreamMessage::Close(None), }; - if daemon_writer.send(message).await.is_err() || close { break } + if upstream_writer.send(message).await.is_err() || close { break } } - message = daemon_reader.next() => { + message = upstream_reader.next() => { let Some(Ok(message)) = message else { break }; - let close = matches!(message, ProviderMessage::Close(_)); + let close = matches!(message, UpstreamMessage::Close(_)); let message = match message { - ProviderMessage::Text(value) => Some(Message::Text(value.to_string().into())), - ProviderMessage::Binary(value) => Some(Message::Binary(value)), - ProviderMessage::Ping(value) => Some(Message::Ping(value)), - ProviderMessage::Pong(value) => Some(Message::Pong(value)), - ProviderMessage::Close(_) => Some(Message::Close(None)), - ProviderMessage::Frame(_) => None, + UpstreamMessage::Text(value) => Some(Message::Text(value.to_string().into())), + UpstreamMessage::Binary(value) => Some(Message::Binary(value)), + UpstreamMessage::Ping(value) => Some(Message::Ping(value)), + UpstreamMessage::Pong(value) => Some(Message::Pong(value)), + UpstreamMessage::Close(_) => Some(Message::Close(None)), + UpstreamMessage::Frame(_) => None, }; if let Some(message) = message && worker.send(message).await.is_err() @@ -887,9 +1018,10 @@ async fn proxy_external_route( } } } - // A worker activity may be cancelled or its client may disappear without - // a close frame. Do not propagate that abrupt teardown to envd. - let _ = daemon_writer.send(ProviderMessage::Close(None)).await; + // Whichever leg ended first, complete the WebSocket closing handshake on + // both sides instead of dropping the opposite stream abruptly; a worker + // that vanished without a close frame never tears envd down abruptly. + let _ = upstream_writer.send(UpstreamMessage::Close(None)).await; let _ = worker.send(Message::Close(None)).await; } @@ -1151,32 +1283,65 @@ mod tests { } #[test] - fn provider_data_url_is_derived_and_path_segments_are_escaped() { + fn provider_data_routes_come_from_the_advertised_base() { let key = RouteKey { universe_id: Uuid::parse_str("6f3a1a52-58c1-4f0e-9c2d-1a2b3c4d5e6f").unwrap(), - environment_id: "environment with space".to_owned(), - incarnation_id: "incarnation/one".to_owned(), + environment_id: "environment_1".to_owned(), + incarnation_id: "incarnation_1".to_owned(), }; - let endpoint = provider_data_endpoint( - "https://provider.example/control", + let owner = target_owner( &key, - "primary", - "target-one", + &environments::EnvironmentProviderBindingId::new("primary"), ) .unwrap(); + let base = + environment_protocol::routes::DataRouteBase::new("wss://provider.example/routes") + .unwrap(); assert_eq!( - endpoint, - "wss://provider.example/routes/6f3a1a52-58c1-4f0e-9c2d-1a2b3c4d5e6f/primary/environment%20with%20space/incarnation%2Fone/target-one" + provider_data_route( + &base, + &owner, + &ProviderTargetId::new("default/pod one").unwrap() + ), + "wss://provider.example/routes/6f3a1a52-58c1-4f0e-9c2d-1a2b3c4d5e6f/primary/environment_1/incarnation_1/default%2Fpod%20one" ); - let nested = provider_data_endpoint( - "https://provider.example/services/incus/control", - &key, - "primary", - "target-one", - ) - .unwrap(); - assert!(nested.starts_with("wss://provider.example/services/incus/routes/")); - assert!(provider_data_endpoint("wss://provider.example/custom", &key, "b", "t").is_err()); + } + + #[test] + fn upstream_refusals_split_into_retryable_and_permanent() { + let refused = |status: u16| { + upstream_failure(tokio_tungstenite::tungstenite::Error::Http(Box::new( + tokio_tungstenite::tungstenite::http::Response::builder() + .status(status) + .body(None) + .unwrap(), + ))) + }; + assert!(matches!(refused(401), UpstreamFailure::Refused(_))); + assert!(matches!(refused(404), UpstreamFailure::Refused(_))); + assert!(matches!(refused(503), UpstreamFailure::Unavailable(_))); + assert!(matches!(refused(429), UpstreamFailure::Unavailable(_))); + } + + #[test] + fn the_registration_origin_comes_from_the_public_base_url() { + let loopback: SocketAddr = "127.0.0.1:18080".parse().unwrap(); + let public: SocketAddr = "0.0.0.0:18080".parse().unwrap(); + assert_eq!( + registration_origin(None, loopback).unwrap().as_str(), + "ws://127.0.0.1:18080" + ); + assert!( + registration_origin(None, public).is_err(), + "a non-loopback bind needs the public base URL" + ); + assert_eq!( + registration_origin(Some("https://Lightspeed.example/base/"), public) + .unwrap() + .as_str(), + "wss://lightspeed.example" + ); + assert!(registration_origin(Some("lightspeed.example"), public).is_err()); } #[test] diff --git a/crates/temporal-runtime/src/gateway/registration.rs b/crates/temporal-runtime/src/gateway/registration.rs index a1c24bdd9..4516e8db9 100644 --- a/crates/temporal-runtime/src/gateway/registration.rs +++ b/crates/temporal-runtime/src/gateway/registration.rs @@ -1,20 +1,27 @@ //! Outbound `envd` registration on the environment gateway. //! -//! One public route accepts control connections: the daemon proves its -//! identity against a nonce, is admitted by a registration key the first +//! One public route accepts control connections: the registration channel +//! of `environment_protocol::registration`, on the shared JSON-RPC +//! envelope with the daemon as the client. The gateway sends the frozen +//! `registration/challenge` first; the daemon proves its identity in +//! `registration/register`, is admitted by a registration key the first //! time and by its identity afterwards, and then holds the socket while the -//! gateway pings it and asks it to open data connections. A second public -//! route accepts those reverse-dialed data sockets and pairs each with the -//! worker route that requested it; the pair is then proxied frame for frame -//! exactly like a passive external environment. +//! gateway pings it and asks it, with `registration/openStream`, to dial +//! data streams. A second public route accepts those reverse-dialed stream +//! sockets and pairs each with the worker route that requested it; the +//! pair is then proxied frame for frame exactly like a passive external +//! environment. The gateway never names a URL: the daemon derives the +//! stream URL from its own connect URL. //! -//! The gateway holds no durable lifecycle authority here. Connect, -//! heartbeat, and disconnect are written as observations on the environment -//! row; the lifecycle reconciler derives ephemeral cleanup and stale -//! repair from them. +//! Connect, heartbeat, and disconnect are written as observations on the +//! environment row; the lifecycle reconciler derives ephemeral cleanup and +//! stale repair from them. The one lifecycle decision the gateway makes +//! itself is a departure: an ephemeral environment whose daemon said +//! `registration/goodbye` is closed at once instead of after the key's +//! disconnect grace. use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, sync::{Arc, Mutex}, time::{Duration, Instant}, }; @@ -27,66 +34,110 @@ use axum::{ http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, }; -use ed25519_dalek::{Signature, Verifier as _, VerifyingKey}; +use ed25519_dalek::{Signature, VerifyingKey}; use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + bytes::SecretString, + error::{BusyReason, NotFoundReason, PermissionReason, PreconditionReason, ProtocolError}, + ids::{ControlConnectionId, DaemonId, EnvironmentId, IncarnationId}, + jsonrpc::{self, Id, Notification, Request}, + limits::{HEARTBEAT_MISSED_INTERVALS, MAX_MESSAGE_BYTES}, + manifest::{ + RegistrationChallenge, RegistrationClosed, RegistrationGoodbye, RegistrationOpenStream, + RegistrationRegister, + }, + metadata::DescriptiveText, registration::{ - DATA_PATH, DaemonControlMessage, GatewayControlMessage, REGISTRATION_NONCE_BYTES, - RegisterParams, RegistrationReceipt, RegistrationRejectionCode, decode_hex, encode_hex, - signed_registration_message, validate_registration_metadata, + Admission, ChallengeNonce, ChallengeParams, ClosedParams, ClosedReason, DaemonPublicKey, + GoodbyeParams, OpenStreamParams, OpenStreamResponse, RegisterParams, RegisterResponse, + RegistrationIdentityMode, RegistrationReceipt, StreamSpec, signed_registration_message, }, - shared::CURRENT_PROTOCOL_VERSION, + routes::CanonicalOrigin, }; use environments::{ - CreateRegisteredEnvironment, EnvironmentRecord, EnvironmentRegistrationKeyStore, - EnvironmentRegistryError, EnvironmentStatus, EnvironmentStore, ObserveRegisteredEnvironment, - RegisteredConnectionObservation, registration_key_hash, validate_daemon_public_key, + BeginCloseEnvironment, CreateRegisteredEnvironment, DaemonBuild, EnvironmentRecord, + EnvironmentRegistrationKeyStore, EnvironmentRegistryError, EnvironmentSource, EnvironmentStore, + ObserveRegisteredEnvironment, RegisteredConnectionObservation, RegisteredIdentityMode, + registration_key_hash, }; use tokio::sync::{Semaphore, mpsc, oneshot}; use uuid::Uuid; use super::http::GatewayState; -use crate::environments::gateway::{RouteKey, close_message}; +use crate::environments::{ + gateway::{RouteKey, close_message}, + registration::departure_closes, +}; /// Interval between gateway pings on a control connection; each pong -/// refreshes the environment's heartbeat stamp. +/// refreshes the environment's heartbeat stamp. Sent to the daemon as +/// `heartbeatIntervalMs`. pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30); -/// Time a daemon has to answer the challenge. +/// A control connection that sends no frame for this long is dead. +const LIVENESS_TIMEOUT: Duration = + Duration::from_secs(HEARTBEAT_INTERVAL.as_secs() * HEARTBEAT_MISSED_INTERVALS as u64); +/// Time from the challenge to the register response; the gateway closes +/// the socket when it passes and the daemon reconnects with backoff. const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(15); -/// Time a daemon has to dial the data route for one open request. -const DATA_DIAL_TIMEOUT: Duration = Duration::from_secs(15); +/// Time a daemon has to dial the stream route and answer one +/// `registration/openStream`. The worker's upgrade waits on this pairing, +/// so worker routes connect with a longer bound +/// (`environments::gateway::WORKER_ROUTE_CONNECT_TIMEOUT`) and receive the +/// gateway's typed 502/504 instead of their own connect timeout. +pub(crate) const STREAM_OPEN_TIMEOUT: Duration = Duration::from_secs(15); /// Interval at which a control connection re-reads durable state. const REAUTHORIZE_INTERVAL: Duration = Duration::from_secs(5); -/// Unauthenticated handshakes in flight at once. +/// Unauthenticated handshakes in flight at once. A permit is held from the +/// upgrade until the handshake resolves, never for an admitted session. const MAX_CONCURRENT_HANDSHAKES: usize = 256; -/// Failed authentications per key prefix (or daemon) per window. +/// Failed admissions per verified daemon identity per window. const MAX_FAILURES_PER_WINDOW: usize = 20; const FAILURE_WINDOW: Duration = Duration::from_secs(60); -/// Control frames are small JSON documents; anything larger is hostile. +/// Daemon identities the failure limiter tracks at once; the identity whose +/// last failure is oldest is forgotten first. +const MAX_LIMITED_IDENTITIES: usize = 4096; +/// Registration frames are small JSON documents (a register request with +/// full metadata is about 12 KiB), so the control route accepts far less +/// than the protocol's message limit before anyone is authenticated. const CONTROL_MAX_MESSAGE_BYTES: usize = 64 * 1024; +/// `openStream` requests one control connection may have unanswered. +const CONTROL_COMMAND_QUEUE: usize = 16; /// Live state of registered daemons on this gateway replica. In-memory by /// design: durable truth is the environment row. pub struct RegisteredConnections { controls: Mutex>, - pending: Mutex>, + pending: Mutex>, handshakes: Arc, - failures: Mutex>>, + failures: Mutex, } struct ControlHandle { - connection_id: String, + connection_id: ControlConnectionId, incarnation_id: String, commands: mpsc::Sender, + /// Fired when a newer connection of the same identity is installed. + /// A oneshot, so backpressure on `commands` can never drop it. + supersede: oneshot::Sender<()>, } enum ControlCommand { - OpenData { token: String, data_url: String }, - Supersede, + /// Ask the daemon for one data stream; `answered` receives its + /// `registration/openStream` response. + OpenStream { + token: SecretString, + answered: oneshot::Sender>, + }, } -struct PendingData { +/// A stream token the gateway issued and is waiting to pair. +struct PendingStream { sender: oneshot::Sender, expires_at: Instant, + key: RouteKey, + /// The control connection the token was sent on; pairing is refused + /// once it is no longer current. + control_connection_id: ControlConnectionId, } impl Default for RegisteredConnections { @@ -101,16 +152,21 @@ impl RegisteredConnections { controls: Mutex::new(HashMap::new()), pending: Mutex::new(HashMap::new()), handshakes: Arc::new(Semaphore::new(MAX_CONCURRENT_HANDSHAKES)), - failures: Mutex::new(HashMap::new()), + failures: Mutex::new(FailureLimiter::default()), } } /// The control connection currently serving an environment, if any. pub fn current_connection_id(&self, key: &RouteKey) -> Option { + self.current(key.universe_id, &key.environment_id) + .map(|id| id.as_str().to_owned()) + } + + fn current(&self, universe_id: Uuid, environment_id: &str) -> Option { self.controls .lock() .expect("registered connections poisoned") - .get(&(key.universe_id, key.environment_id.clone())) + .get(&(universe_id, environment_id.to_owned())) .map(|handle| handle.connection_id.clone()) } @@ -132,7 +188,7 @@ impl RegisteredConnections { &self, universe_id: Uuid, environment_id: &str, - connection_id: &str, + connection_id: &ControlConnectionId, ) -> bool { let mut controls = self .controls @@ -141,7 +197,7 @@ impl RegisteredConnections { let key = (universe_id, environment_id.to_owned()); if controls .get(&key) - .is_some_and(|handle| handle.connection_id == connection_id) + .is_some_and(|handle| &handle.connection_id == connection_id) { controls.remove(&key); true @@ -150,103 +206,141 @@ impl RegisteredConnections { } } - fn record_failure(&self, subject: &str) -> bool { + fn take_pending(&self, token: &str) -> Option { + let mut pending = self.pending.lock().expect("pending streams poisoned"); let now = Instant::now(); - let mut failures = self.failures.lock().expect("failures poisoned"); - let entry = failures.entry(subject.to_owned()).or_default(); - entry.retain(|at| now.duration_since(*at) < FAILURE_WINDOW); - entry.push(now); - entry.len() > MAX_FAILURES_PER_WINDOW + pending.retain(|_, entry| entry.expires_at > now); + pending.remove(token) } - fn over_failure_limit(&self, subject: &str) -> bool { - let now = Instant::now(); - let failures = self.failures.lock().expect("failures poisoned"); - failures.get(subject).is_some_and(|entries| { - entries + fn retry_after(&self, identity: &DaemonPublicKey) -> Option { + self.failures + .lock() + .expect("failure limiter poisoned") + .retry_after(identity, Instant::now()) + } + + fn record_failure(&self, identity: &DaemonPublicKey) { + self.failures + .lock() + .expect("failure limiter poisoned") + .record(*identity, Instant::now()); + } +} + +/// Failed admissions per verified daemon identity. Keyed only on identities +/// whose signature verified, so nobody can spend another daemon's +/// allowance with claimed values; pruned on every insert and capped. +#[derive(Default)] +struct FailureLimiter { + identities: HashMap>, +} + +impl FailureLimiter { + /// How long `identity` must wait before its next attempt, when it is + /// over the limit. + fn retry_after(&self, identity: &DaemonPublicKey, now: Instant) -> Option { + let failures = self.identities.get(identity)?; + let live: Vec<&Instant> = failures + .iter() + .filter(|at| now.duration_since(**at) < FAILURE_WINDOW) + .collect(); + if live.len() < MAX_FAILURES_PER_WINDOW { + return None; + } + let oldest = live.first()?; + Some((FAILURE_WINDOW - now.duration_since(**oldest)).max(Duration::from_millis(1))) + } + + fn record(&mut self, identity: DaemonPublicKey, now: Instant) { + self.identities.retain(|_, failures| { + while failures + .front() + .is_some_and(|at| now.duration_since(*at) >= FAILURE_WINDOW) + { + failures.pop_front(); + } + !failures.is_empty() + }); + if !self.identities.contains_key(&identity) + && self.identities.len() >= MAX_LIMITED_IDENTITIES + && let Some(stalest) = self + .identities .iter() - .filter(|at| now.duration_since(**at) < FAILURE_WINDOW) - .count() - >= MAX_FAILURES_PER_WINDOW - }) + .min_by_key(|(_, failures)| failures.back().copied()) + .map(|(identity, _)| *identity) + { + self.identities.remove(&stalest); + } + let failures = self.identities.entry(identity).or_default(); + failures.push_back(now); + while failures.len() > MAX_FAILURES_PER_WINDOW { + failures.pop_front(); + } } - fn take_pending(&self, token: &str) -> Option { - let mut pending = self.pending.lock().expect("pending data poisoned"); - let now = Instant::now(); - pending.retain(|_, entry| entry.expires_at > now); - pending.remove(token) + #[cfg(test)] + fn len(&self) -> usize { + self.identities.len() } } /// Outcome of admitting one register request against durable state. #[derive(Debug)] -pub struct Admission { +pub struct AdmittedEnvironment { pub environment: EnvironmentRecord, - pub created: bool, + pub admission: Admission, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RegistrationRejection { - pub code: RegistrationRejectionCode, - pub message: String, -} - -impl RegistrationRejection { - fn new(code: RegistrationRejectionCode, message: impl Into) -> Self { - Self { - code, - message: message.into(), - } +/// Decode `registration/register` params. The protocol version is read +/// first, so a daemon of another version learns it is stale even when the +/// rest of its request has a shape this gateway does not know; then out of +/// bounds descriptive values are `invalidRequest` and every other decode +/// failure `invalidParams`. +pub fn decode_register(params: &serde_json::Value) -> Result { + if let Some(version) = params + .get("protocolVersion") + .and_then(serde_json::Value::as_u64) + && version != u64::from(ENVD_PROTOCOL_VERSION) + { + return Err(ProtocolError::version_mismatch( + ENVD_PROTOCOL_VERSION, + u32::try_from(version).unwrap_or(u32::MAX), + )); + } + let params = RegisterParams::decode(params)?; + if params.protocol_version != ENVD_PROTOCOL_VERSION { + return Err(ProtocolError::version_mismatch( + ENVD_PROTOCOL_VERSION, + params.protocol_version, + )); } + Ok(params) } -/// Check a register request's shape and signature against the challenge -/// nonce. Pure: no store access. +/// Check a register request's signature over the challenge nonce bound to +/// one of the gateway's expected origins. Pure: no store access. pub fn verify_register_request( params: &RegisterParams, - nonce: &[u8], -) -> Result<(), RegistrationRejection> { - if params.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(RegistrationRejection::new( - RegistrationRejectionCode::UnsupportedProtocol, - format!( - "unsupported registration protocol version {}; expected {CURRENT_PROTOCOL_VERSION}", - params.protocol_version - ), - )); + nonce: &ChallengeNonce, + origins: &[CanonicalOrigin], +) -> Result<(), ProtocolError> { + let invalid = |message: &str| { + ProtocolError::permission_denied(Some(PermissionReason::InvalidSignature), message) + }; + let public_key = VerifyingKey::from_bytes(params.daemon_public_key.as_bytes()) + .map_err(|_| invalid("daemon public key is not a valid Ed25519 key"))?; + let signature = Signature::from_bytes(params.signature.as_bytes()); + if origins.iter().any(|origin| { + public_key + .verify_strict(&signed_registration_message(nonce, origin), &signature) + .is_ok() + }) { + return Ok(()); } - validate_daemon_public_key(¶ms.daemon_public_key).map_err(|error| { - RegistrationRejection::new(RegistrationRejectionCode::InvalidRequest, error.to_string()) - })?; - validate_registration_metadata(params.display_name.as_deref(), ¶ms.metadata).map_err( - |message| RegistrationRejection::new(RegistrationRejectionCode::InvalidRequest, message), - )?; - let public_key = decode_hex(¶ms.daemon_public_key) - .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok()) - .and_then(|bytes| VerifyingKey::from_bytes(&bytes).ok()) - .ok_or_else(|| { - RegistrationRejection::new( - RegistrationRejectionCode::InvalidSignature, - "daemon public key is not a valid Ed25519 key", - ) - })?; - let signature = decode_hex(¶ms.signature) - .and_then(|bytes| Signature::from_slice(&bytes).ok()) - .ok_or_else(|| { - RegistrationRejection::new( - RegistrationRejectionCode::InvalidSignature, - "signature is not a valid Ed25519 signature", - ) - })?; - public_key - .verify(&signed_registration_message(nonce), &signature) - .map_err(|_| { - RegistrationRejection::new( - RegistrationRejectionCode::InvalidSignature, - "signature does not verify against the challenge", - ) - }) + Err(invalid( + "signature does not verify against the challenge and this gateway's origin", + )) } /// Admit a verified register request inside its universe: reconnect a known @@ -256,112 +350,172 @@ pub async fn admit_registration( store: &S, params: &RegisterParams, now_ms: i64, -) -> Result +) -> Result where S: EnvironmentStore + EnvironmentRegistrationKeyStore + ?Sized, { + let daemon_public_key = params.daemon_public_key.to_hex(); let known = store - .read_environment_by_daemon_public_key(¶ms.daemon_public_key) + .read_environment_by_daemon_public_key(&daemon_public_key) .await .map_err(unavailable)?; if let Some(environment) = known { - if matches!( - environment.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) { - return Err(RegistrationRejection::new( - RegistrationRejectionCode::EnvironmentClosed, - format!( - "environment {} is closed; this daemon identity is spent", - environment.environment_id - ), - )); + if environment.status.is_closing_or_closed() { + return Err(environment_closed(&environment)); } - return Ok(Admission { + return Ok(AdmittedEnvironment { environment, - created: false, + admission: Admission::Reconnected, }); } let Some(secret) = params.registration_key.as_ref() else { - return Err(RegistrationRejection::new( - RegistrationRejectionCode::UnknownDaemon, - "unknown daemon identity and no registration key presented", - )); + return Err(unknown_daemon()); }; let key = store .resolve_registration_key(®istration_key_hash(secret.expose())) .await .map_err(unavailable)? - .ok_or_else(|| { - RegistrationRejection::new( - RegistrationRejectionCode::InvalidRegistrationKey, - "registration key is not recognized", - ) - })?; + .ok_or_else(unknown_registration_key)?; key.check_admits(now_ms).map_err(map_registry_error)?; - let mut metadata = params.metadata.clone(); + let mut metadata = params.metadata.as_map().clone(); metadata.extend( - environments::RegisteredDaemonBuild::from_registration( - ¶ms.implementation, - params.protocol_version, - ) - .metadata(), + DaemonBuild::from_implementation(¶ms.implementation, params.protocol_version) + .metadata(), ); let environment = store .create_registered_environment(CreateRegisteredEnvironment { registration_key_id: key.registration_key_id.clone(), environment_id: super::service::allocate_environment_id_public(), incarnation_id: super::service::allocate_incarnation_id_public(), - daemon_public_key: params.daemon_public_key.clone(), - display_name: params.display_name.clone(), + daemon_public_key, + display_name: params + .display_name + .as_ref() + .map(|name| name.as_str().to_owned()), metadata, created_at_ms: now_ms, }) .await .map_err(map_registry_error)?; - Ok(Admission { + Ok(AdmittedEnvironment { environment, - created: true, + admission: Admission::Created, }) } -fn map_registry_error(error: EnvironmentRegistryError) -> RegistrationRejection { +/// The receipt of an admitted environment, built only from a registered +/// record. +fn registration_receipt( + environment: &EnvironmentRecord, + control_connection_id: &ControlConnectionId, + admission: Admission, +) -> Result { + let EnvironmentSource::Registered { + daemon_id, + identity_mode, + .. + } = &environment.source + else { + return Err(ProtocolError::internal( + "the admitted environment is not a registered environment", + )); + }; + let internal = |error: environment_protocol::ids::IdError| { + ProtocolError::internal(format!("admitted environment id: {error}")) + }; + Ok(RegistrationReceipt { + environment_id: EnvironmentId::new(environment.environment_id.as_str()) + .map_err(internal)?, + incarnation_id: IncarnationId::new(environment.incarnation.incarnation_id.as_str()) + .map_err(internal)?, + daemon_id: DaemonId::new(daemon_id.as_str()).map_err(internal)?, + control_connection_id: control_connection_id.clone(), + identity_mode: match identity_mode { + RegisteredIdentityMode::Persistent => RegistrationIdentityMode::Persistent, + RegisteredIdentityMode::Ephemeral => RegistrationIdentityMode::Ephemeral, + }, + admission, + }) +} + +fn environment_closed(environment: &EnvironmentRecord) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::EnvironmentClosed, + format!( + "environment {} is closed; this daemon identity is spent", + environment.environment_id + ), + ) +} + +fn unknown_daemon() -> ProtocolError { + ProtocolError::not_found_because( + NotFoundReason::UnknownDaemon, + "unknown daemon identity and no registration key presented", + ) +} + +fn unknown_registration_key() -> ProtocolError { + ProtocolError::permission_denied( + Some(PermissionReason::InvalidRegistrationKey), + "registration key is not recognized", + ) +} + +fn map_registry_error(error: EnvironmentRegistryError) -> ProtocolError { match error { + EnvironmentRegistryError::RegistrationKeyUnavailable { + reason: "expired", .. + } => ProtocolError::expired("registration key is expired"), EnvironmentRegistryError::RegistrationKeyUnavailable { reason, .. } => { - RegistrationRejection::new( - if reason == "expired" { - RegistrationRejectionCode::RegistrationKeyExpired - } else { - RegistrationRejectionCode::RegistrationKeyRevoked - }, + ProtocolError::permission_denied( + Some(PermissionReason::RegistrationKeyRevoked), format!("registration key is {reason}"), ) } EnvironmentRegistryError::RegistrationCapacityExhausted { limit, .. } => { - RegistrationRejection::new( - RegistrationRejectionCode::CapacityExhausted, + ProtocolError::busy( + Some(BusyReason::CapacityExhausted), + None, format!("registration key has reached its active environment limit of {limit}"), ) } EnvironmentRegistryError::AlreadyExists { kind: "daemon_identity", .. - } => RegistrationRejection::new( - RegistrationRejectionCode::IdentityInUse, + } => ProtocolError::failed_precondition( + PreconditionReason::IdentityInUse, "daemon identity is already bound to an environment elsewhere", ), + // The record the request would create breaks the registry's bounds: + // admission is refused, never granted with the value dropped. EnvironmentRegistryError::InvalidInput { message } => { - RegistrationRejection::new(RegistrationRejectionCode::InvalidRequest, message) + ProtocolError::invalid_request(message) } other => unavailable(other), } } -fn unavailable(error: EnvironmentRegistryError) -> RegistrationRejection { +fn unavailable(error: EnvironmentRegistryError) -> ProtocolError { tracing::warn!(target: "temporal_runtime", %error, "registration store failure"); - RegistrationRejection::new( - RegistrationRejectionCode::Unavailable, - "registration is temporarily unavailable", + registration_unavailable() +} + +fn registration_unavailable() -> ProtocolError { + ProtocolError::unavailable("registration is temporarily unavailable") +} + +/// True for refusals that count against a verified identity's allowance: +/// credentials that did not admit it. +fn counts_as_failure(error: &ProtocolError) -> bool { + use environment_protocol::error::ErrorData; + matches!( + error.data, + ErrorData::PermissionDenied { .. } + | ErrorData::Expired {} + | ErrorData::NotFound { + reason: Some(NotFoundReason::UnknownDaemon) + } ) } @@ -370,22 +524,25 @@ pub(super) async fn connect_upgrade( State(state): State>, upgrade: WebSocketUpgrade, ) -> Response { + let Some(origin) = state.registration_origin().cloned() else { + tracing::error!( + target: "temporal_runtime", + "registration refused: the gateway has no valid public base URL to verify daemon signatures against" + ); + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + }; let Ok(permit) = state.registrations().handshakes.clone().try_acquire_owned() else { return StatusCode::SERVICE_UNAVAILABLE.into_response(); }; upgrade .max_message_size(CONTROL_MAX_MESSAGE_BYTES) - .on_upgrade(move |socket| async move { - let outcome = control_session(state, socket, permit).await; - if let Err(error) = outcome { - tracing::debug!(target: "temporal_runtime", %error, "registration control session ended with error"); - } - }) + .max_frame_size(CONTROL_MAX_MESSAGE_BYTES) + .on_upgrade(move |socket| control_connection(state, socket, permit, origin)) .into_response() } -/// `GET /environment-gateway/data`: a reverse-dialed data socket presenting -/// its one-time token as a bearer header. +/// `GET /environment-gateway/data`: a reverse-dialed stream socket +/// presenting its one-time token as a bearer header. pub(super) async fn data_upgrade( State(state): State>, headers: HeaderMap, @@ -394,16 +551,24 @@ pub(super) async fn data_upgrade( let token = headers .get(axum::http::header::AUTHORIZATION) .and_then(|value| value.to_str().ok()) - .and_then(|value| value.strip_prefix("Bearer ")) - .map(str::to_owned); + .and_then(|value| value.strip_prefix("Bearer ")); let Some(token) = token else { return StatusCode::UNAUTHORIZED.into_response(); }; - let Some(pending) = state.registrations().take_pending(&token) else { + let registrations = state.registrations(); + let Some(pending) = registrations.take_pending(token) else { return StatusCode::UNAUTHORIZED.into_response(); }; + if registrations.current(pending.key.universe_id, &pending.key.environment_id) + != Some(pending.control_connection_id.clone()) + { + // The control connection that issued the token was superseded or + // ended: whoever dials now is not the daemon it was sent to. + return StatusCode::UNAUTHORIZED.into_response(); + } upgrade - .max_message_size(64 * 1024 * 1024) + .max_message_size(MAX_MESSAGE_BYTES as usize) + .max_frame_size(MAX_MESSAGE_BYTES as usize) .on_upgrade(move |socket| async move { // The waiting route owns the socket from here; if it gave up, // dropping the socket closes the daemon's dial. @@ -412,7 +577,7 @@ pub(super) async fn data_upgrade( .into_response() } -/// Ask the daemon serving `key` to dial a data socket for one worker route +/// Ask the daemon serving `key` to dial a data stream for one worker route /// and wait for it. Returns the paired socket and the control connection it /// was issued under, so the proxy can fence on it. pub(super) async fn open_registered_route( @@ -434,31 +599,53 @@ pub(super) async fn open_registered_route( (handle.commands.clone(), handle.connection_id.clone()) }; let token = new_token(); - let (sender, receiver) = oneshot::channel(); + let (sender, socket) = oneshot::channel(); registrations .pending .lock() - .expect("pending data poisoned") + .expect("pending streams poisoned") .insert( token.clone(), - PendingData { + PendingStream { sender, - expires_at: Instant::now() + DATA_DIAL_TIMEOUT, + expires_at: Instant::now() + STREAM_OPEN_TIMEOUT, + key: key.clone(), + control_connection_id: control_connection_id.clone(), }, ); - let sent = commands - .send(ControlCommand::OpenData { - token: token.clone(), - data_url: state.registration_data_url(), - }) - .await; - if sent.is_err() { - registrations.take_pending(&token); - return Err(StatusCode::SERVICE_UNAVAILABLE); - } - match tokio::time::timeout(DATA_DIAL_TIMEOUT, receiver).await { - Ok(Ok(socket)) => Ok((socket, control_connection_id)), - _ => { + let opened = tokio::time::timeout(STREAM_OPEN_TIMEOUT, async { + let (answered, answer) = oneshot::channel(); + commands + .send(ControlCommand::OpenStream { + token: SecretString::new(token.clone()), + answered, + }) + .await + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + match answer.await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::warn!( + target: "temporal_runtime", + environment = %key.environment_id, + %error, + "registered daemon could not open a stream" + ); + return Err(StatusCode::BAD_GATEWAY); + } + // The control connection ended before the daemon answered. + Err(_) => return Err(StatusCode::SERVICE_UNAVAILABLE), + } + socket.await.map_err(|_| StatusCode::BAD_GATEWAY) + }) + .await; + match opened { + Ok(Ok(socket)) => Ok((socket, control_connection_id.as_str().to_owned())), + Ok(Err(status)) => { + registrations.take_pending(&token); + Err(status) + } + Err(_) => { registrations.take_pending(&token); Err(StatusCode::GATEWAY_TIMEOUT) } @@ -517,147 +704,319 @@ async fn authorize_registered_route( EnvironmentStore::read_environment(api.store().as_ref(), &environment_id).await?; if !environment.is_registered() || environment.incarnation.incarnation_id.as_str() != key.incarnation_id - || matches!( - environment.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) + || environment.status.is_closing_or_closed() { anyhow::bail!("registered route no longer valid"); } Ok(()) } -async fn control_session( +/// What the gateway does with one frame before admission. +#[derive(Debug, PartialEq)] +enum PreAdmission { + /// A `registration/register` request. + Register(Request), + /// Answer and keep waiting: a framing error. + Answer(jsonrpc::Response), + /// Answer, then close: a request other than register. + AnswerAndClose(jsonrpc::Response), + /// Close without an answer: a notification before admission. + Close, + /// Nothing to do: a response to an id the gateway never sent, a + /// malformed notification, a ping or pong. + Ignore, +} + +fn classify_pre_admission(message: &Message) -> PreAdmission { + match message { + Message::Text(text) => match jsonrpc::parse(text.as_str()) { + Err(rejection) => match rejection.into_response() { + Some(response) => PreAdmission::Answer(response), + None => PreAdmission::Ignore, + }, + Ok(jsonrpc::Message::Request(request)) + if request.method == RegistrationRegister::NAME => + { + PreAdmission::Register(request) + } + Ok(jsonrpc::Message::Request(request)) => PreAdmission::AnswerAndClose( + jsonrpc::Response::failure(Some(request.id), ProtocolError::not_initialized()), + ), + Ok(jsonrpc::Message::Notification(_)) => PreAdmission::Close, + Ok(jsonrpc::Message::Response(_)) => PreAdmission::Ignore, + }, + Message::Binary(_) => PreAdmission::Answer(binary_frame_refusal()), + Message::Close(_) => PreAdmission::Close, + Message::Ping(_) | Message::Pong(_) => PreAdmission::Ignore, + } +} + +/// What the gateway does with one text frame after admission. +#[derive(Debug, PartialEq)] +enum Admitted { + /// Send this answer; the connection stays open. + Answer(jsonrpc::Response), + /// The daemon answered the gateway request with this id. + StreamAnswer(Id, Result<(), ProtocolError>), + /// `registration/goodbye`: the daemon is stopping deliberately. + Goodbye, + /// Ignore and log. + Ignore(String), +} + +fn classify_admitted(text: &str) -> Admitted { + match jsonrpc::parse(text) { + Err(rejection) => match rejection.into_response() { + Some(response) => Admitted::Answer(response), + None => Admitted::Ignore("malformed notification".to_owned()), + }, + Ok(jsonrpc::Message::Request(request)) => { + let error = if request.method == RegistrationRegister::NAME { + ProtocolError::invalid_request("the daemon is already admitted") + } else { + ProtocolError::method_not_found(&request.method) + }; + Admitted::Answer(jsonrpc::Response::failure(Some(request.id), error)) + } + Ok(jsonrpc::Message::Notification(notification)) + if notification.method == RegistrationGoodbye::NAME => + { + match notification.params::() { + Ok(_) => Admitted::Goodbye, + Err(error) => Admitted::Ignore(format!("undecodable goodbye: {error}")), + } + } + Ok(jsonrpc::Message::Notification(notification)) => { + Admitted::Ignore(format!("unknown notification {}", notification.method)) + } + Ok(jsonrpc::Message::Response(response)) => match response.id.clone() { + Some(id) => { + let answer = response + .decode::() + .map(|_| ()) + .map_err(|error| match error { + jsonrpc::ResponseError::Failed(error) => error, + jsonrpc::ResponseError::Undecodable(message) => ProtocolError::internal( + format!("the openStream response breaks the contract: {message}"), + ), + }); + Admitted::StreamAnswer(id, answer) + } + None => Admitted::Ignore(format!( + "error response without an id: {}", + response + .result + .err() + .map(|error| error.to_string()) + .unwrap_or_default() + )), + }, + } +} + +fn binary_frame_refusal() -> jsonrpc::Response { + jsonrpc::Response::failure( + None, + ProtocolError::invalid_request("binary frames are not part of the registration channel"), + ) +} + +/// Everything the handshake established. +struct Handshake { + universe_id: Uuid, + store: Arc, + register: RegisterParams, + register_id: Id, + admitted: AdmittedEnvironment, +} + +/// Why a handshake produced no admitted session. +enum Refused { + /// The socket is gone or the daemon gave up; nothing to answer. + Silent, + /// Answer the register request with this error, then close. + Answer(Id, ProtocolError), +} + +async fn control_connection( state: Arc, mut socket: WebSocket, - _permit: tokio::sync::OwnedSemaphorePermit, -) -> anyhow::Result<()> { - let nonce = { - use rand::TryRng as _; - let mut bytes = [0u8; REGISTRATION_NONCE_BYTES]; - rand::rngs::SysRng - .try_fill_bytes(&mut bytes) - .expect("OS random source unavailable"); - bytes - }; - send_control( - &mut socket, - &GatewayControlMessage::Challenge { - protocol_version: CURRENT_PROTOCOL_VERSION, - nonce: encode_hex(&nonce), + permit: tokio::sync::OwnedSemaphorePermit, + origin: CanonicalOrigin, +) { + let handshake = + tokio::time::timeout(HANDSHAKE_TIMEOUT, handshake(&state, &mut socket, &origin)).await; + // The permit bounds unauthenticated handshakes only: it is released as + // soon as the handshake resolved, whatever its outcome. + drop(permit); + match handshake { + Ok(Ok(handshake)) => { + if let Err(error) = admitted_session(&state, socket, handshake).await { + tracing::debug!(target: "temporal_runtime", %error, "registration control session ended with error"); + } + } + Ok(Err(Refused::Answer(id, error))) => { + tracing::info!(target: "temporal_runtime", kind = error.kind().as_str(), %error, "registration refused"); + let _ = send_text( + &mut socket, + jsonrpc::Response::failure(Some(id), error).to_text(), + ) + .await; + let _ = socket.send(Message::Close(None)).await; + } + Ok(Err(Refused::Silent)) => { + let _ = socket.send(Message::Close(None)).await; + } + Err(_) => { + // A daemon that is paused or starved retries with backoff. + tracing::info!(target: "temporal_runtime", "registration handshake timed out"); + let _ = socket.send(Message::Close(None)).await; + } + } +} + +/// Challenge, read the register request, verify and admit it. +async fn handshake( + state: &Arc, + socket: &mut WebSocket, + origin: &CanonicalOrigin, +) -> Result { + let nonce = new_nonce(); + let challenge = Notification::new( + RegistrationChallenge::NAME, + &ChallengeParams { + protocol_version: ENVD_PROTOCOL_VERSION, + nonce, }, ) - .await?; - let register = - match tokio::time::timeout(HANDSHAKE_TIMEOUT, next_daemon_message(&mut socket)).await { - Ok(Ok(Some(DaemonControlMessage::Register(params)))) => params, - Ok(Ok(None)) => return Ok(()), - Ok(Err(error)) => { - reject( - &mut socket, - RegistrationRejectionCode::InvalidRequest, - error.to_string(), - ) - .await; - return Ok(()); + .map_err(|_| Refused::Silent)?; + send_text(socket, challenge.to_text()) + .await + .map_err(|_| Refused::Silent)?; + let request = loop { + let Some(Ok(message)) = socket.recv().await else { + return Err(Refused::Silent); + }; + match classify_pre_admission(&message) { + PreAdmission::Register(request) => break request, + PreAdmission::Answer(response) => { + send_text(socket, response.to_text()) + .await + .map_err(|_| Refused::Silent)?; } - Err(_) => { - reject( - &mut socket, - RegistrationRejectionCode::InvalidRequest, - "handshake timed out", - ) - .await; - return Ok(()); + PreAdmission::AnswerAndClose(response) => { + let _ = send_text(socket, response.to_text()).await; + return Err(Refused::Silent); } - }; + PreAdmission::Close => return Err(Refused::Silent), + PreAdmission::Ignore => {} + } + }; + let id = request.id.clone(); + let refuse = |error: ProtocolError| Refused::Answer(id.clone(), error); + let register = decode_register(&request.params).map_err(refuse)?; + verify_register_request(®ister, &nonce, std::slice::from_ref(origin)).map_err(refuse)?; + + // From here the identity is verified: rate limits key on it alone. let registrations = state.registrations(); - let subject = register - .registration_key - .as_ref() - .map(|key| format!("key:{}", key.expose().chars().take(12).collect::())) - .unwrap_or_else(|| { - format!( - "daemon:{}", - ®ister.daemon_public_key[..16.min(register.daemon_public_key.len())] - ) - }); - if registrations.over_failure_limit(&subject) { - reject( - &mut socket, - RegistrationRejectionCode::RateLimited, - "too many failed registrations; retry later", - ) - .await; - return Ok(()); - } - if let Err(rejection) = verify_register_request(®ister, &nonce) { - registrations.record_failure(&subject); - reject(&mut socket, rejection.code, rejection.message).await; - return Ok(()); + let identity = register.daemon_public_key; + if let Some(retry_after) = registrations.retry_after(&identity) { + return Err(refuse(ProtocolError::busy( + Some(BusyReason::RateLimited), + Some(u64::try_from(retry_after.as_millis()).unwrap_or(u64::MAX)), + "too many failed registrations for this daemon identity; retry later", + ))); } + let refuse_counted = |error: ProtocolError| { + if counts_as_failure(&error) { + registrations.record_failure(&identity); + } + refuse(error) + }; let universe_id = match state.registration_universe(®ister).await { Ok(Some(universe_id)) => universe_id, Ok(None) => { - registrations.record_failure(&subject); - let (code, message) = if register.registration_key.is_some() { - ( - RegistrationRejectionCode::InvalidRegistrationKey, - "registration key is not recognized", - ) + return Err(refuse_counted(if register.registration_key.is_some() { + unknown_registration_key() } else { - ( - RegistrationRejectionCode::UnknownDaemon, - "unknown daemon identity and no registration key presented", - ) - }; - reject(&mut socket, code, message).await; - return Ok(()); + unknown_daemon() + })); } Err(error) => { tracing::warn!(target: "temporal_runtime", error = %error.message, "registration universe lookup failed"); - reject( - &mut socket, - RegistrationRejectionCode::Unavailable, - "registration is temporarily unavailable", - ) - .await; - return Ok(()); + return Err(refuse(registration_unavailable())); } }; - let api = match state.api_for_daemon(universe_id).await { - Ok(api) => api, + let api = state.api_for_daemon(universe_id).await.map_err(|error| { + tracing::warn!(target: "temporal_runtime", error = %error.message, "registration universe unavailable"); + refuse(registration_unavailable()) + })?; + let store = api.store().clone(); + let admitted = admit_registration(store.as_ref(), ®ister, now_ms()) + .await + .map_err(refuse_counted)?; + + // Record the connect, with the build that serves the environment now, + // before the daemon learns it is admitted: a row that cannot take the + // observation must not report a live daemon. + EnvironmentStore::observe_registered_environment( + store.as_ref(), + ObserveRegisteredEnvironment { + environment_id: admitted.environment.environment_id.clone(), + observation: RegisteredConnectionObservation::Connected, + observed_at_ms: now_ms(), + metadata: DaemonBuild::from_implementation( + ®ister.implementation, + register.protocol_version, + ) + .metadata(), + }, + ) + .await + .map_err(|error| refuse(map_registry_error(error)))?; + Ok(Handshake { + universe_id, + store, + register, + register_id: id, + admitted, + }) +} + +/// The admitted control connection: answer the register request, then +/// ping, pair streams and watch the environment until the session ends. +/// Cleanup runs on every exit path. +async fn admitted_session( + state: &Arc, + mut socket: WebSocket, + handshake: Handshake, +) -> anyhow::Result<()> { + let Handshake { + universe_id, + store, + register, + register_id, + admitted, + } = handshake; + let environment = admitted.environment; + let environment_id = environment.environment_id.to_string(); + let connection_id = ControlConnectionId::new(format!("connection_{}", Uuid::new_v4().simple())) + .expect("generated control connection ids are valid"); + let receipt = match registration_receipt(&environment, &connection_id, admitted.admission) { + Ok(receipt) => receipt, Err(error) => { - tracing::warn!(target: "temporal_runtime", error = %error.message, "registration universe unavailable"); - reject( + let _ = send_text( &mut socket, - RegistrationRejectionCode::Unavailable, - "registration is temporarily unavailable", + jsonrpc::Response::failure(Some(register_id), error).to_text(), ) .await; + let _ = socket.send(Message::Close(None)).await; return Ok(()); } }; - let store = api.store().clone(); - let admission = match admit_registration(store.as_ref(), ®ister, now_ms()).await { - Ok(admission) => admission, - Err(rejection) => { - if matches!( - rejection.code, - RegistrationRejectionCode::InvalidRegistrationKey - | RegistrationRejectionCode::RegistrationKeyRevoked - | RegistrationRejectionCode::RegistrationKeyExpired - ) { - registrations.record_failure(&subject); - } - reject(&mut socket, rejection.code, rejection.message).await; - return Ok(()); - } - }; - let environment = admission.environment; - let environment_id = environment.environment_id.to_string(); - let connection_id = format!("connection_{}", Uuid::new_v4().simple()); - let (commands, mut command_rx) = mpsc::channel::(16); + let registrations = state.registrations(); + let (commands, command_rx) = mpsc::channel(CONTROL_COMMAND_QUEUE); + let (supersede, superseded) = oneshot::channel(); if let Some(previous) = registrations.install( universe_id, environment_id.clone(), @@ -665,106 +1024,293 @@ async fn control_session( connection_id: connection_id.clone(), incarnation_id: environment.incarnation.incarnation_id.to_string(), commands, + supersede, }, ) { - let _ = previous.commands.try_send(ControlCommand::Supersede); + let _ = previous.supersede.send(()); } - // Every admission re-records which build serves the environment, so a - // replaced daemon shows its new version, commit, and protocol on the row - // instead of the one it first registered with. - let daemon_build = environments::RegisteredDaemonBuild::from_registration( - ®ister.implementation, - register.protocol_version, + tracing::info!( + target: "temporal_runtime", + %universe_id, + environment = %environment_id, + daemon = %receipt.daemon_id, + daemon_public_key = %register.daemon_public_key, + connection = %connection_id, + admission = ?receipt.admission, + "registered environment control connection admitted" ); - let observe = |observation: RegisteredConnectionObservation| { - let store = store.clone(); - let environment_id = environment.environment_id.clone(); - let metadata = match observation { - RegisteredConnectionObservation::Connected => daemon_build.metadata(), - _ => std::collections::BTreeMap::new(), - }; - async move { - let result = EnvironmentStore::observe_registered_environment( - store.as_ref(), - ObserveRegisteredEnvironment { - environment_id, - observation, - observed_at_ms: now_ms(), - metadata, - }, - ) - .await; - if let Err(error) = result { - tracing::warn!(target: "temporal_runtime", %error, "registered observation failed"); - } - } - }; - observe(RegisteredConnectionObservation::Connected).await; - let identity_mode = environment - .source - .identity_mode() - .map(|mode| mode.as_str().to_owned()) - .unwrap_or_default(); - let daemon_id = match &environment.source { - environments::EnvironmentSource::Registered { daemon_id, .. } => daemon_id.to_string(), - _ => String::new(), + let session = ControlSession { + state, + store: &store, + universe_id, + environment: &environment, + connection_id: &connection_id, }; + let (end, heartbeat_write) = session + .serve(&mut socket, register_id, receipt, command_rx, superseded) + .await; + let was_current = registrations.remove_if_current(universe_id, &environment_id, &connection_id); + // A heartbeat write that already passed its "still current" check + // must land before the disconnect, or a late `Connected` would report + // a live daemon on a row nobody holds. Later writes see the removal. + if let Some(write) = heartbeat_write { + settle_heartbeat_write(write).await; + } tracing::info!( target: "temporal_runtime", %universe_id, environment = %environment_id, - daemon = %daemon_id, connection = %connection_id, - created = admission.created, - "registered environment control connection admitted" + reason = end.reason(), + current = was_current, + "registered environment control connection ended" ); - send_control( - &mut socket, - &GatewayControlMessage::Accepted { - receipt: RegistrationReceipt { - environment_id: environment_id.clone(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - daemon_id, - connection_id: connection_id.clone(), - identity_mode, - }, - heartbeat_interval_ms: HEARTBEAT_INTERVAL.as_millis() as u64, + if was_current { + observe( + &store, + &environment.environment_id, + RegisteredConnectionObservation::Disconnected, + ) + .await; + if end == SessionEnd::Goodbye && departure_closes(&environment) { + close_departed(&store, &environment).await; + } + } + let _ = socket.send(Message::Close(None)).await; + Ok(()) +} + +/// Longest wait for an in-flight heartbeat write when a session ends; a +/// write still running after it is aborted. +const HEARTBEAT_SETTLE_TIMEOUT: Duration = Duration::from_secs(5); + +async fn settle_heartbeat_write(mut write: tokio::task::JoinHandle<()>) { + if tokio::time::timeout(HEARTBEAT_SETTLE_TIMEOUT, &mut write) + .await + .is_err() + { + write.abort(); + tracing::warn!( + target: "temporal_runtime", + "registered heartbeat write did not finish before the disconnect; aborted" + ); + } +} + +async fn close_departed(store: &Arc, environment: &EnvironmentRecord) { + let result = EnvironmentStore::begin_close_environment( + store.as_ref(), + BeginCloseEnvironment { + environment_id: environment.environment_id.clone(), + updated_at_ms: now_ms(), }, ) - .await?; + .await; + match result { + Ok(_) => tracing::info!( + target: "temporal_runtime", + environment = %environment.environment_id, + "ephemeral registered daemon departed; closing its environment" + ), + // The reconciler's grace sweep closes it later. + Err(error) => tracing::warn!( + target: "temporal_runtime", + environment = %environment.environment_id, + %error, + "closing a departed ephemeral environment failed" + ), + } +} - let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); - heartbeat.tick().await; - let mut reauthorize = tokio::time::interval(REAUTHORIZE_INTERVAL); - reauthorize.tick().await; - let end_reason: &str = loop { - tokio::select! { - command = command_rx.recv() => match command { - Some(ControlCommand::OpenData { token, data_url }) => { - let message = GatewayControlMessage::OpenData { - token: token.into(), - data_url, +/// Why an admitted session ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SessionEnd { + Superseded, + EnvironmentClosed, + Goodbye, + HeartbeatTimeout, + DaemonClosed, + SendFailed, +} + +impl SessionEnd { + fn reason(self) -> &'static str { + match self { + Self::Superseded => "superseded", + Self::EnvironmentClosed => "environment closed", + Self::Goodbye => "goodbye", + Self::HeartbeatTimeout => "heartbeat timeout", + Self::DaemonClosed => "daemon closed", + Self::SendFailed => "send failed", + } + } +} + +struct ControlSession<'a> { + state: &'a Arc, + store: &'a Arc, + universe_id: Uuid, + environment: &'a EnvironmentRecord, + connection_id: &'a ControlConnectionId, +} + +impl ControlSession<'_> { + /// Serve the admitted session until it ends. Also returns the heartbeat + /// write still in flight, which the caller settles before recording + /// the disconnect. + async fn serve( + &self, + socket: &mut WebSocket, + register_id: Id, + receipt: RegistrationReceipt, + commands: mpsc::Receiver, + superseded: oneshot::Receiver<()>, + ) -> (SessionEnd, Option>) { + let response = jsonrpc::Response::success( + register_id, + &RegisterResponse { + receipt, + heartbeat_interval_ms: u64::try_from(HEARTBEAT_INTERVAL.as_millis()) + .unwrap_or(u64::MAX), + }, + ); + if send_text(socket, response.to_text()).await.is_err() { + return (SessionEnd::SendFailed, None); + } + let mut heartbeat_write = None; + let end = self + .run(socket, commands, superseded, &mut heartbeat_write) + .await; + (end, heartbeat_write) + } + + async fn run( + &self, + socket: &mut WebSocket, + mut commands: mpsc::Receiver, + mut superseded: oneshot::Receiver<()>, + heartbeat_write: &mut Option>, + ) -> SessionEnd { + let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + heartbeat.tick().await; + let (mut environment_closed, _watcher) = self.watch_environment_closed(); + let mut watching = true; + let mut deadline = tokio::time::Instant::now() + LIVENESS_TIMEOUT; + let mut next_request = 0_u64; + let mut streams: HashMap>> = HashMap::new(); + let mut commands_closed = false; + loop { + // Biased: a frame that already arrived is read before the + // liveness deadline is checked, so a loop that fell behind + // never declares an answering daemon dead. + tokio::select! { + biased; + _ = &mut superseded => { + self.closed(socket, ClosedReason::Superseded, "another process with this daemon identity connected").await; + return SessionEnd::Superseded; + } + closed = &mut environment_closed, if watching => { + if closed.is_ok() { + self.closed(socket, ClosedReason::EnvironmentClosed, "the environment was closed").await; + return SessionEnd::EnvironmentClosed; + } + watching = false; + } + command = commands.recv(), if !commands_closed => { + // The channel closes only when a newer connection + // replaced this session's handle; supersede follows. + let Some(ControlCommand::OpenStream { token, answered }) = command else { + commands_closed = true; + continue; + }; + next_request += 1; + let id = Id::String(format!("g{next_request}")); + let request = Request::new( + id.clone(), + RegistrationOpenStream::NAME, + &OpenStreamParams { token, stream: StreamSpec::DataProtocol {} }, + ); + let Ok(request) = request else { + let _ = answered.send(Err(ProtocolError::internal("encode openStream"))); + continue; }; - if send_control(&mut socket, &message).await.is_err() { - break "send failed"; + if send_text(socket, request.to_text()).await.is_err() { + return SessionEnd::SendFailed; + } + streams.retain(|_, waiting| !waiting.is_closed()); + streams.insert(id, answered); + } + _ = heartbeat.tick() => { + if socket.send(Message::Ping(Default::default())).await.is_err() { + return SessionEnd::SendFailed; } } - Some(ControlCommand::Supersede) => { - let _ = socket.send(close_message("superseded by a newer connection of the same daemon")).await; - break "superseded"; + message = socket.recv() => { + let Some(Ok(message)) = message else { + return SessionEnd::DaemonClosed; + }; + deadline = tokio::time::Instant::now() + LIVENESS_TIMEOUT; + let answer = match message { + Message::Pong(_) => { + if heartbeat_write.as_ref().is_none_or(|write| write.is_finished()) { + *heartbeat_write = Some(self.record_heartbeat()); + } + None + } + Message::Ping(_) => None, + Message::Close(_) => return SessionEnd::DaemonClosed, + Message::Binary(_) => Some(binary_frame_refusal()), + Message::Text(text) => match classify_admitted(text.as_str()) { + Admitted::Answer(response) => Some(response), + Admitted::StreamAnswer(id, answer) => { + match streams.remove(&id) { + Some(waiting) => { let _ = waiting.send(answer); } + None => tracing::debug!(target: "temporal_runtime", %id, "ignoring a response to an unknown request id"), + } + None + } + Admitted::Goodbye => { + tracing::info!(target: "temporal_runtime", environment = %self.environment.environment_id, "registered daemon said goodbye"); + return SessionEnd::Goodbye; + } + Admitted::Ignore(reason) => { + tracing::debug!(target: "temporal_runtime", %reason, "ignoring a registration frame"); + None + } + }, + }; + if let Some(answer) = answer + && send_text(socket, answer.to_text()).await.is_err() + { + return SessionEnd::SendFailed; + } } - None => break "command channel closed", - }, - _ = heartbeat.tick() => { - if socket.send(Message::Ping(Default::default())).await.is_err() { - break "ping failed"; + _ = tokio::time::sleep_until(deadline) => { + return SessionEnd::HeartbeatTimeout; } } - _ = reauthorize.tick() => { - match EnvironmentStore::read_environment(store.as_ref(), &environment.environment_id).await { - Ok(current) if matches!(current.status, EnvironmentStatus::Closing | EnvironmentStatus::Closed) => { - let _ = socket.send(close_message("environment closed")).await; - break "environment closed"; + } + } + + /// Re-read the environment every [`REAUTHORIZE_INTERVAL`] off the + /// session loop, so a slow store never delays pings, pongs or stream + /// requests. The receiver fires once the environment is closing or + /// closed; the watcher stops when the returned guard drops. + fn watch_environment_closed(&self) -> (oneshot::Receiver<()>, AbortOnDrop) { + let store = self.store.clone(); + let environment_id = self.environment.environment_id.clone(); + let (closed, receiver) = oneshot::channel(); + let task = tokio::spawn(async move { + let mut interval = tokio::time::interval(REAUTHORIZE_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + interval.tick().await; + loop { + interval.tick().await; + match EnvironmentStore::read_environment(store.as_ref(), &environment_id).await { + Ok(current) if current.status.is_closing_or_closed() => { + let _ = closed.send(()); + return; } Ok(_) => {} Err(error) => { @@ -772,89 +1318,102 @@ async fn control_session( } } } - message = socket.recv() => match message { - Some(Ok(Message::Pong(_))) => observe(RegisteredConnectionObservation::Heartbeat).await, - Some(Ok(Message::Ping(bytes))) => { - if socket.send(Message::Pong(bytes)).await.is_err() { - break "pong failed"; - } - } - Some(Ok(Message::Close(_))) | None => break "daemon closed", - Some(Ok(_)) => {} - Some(Err(_)) => break "socket error", + }); + (receiver, AbortOnDrop(task)) + } + + /// Tell the daemon its session ends for good, then close. + async fn closed(&self, socket: &mut WebSocket, reason: ClosedReason, message: &str) { + if let Ok(notification) = Notification::new( + RegistrationClosed::NAME, + &ClosedParams { + reason, + message: DescriptiveText::lossy(message), }, + ) { + let _ = send_text(socket, notification.to_text()).await; } - }; - let was_current = registrations.remove_if_current(universe_id, &environment_id, &connection_id); - tracing::info!( - target: "temporal_runtime", - %universe_id, - environment = %environment_id, - connection = %connection_id, - reason = end_reason, - current = was_current, - "registered environment control connection ended" - ); - if was_current { - observe(RegisteredConnectionObservation::Disconnected).await; + let _ = socket + .send(close_message(match reason { + ClosedReason::Superseded => "superseded by a newer connection of the same daemon", + ClosedReason::EnvironmentClosed => "environment closed", + })) + .await; + } + + /// A pong from the current control connection means the daemon is + /// connected: refresh the stamp and restore `Ready` if a lost write or + /// the stale sweep marked the row offline. Written off the session + /// loop so a slow store never delays pings or stream requests. + fn record_heartbeat(&self) -> tokio::task::JoinHandle<()> { + let registrations = self.state.registrations().clone(); + let store = self.store.clone(); + let universe_id = self.universe_id; + let environment_id = self.environment.environment_id.clone(); + let connection_id = self.connection_id.clone(); + tokio::spawn(async move { + if registrations.current(universe_id, environment_id.as_str()) != Some(connection_id) { + return; + } + observe( + &store, + &environment_id, + RegisteredConnectionObservation::Connected, + ) + .await; + }) } - Ok(()) } -async fn send_control( - socket: &mut WebSocket, - message: &GatewayControlMessage, -) -> anyhow::Result<()> { - socket - .send(Message::Text(serde_json::to_string(message)?.into())) - .await - .map_err(|error| anyhow::anyhow!("send control message: {error}")) +/// Aborts a background task when the session that owns it ends. +struct AbortOnDrop(tokio::task::JoinHandle<()>); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } } -async fn reject( - socket: &mut WebSocket, - code: RegistrationRejectionCode, - message: impl Into, +async fn observe( + store: &Arc, + environment_id: &environments::EnvironmentId, + observation: RegisteredConnectionObservation, ) { - let message = message.into(); - tracing::info!(target: "temporal_runtime", ?code, %message, "registration rejected"); - let _ = send_control(socket, &GatewayControlMessage::Rejected { code, message }).await; - let _ = socket.send(Message::Close(None)).await; + let result = EnvironmentStore::observe_registered_environment( + store.as_ref(), + ObserveRegisteredEnvironment { + environment_id: environment_id.clone(), + observation, + observed_at_ms: now_ms(), + metadata: Default::default(), + }, + ) + .await; + if let Err(error) = result { + tracing::warn!(target: "temporal_runtime", %error, ?observation, "registered observation failed"); + } } -async fn next_daemon_message( - socket: &mut WebSocket, -) -> anyhow::Result> { - loop { - match socket.recv().await { - Some(Ok(Message::Text(text))) => { - return serde_json::from_str(&text) - .map(Some) - .map_err(|error| anyhow::anyhow!("decode register message: {error}")); - } - Some(Ok(Message::Binary(bytes))) => { - return serde_json::from_slice(&bytes) - .map(Some) - .map_err(|error| anyhow::anyhow!("decode register message: {error}")); - } - Some(Ok(Message::Close(_))) | None => return Ok(None), - Some(Ok(Message::Ping(bytes))) => { - socket.send(Message::Pong(bytes)).await?; - } - Some(Ok(_)) => {} - Some(Err(error)) => return Err(error.into()), - } - } +async fn send_text(socket: &mut WebSocket, text: String) -> Result<(), axum::Error> { + socket.send(Message::Text(text.into())).await +} + +fn new_nonce() -> ChallengeNonce { + ChallengeNonce::from_bytes(random_bytes()) } fn new_token() -> String { use base64::Engine as _; + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(random_bytes::<32>()) +} + +fn random_bytes() -> [u8; N] { use rand::TryRng as _; - let mut bytes = [0u8; 32]; + let mut bytes = [0u8; N]; rand::rngs::SysRng .try_fill_bytes(&mut bytes) .expect("OS random source unavailable"); - base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes) + bytes } fn now_ms() -> i64 { @@ -864,25 +1423,22 @@ fn now_ms() -> i64 { .unwrap_or(0) } -/// The public data route daemons dial for reverse data connections. -pub fn data_url(public_base_url: &str) -> String { - format!( - "{}{DATA_PATH}", - crate::environments::gateway::websocket_base(public_base_url.trim_end_matches('/')) - ) -} - #[cfg(test)] mod tests { - use std::collections::BTreeMap; - use ed25519_dalek::{Signer as _, SigningKey}; - use environment_protocol::shared::{ImplementationInfo, SecretString}; + use environment_protocol::{ + error::{ErrorData, ProtocolErrorKind}, + handshake::ImplementationInfo, + metadata::{DisplayName, Metadata}, + registration::Ed25519Signature, + routes::canonical_origin, + }; use environments::{ BeginCloseEnvironment, CreateEnvironmentRegistrationKey, EnvironmentRegistrationKeyId, - FinishCloseEnvironment, InMemoryEnvironmentRegistryStore, RegisteredIdentityMode, + EnvironmentStatus, FinishCloseEnvironment, InMemoryEnvironmentRegistryStore, RegistrationKeyPolicy, RevokeEnvironmentRegistrationKey, mint_registration_key, }; + use serde_json::json; use super::*; @@ -890,23 +1446,38 @@ mod tests { SigningKey::from_bytes(&[seed; 32]) } - fn register(key: &SigningKey, nonce: &[u8], secret: Option<&str>) -> RegisterParams { + fn origin() -> CanonicalOrigin { + canonical_origin("wss://gateway.example/environment-gateway/connect").unwrap() + } + + fn register_signed( + key: &SigningKey, + nonce: &ChallengeNonce, + origin: &CanonicalOrigin, + secret: Option<&str>, + ) -> RegisterParams { RegisterParams { - protocol_version: CURRENT_PROTOCOL_VERSION, + protocol_version: ENVD_PROTOCOL_VERSION, registration_key: secret.map(SecretString::new), - daemon_public_key: encode_hex(key.verifying_key().as_bytes()), - signature: encode_hex(&key.sign(&signed_registration_message(nonce)).to_bytes()), - display_name: Some("worker".to_owned()), - metadata: BTreeMap::new(), - implementation: ImplementationInfo { - name: "lightspeed-envd".to_owned(), - version: Some("0.1.0".to_owned()), - git_sha: Some("1111111111111111111111111111111111111111".to_owned()), - target: Some("x86_64-unknown-linux-musl".to_owned()), - }, + daemon_public_key: DaemonPublicKey::from_bytes(key.verifying_key().to_bytes()), + signature: Ed25519Signature::from_bytes( + key.sign(&signed_registration_message(nonce, origin)) + .to_bytes(), + ), + display_name: Some(DisplayName::new("worker").unwrap()), + metadata: Metadata::default(), + implementation: ImplementationInfo::new("lightspeed-envd") + .and_then(|info| info.with_version("0.1.0")) + .and_then(|info| info.with_git_sha("1111111111111111111111111111111111111111")) + .and_then(|info| info.with_target("x86_64-unknown-linux-musl")) + .unwrap(), } } + fn register(key: &SigningKey, secret: Option<&str>) -> RegisterParams { + register_signed(key, &ChallengeNonce::from_bytes([1; 32]), &origin(), secret) + } + async fn store_with_key( id: &str, policy: RegistrationKeyPolicy, @@ -935,85 +1506,205 @@ mod tests { } #[test] - fn register_requests_are_verified_against_the_nonce_and_domain() { + fn signatures_bind_the_nonce_and_the_gateway_origin() { let key = daemon(1); - let nonce = [5u8; 32]; - assert!(verify_register_request(®ister(&key, &nonce, None), &nonce).is_ok()); - let replayed = verify_register_request(®ister(&key, &nonce, None), &[6u8; 32]); + let nonce = ChallengeNonce::from_bytes([5; 32]); + let params = register_signed(&key, &nonce, &origin(), None); + assert!(verify_register_request(¶ms, &nonce, &[origin()]).is_ok()); + + let replayed = + verify_register_request(¶ms, &ChallengeNonce::from_bytes([6; 32]), &[origin()]) + .unwrap_err(); + assert!(matches!( + replayed.data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::InvalidSignature) + } + )); + + // A nonce relayed from another gateway was signed for that origin. + let elsewhere = canonical_origin("wss://other.example").unwrap(); + let relayed = register_signed(&key, &nonce, &elsewhere, None); assert_eq!( - replayed.unwrap_err().code, - RegistrationRejectionCode::InvalidSignature + verify_register_request(&relayed, &nonce, &[origin()]) + .unwrap_err() + .kind(), + ProtocolErrorKind::PermissionDenied ); - let mut wrong_key = register(&key, &nonce, None); - wrong_key.daemon_public_key = encode_hex(daemon(2).verifying_key().as_bytes()); + assert!(verify_register_request(&relayed, &nonce, &[origin(), elsewhere]).is_ok()); + + let mut wrong_key = params.clone(); + wrong_key.daemon_public_key = + DaemonPublicKey::from_bytes(daemon(2).verifying_key().to_bytes()); assert_eq!( - verify_register_request(&wrong_key, &nonce) + verify_register_request(&wrong_key, &nonce, &[origin()]) .unwrap_err() - .code, - RegistrationRejectionCode::InvalidSignature + .kind(), + ProtocolErrorKind::PermissionDenied ); - let mut old = register(&key, &nonce, None); - old.protocol_version = 0; + } + + fn register_value() -> serde_json::Value { + serde_json::to_value(register(&daemon(1), None)).unwrap() + } + + #[test] + fn register_decoding_checks_the_version_first_and_refuses_out_of_bounds_facts() { + assert!(decode_register(®ister_value()).is_ok()); + + // A future daemon with members this gateway does not know still + // learns the version it must speak. + let mut future = register_value(); + future["protocolVersion"] = json!(ENVD_PROTOCOL_VERSION + 1); + future["newMember"] = json!(true); + let mismatch = decode_register(&future).unwrap_err(); assert_eq!( - verify_register_request(&old, &nonce).unwrap_err().code, - RegistrationRejectionCode::UnsupportedProtocol + mismatch.data, + ErrorData::VersionMismatch { + expected: ENVD_PROTOCOL_VERSION, + received: ENVD_PROTOCOL_VERSION + 1, + } ); - let mut reserved = register(&key, &nonce, None); - reserved - .metadata - .insert("lightspeed.x".to_owned(), "y".to_owned()); + + let mut poisoned = register_value(); + poisoned["implementation"]["version"] = json!("x".repeat(300)); assert_eq!( - verify_register_request(&reserved, &nonce).unwrap_err().code, - RegistrationRejectionCode::InvalidRequest + decode_register(&poisoned).unwrap_err().kind(), + ProtocolErrorKind::InvalidRequest ); + let mut reserved = register_value(); + reserved["metadata"] = json!({"lightspeed.x": "y"}); + assert_eq!( + decode_register(&reserved).unwrap_err().kind(), + ProtocolErrorKind::InvalidRequest + ); + + // Hex that is not canonical, or not ASCII, never panics: it is an + // ordinary invalidParams before anything else runs. + for key in [ + "AB".repeat(32), + format!("{}\u{e9}", "a".repeat(62)), + "+1".repeat(32), + ] { + let mut value = register_value(); + value["daemonPublicKey"] = json!(key); + assert_eq!( + decode_register(&value).unwrap_err().kind(), + ProtocolErrorKind::InvalidParams + ); + } + let mut data_url = register_value(); + data_url["dataUrl"] = json!("wss://elsewhere.example/data"); + assert_eq!( + decode_register(&data_url).unwrap_err().kind(), + ProtocolErrorKind::InvalidParams + ); + } + + #[test] + fn the_failure_limiter_keys_on_identities_and_hints_a_retry() { + let start = Instant::now(); + let mut limiter = FailureLimiter::default(); + let first = DaemonPublicKey::from_bytes([1; 32]); + let second = DaemonPublicKey::from_bytes([2; 32]); + for _ in 0..MAX_FAILURES_PER_WINDOW { + assert_eq!(limiter.retry_after(&first, start), None); + limiter.record(first, start); + } + let hint = limiter.retry_after(&first, start).expect("over the limit"); + assert!(hint <= FAILURE_WINDOW && hint > Duration::ZERO); + assert_eq!(limiter.retry_after(&second, start), None); + assert_eq!( + limiter.retry_after(&first, start + FAILURE_WINDOW), + None, + "the window slides" + ); + + // Expired entries are pruned on the next insert. + limiter.record(second, start + FAILURE_WINDOW); + assert_eq!(limiter.len(), 1); + } + + #[test] + fn the_failure_limiter_is_capped() { + let now = Instant::now(); + let mut limiter = FailureLimiter::default(); + for index in 0..=MAX_LIMITED_IDENTITIES { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&(index as u64).to_le_bytes()); + limiter.record( + DaemonPublicKey::from_bytes(bytes), + now + Duration::from_millis(index as u64), + ); + } + assert_eq!(limiter.len(), MAX_LIMITED_IDENTITIES); + assert!( + !limiter + .identities + .contains_key(&DaemonPublicKey::from_bytes([0; 32])), + "the identity whose last failure is oldest is forgotten first" + ); + } + + #[test] + fn only_credential_refusals_count_against_an_identity() { + assert!(counts_as_failure(&unknown_registration_key())); + assert!(counts_as_failure(&unknown_daemon())); + assert!(counts_as_failure(&ProtocolError::expired("x"))); + assert!(!counts_as_failure(®istration_unavailable())); + assert!(!counts_as_failure(&ProtocolError::busy( + Some(BusyReason::CapacityExhausted), + None, + "x" + ))); } #[tokio::test(flavor = "current_thread")] async fn admission_creates_once_reconnects_by_identity_and_spends_closed_identities() { let (store, secret) = store_with_key("rk", policy(RegisteredIdentityMode::Ephemeral)).await; let key = daemon(3); - let nonce = [1u8; 32]; - let unknown = admit_registration(&store, ®ister(&key, &nonce, None), 2_000) + let unknown = admit_registration(&store, ®ister(&key, None), 2_000) .await .unwrap_err(); - assert_eq!(unknown.code, RegistrationRejectionCode::UnknownDaemon); + assert!(matches!( + unknown.data, + ErrorData::NotFound { + reason: Some(NotFoundReason::UnknownDaemon) + } + )); - let created = admit_registration(&store, ®ister(&key, &nonce, Some(&secret)), 2_000) + let created = admit_registration(&store, ®ister(&key, Some(&secret)), 2_000) .await .expect("created"); - assert!(created.created); + assert_eq!(created.admission, Admission::Created); assert_eq!(created.environment.status, EnvironmentStatus::Ready); + let metadata = &created.environment.metadata; assert_eq!( - created - .environment - .metadata - .get("lightspeed.envd.version") - .map(String::as_str), + metadata.get("lightspeed.envd.version").map(String::as_str), Some("0.1.0") ); assert_eq!( - created - .environment - .metadata - .get("lightspeed.envd.gitSha") - .map(String::as_str), - Some("1111111111111111111111111111111111111111") + metadata.get("lightspeed.envd.protocolVersion"), + Some(&ENVD_PROTOCOL_VERSION.to_string()) ); + + let connection = ControlConnectionId::new("connection_1").unwrap(); + let receipt = + registration_receipt(&created.environment, &connection, created.admission).unwrap(); + assert_eq!(receipt.identity_mode, RegistrationIdentityMode::Ephemeral); + assert_eq!(receipt.admission, Admission::Created); + assert_eq!(receipt.control_connection_id, connection); assert_eq!( - created - .environment - .metadata - .get("lightspeed.envd.protocolVersion") - .map(String::as_str), - Some(CURRENT_PROTOCOL_VERSION.to_string().as_str()) + receipt.environment_id.as_str(), + created.environment.environment_id.as_str() ); + assert!(receipt.daemon_id.as_str().starts_with("daemon_")); // Reconnects ignore the key entirely, even a wrong one. - let reconnected = - admit_registration(&store, ®ister(&key, &nonce, Some("lsrk_wrong")), 3_000) - .await - .expect("reconnect"); - assert!(!reconnected.created); + let reconnected = admit_registration(&store, ®ister(&key, Some("lsrk_wrong")), 3_000) + .await + .expect("reconnect"); + assert_eq!(reconnected.admission, Admission::Reconnected); assert_eq!( reconnected.environment.environment_id, created.environment.environment_id @@ -1026,10 +1717,15 @@ mod tests { }) .await .expect("close"); - let closing = admit_registration(&store, ®ister(&key, &nonce, Some(&secret)), 4_100) + let closing = admit_registration(&store, ®ister(&key, Some(&secret)), 4_100) .await .unwrap_err(); - assert_eq!(closing.code, RegistrationRejectionCode::EnvironmentClosed); + assert!(matches!( + closing.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::EnvironmentClosed + } + )); store .finish_close_environment(FinishCloseEnvironment { environment_id: created.environment.environment_id.clone(), @@ -1037,15 +1733,15 @@ mod tests { }) .await .expect("finish"); - let closed = admit_registration(&store, ®ister(&key, &nonce, Some(&secret)), 4_300) + let closed = admit_registration(&store, ®ister(&key, Some(&secret)), 4_300) .await .unwrap_err(); - assert_eq!(closed.code, RegistrationRejectionCode::EnvironmentClosed); - assert!(closed.code.is_terminal()); + assert!(!closed.is_retryable()); + assert_eq!(closed.kind(), ProtocolErrorKind::FailedPrecondition); } #[tokio::test(flavor = "current_thread")] - async fn admission_maps_key_policy_refusals_to_typed_codes() { + async fn admission_maps_key_policy_refusals_to_typed_kinds() { let (store, secret) = store_with_key( "rk", RegistrationKeyPolicy { @@ -1054,24 +1750,30 @@ mod tests { }, ) .await; - let nonce = [2u8; 32]; - admit_registration(&store, ®ister(&daemon(4), &nonce, Some(&secret)), 2_000) + admit_registration(&store, ®ister(&daemon(4), Some(&secret)), 2_000) .await .expect("first"); - let full = admit_registration(&store, ®ister(&daemon(5), &nonce, Some(&secret)), 2_100) + let full = admit_registration(&store, ®ister(&daemon(5), Some(&secret)), 2_100) .await .unwrap_err(); - assert_eq!(full.code, RegistrationRejectionCode::CapacityExhausted); - assert!(!full.code.is_terminal()); + assert!(matches!( + full.data, + ErrorData::Busy { + reason: Some(BusyReason::CapacityExhausted), + .. + } + )); + assert!(full.is_retryable()); - let bad = admit_registration( - &store, - ®ister(&daemon(6), &nonce, Some("lsrk_nope")), - 2_200, - ) - .await - .unwrap_err(); - assert_eq!(bad.code, RegistrationRejectionCode::InvalidRegistrationKey); + let bad = admit_registration(&store, ®ister(&daemon(6), Some("lsrk_nope")), 2_200) + .await + .unwrap_err(); + assert!(matches!( + bad.data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::InvalidRegistrationKey) + } + )); store .revoke_registration_key(RevokeEnvironmentRegistrationKey { @@ -1080,19 +1782,20 @@ mod tests { }) .await .expect("revoke"); - let revoked = - admit_registration(&store, ®ister(&daemon(7), &nonce, Some(&secret)), 3_100) - .await - .unwrap_err(); - assert_eq!( - revoked.code, - RegistrationRejectionCode::RegistrationKeyRevoked - ); + let revoked = admit_registration(&store, ®ister(&daemon(7), Some(&secret)), 3_100) + .await + .unwrap_err(); + assert!(matches!( + revoked.data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::RegistrationKeyRevoked) + } + )); // The already registered daemon still reconnects after revocation. - let reconnect = admit_registration(&store, ®ister(&daemon(4), &nonce, None), 3_200) + let reconnect = admit_registration(&store, ®ister(&daemon(4), None), 3_200) .await .expect("reconnect"); - assert!(!reconnect.created); + assert_eq!(reconnect.admission, Admission::Reconnected); let (expired_store, expired_secret) = store_with_key( "rk-exp", @@ -1104,36 +1807,112 @@ mod tests { .await; let expired = admit_registration( &expired_store, - ®ister(&daemon(8), &nonce, Some(&expired_secret)), + ®ister(&daemon(8), Some(&expired_secret)), 2_000, ) .await .unwrap_err(); + assert_eq!(expired.kind(), ProtocolErrorKind::Expired); + } + + fn text(value: serde_json::Value) -> Message { + Message::Text(value.to_string().into()) + } + + #[test] + fn before_admission_only_register_is_served() { + let register = text(json!({ + "jsonrpc": "2.0", "id": 1, "method": "registration/register", "params": {} + })); + assert!(matches!( + classify_pre_admission(®ister), + PreAdmission::Register(_) + )); + let other = text(json!({"jsonrpc": "2.0", "id": 2, "method": "env/getIdle"})); + let PreAdmission::AnswerAndClose(response) = classify_pre_admission(&other) else { + panic!("a request other than register is refused and closes"); + }; assert_eq!( - expired.code, - RegistrationRejectionCode::RegistrationKeyExpired + response.result.unwrap_err().kind(), + ProtocolErrorKind::NotInitialized ); + let goodbye = text(json!({ + "jsonrpc": "2.0", "method": "registration/goodbye", "params": {"reason": "shutdown"} + })); + assert_eq!(classify_pre_admission(&goodbye), PreAdmission::Close); + let PreAdmission::Answer(response) = + classify_pre_admission(&Message::Text("not json".into())) + else { + panic!("a parse error is answered and the socket stays open"); + }; + assert_eq!(response.id, None); + assert_eq!( + response.result.unwrap_err().kind(), + ProtocolErrorKind::ParseError + ); + assert!(matches!( + classify_pre_admission(&Message::Binary(vec![1_u8].into())), + PreAdmission::Answer(_) + )); + let stray = text(json!({"jsonrpc": "2.0", "id": "g9", "result": {}})); + assert_eq!(classify_pre_admission(&stray), PreAdmission::Ignore); } #[test] - fn data_url_is_the_public_websocket_base_plus_the_data_path() { + fn after_admission_stream_answers_and_goodbye_are_recognized() { assert_eq!( - data_url("https://lightspeed.example/"), - "wss://lightspeed.example/environment-gateway/data" + classify_admitted(&json!({"jsonrpc": "2.0", "id": "g1", "result": {}}).to_string()), + Admitted::StreamAnswer(Id::String("g1".to_owned()), Ok(())) ); + let Admitted::StreamAnswer(_, Err(error)) = classify_admitted( + &jsonrpc::Response::failure( + Some(Id::String("g2".to_owned())), + ProtocolError::unavailable("dial failed"), + ) + .to_text(), + ) else { + panic!("a failed dial is an error answer"); + }; + assert_eq!(error.kind(), ProtocolErrorKind::Unavailable); assert_eq!( - data_url("http://127.0.0.1:18080"), - "ws://127.0.0.1:18080/environment-gateway/data" + classify_admitted( + &json!({ + "jsonrpc": "2.0", "method": "registration/goodbye", + "params": {"reason": "shutdown"} + }) + .to_string() + ), + Admitted::Goodbye + ); + let Admitted::Answer(again) = classify_admitted( + &json!({"jsonrpc": "2.0", "id": 1, "method": "registration/register"}).to_string(), + ) else { + panic!("a second register is answered"); + }; + assert_eq!( + again.result.unwrap_err().kind(), + ProtocolErrorKind::InvalidRequest + ); + let Admitted::Answer(unknown) = classify_admitted( + &json!({"jsonrpc": "2.0", "id": 3, "method": "registration/unknown"}).to_string(), + ) else { + panic!("an unknown request is answered"); + }; + assert_eq!( + unknown.result.unwrap_err().kind(), + ProtocolErrorKind::MethodNotFound ); + assert!(matches!( + classify_admitted(&json!({"jsonrpc": "2.0", "method": "x/y"}).to_string()), + Admitted::Ignore(_) + )); } #[test] - fn failure_accounting_rate_limits_a_subject() { - let registrations = RegisteredConnections::new(); - for _ in 0..MAX_FAILURES_PER_WINDOW { - registrations.record_failure("key:abc"); - } - assert!(registrations.over_failure_limit("key:abc")); - assert!(!registrations.over_failure_limit("key:other")); + fn the_liveness_deadline_is_three_heartbeats() { + assert_eq!( + LIVENESS_TIMEOUT, + HEARTBEAT_INTERVAL * HEARTBEAT_MISSED_INTERVALS + ); } } diff --git a/crates/temporal-runtime/src/gateway/request_context.rs b/crates/temporal-runtime/src/gateway/request_context.rs index 4349766e8..cb6a0165d 100644 --- a/crates/temporal-runtime/src/gateway/request_context.rs +++ b/crates/temporal-runtime/src/gateway/request_context.rs @@ -121,7 +121,7 @@ pub fn initialize_response(caller: CallerAccess) -> api::AgentApiOutcome bool { pub(super) fn validate_caller_metadata( metadata: &BTreeMap, ) -> Result<(), AgentApiError> { - environment_protocol::registration::validate_registration_metadata(None, metadata) - .map_err(|message| AgentApiError::invalid_request(format!("invalid metadata: {message}"))) + environment_protocol::metadata::validate_caller_metadata(metadata) + .map_err(|error| AgentApiError::invalid_request(format!("invalid metadata: {error}"))) } fn validate_delete_after_close_ms(value: Option) -> Result<(), AgentApiError> { @@ -830,7 +830,7 @@ impl GatewayAgentApi { github_api_client: None, model_discovery_openai: None, model_discovery_anthropic: None, - provider_controller_connector: Arc::new(WebSocketProviderControllerConnector::default()), + provider_controller_connector: Arc::new(WebSocketProviderControllerConnector), environment_gateway, } } diff --git a/crates/temporal-runtime/src/gateway/service/session_jobs.rs b/crates/temporal-runtime/src/gateway/service/session_jobs.rs index 06a74520d..31696a893 100644 --- a/crates/temporal-runtime/src/gateway/service/session_jobs.rs +++ b/crates/temporal-runtime/src/gateway/service/session_jobs.rs @@ -2,29 +2,40 @@ use super::*; use ::environments::{EnvironmentId, EnvironmentJobGroupId, EnvironmentRecord}; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; -use environment_client::{EnvironmentClientError, EnvironmentDataClient, WebSocketConnectOptions}; +use environment_client::{ClientError, DataClient}; use environment_protocol::{ data::{ - handshake::{ - InitializeParams as DataInitializeParams, InitializedParams as DataInitializedParams, + exec::{ + ExitStatus as ProtocolExitStatus, GapReason as ProtocolGapReason, + OutputChunk as ProtocolOutputChunk, OutputGap as ProtocolOutputGap, + OutputStream as ProtocolOutputStream, ReturnOn as ProtocolReturnOn, + StoppedBy as ProtocolStoppedBy, }, - jobs::{ - CancelJobsParams as ProtocolCancelJobsParams, JobArtifact as ProtocolJobArtifact, - JobCancelScope as ProtocolJobCancelScope, JobDependency as ProtocolJobDependency, - JobDependencyPolicy as ProtocolJobDependencyPolicy, - JobOutputChunk as ProtocolJobOutputChunk, JobOutputStream as ProtocolJobOutputStream, - JobReadResult as ProtocolJobReadResult, JobStartSpec as ProtocolJobStartSpec, + handshake::JobLimits, + job::{ + CancelJobsParams as ProtocolCancelJobsParams, JobCancelScope as ProtocolJobCancelScope, + JobReadEntry as ProtocolJobReadEntry, JobReadRequest as ProtocolJobReadRequest, JobStatus as ProtocolJobStatus, JobSummary as ProtocolJobSummary, - ReadJobsParams as ProtocolReadJobsParams, StartJobsParams as ProtocolStartJobsParams, + JobSummaryEntry as ProtocolJobSummaryEntry, ReadJobsParams as ProtocolReadJobsParams, }, }, - error::EnvironmentProtocolErrorCode, - shared::{ - ByteChunk, CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentDataConnection, - EnvironmentPath, EnvironmentTransport, JobId, - }, + error::ProtocolErrorKind, + ids::{JobId, JobName, JobRequestId, QueueKey}, + paths::EnvironmentPath, +}; +use temporal_workflow::{ + EnvironmentJobCommand, EnvironmentJobDependency, EnvironmentJobDependencyPolicy, + EnvironmentJobId, EnvironmentJobRecord, EnvironmentJobSpec, EnvironmentJobStartPayloadV1, + EnvironmentJobStatus, }; -use harness::validate_general_string_id; +use tools::environment::jobs::DEFAULT_JOB_OUTPUT_BYTES; + +/// `User-Agent` of the job API's worker-route upgrades; `clientInfo` is +/// always the runtime build. +const JOB_API_USER_AGENT: &str = "lightspeed-temporal-runtime"; + +/// Deadline of each data-plane call the job API makes. +const JOB_API_REQUEST_TIMEOUT: Duration = Duration::from_secs(60); impl GatewayAgentApi { pub(super) async fn create_environment_job_records( @@ -32,19 +43,16 @@ impl GatewayAgentApi { params: EnvironmentJobCreateParams, ) -> Result { let environment_id = parse_environment_job_environment_id(params.environment_id)?; - self.start_environment_jobs(environment_id, params.request_id, params.jobs, None) - .await - } - - async fn start_environment_jobs( - &self, - environment_id: EnvironmentId, - request_id: String, - jobs: Vec, - default_cwd: Option, - ) -> Result { - let mut protocol_specs = Vec::with_capacity(jobs.len()); - for (index, spec) in jobs.into_iter().enumerate() { + let request_id = JobRequestId::new(params.request_id).map_err(|error| { + AgentApiError::invalid_request(format!("invalid request_id: {error}")) + })?; + if params.jobs.is_empty() { + return Err(AgentApiError::invalid_request( + "environments/jobs/create requires at least one job", + )); + } + let mut specs = Vec::with_capacity(params.jobs.len()); + for (index, spec) in params.jobs.into_iter().enumerate() { let job_id = spec .job_id .as_deref() @@ -52,68 +60,42 @@ impl GatewayAgentApi { .transpose() .map_err(AgentApiError::invalid_request)? .unwrap_or_else(|| derived_job_id(&environment_id, &request_id, index)); - let mut spec = - api_start_spec_to_protocol(spec, job_id).map_err(AgentApiError::invalid_request)?; - if spec.cwd.is_none() { - spec.cwd = default_cwd.clone(); - } - protocol_specs.push(spec); + specs.push( + api_start_spec_to_workflow(spec, job_id).map_err(AgentApiError::invalid_request)?, + ); } let instance = self.read_job_instance(&environment_id).await?; - self.start_environment_jobs_protocol_specs(instance, request_id, protocol_specs) + // Refuse what the daemon would refuse before a workflow exists for + // it: a daemon without jobs, or more jobs than one start takes. + let client = self.connect_job_client(&instance).await?; + let limits = client.initialize_response().limits.job.clone(); + let _ = client.close().await; + if specs.len() > limits.max_request_jobs as usize { + return Err(AgentApiError::invalid_request(format!( + "environments/jobs/create names {} jobs; the environment starts at most {} at once", + specs.len(), + limits.max_request_jobs + ))); + } + self.start_environment_jobs(instance, request_id, specs) .await } - async fn start_environment_jobs_protocol_specs( + async fn start_environment_jobs( &self, instance: EnvironmentRecord, - request_id: String, - jobs: Vec, + request_id: JobRequestId, + jobs: Vec, ) -> Result { - validate_job_request_id(&request_id)?; - if jobs.is_empty() { - return Err(AgentApiError::invalid_request( - "environments/jobs/create requires at least one job", - )); - } - let key = self - .environment_gateway - .route_key(self.store.config().universe_id, &instance); - let connection = self.environment_gateway.connection(&key); - let (_client, capabilities) = connect_initialized_environment_data_client( - &connection, - self.environment_gateway - .connect_options("lightspeed-temporal-runtime"), - ) - .await - .map_err(|error| AgentApiError::rejected(error.to_string()))?; - if !capabilities.job_start { - return Err(AgentApiError::rejected(format!( - "environment does not support durable job start: {}", - instance.environment_id - ))); - } - let request = ProtocolStartJobsParams { - namespace: instance.environment_id.as_str().to_owned(), - request_id: request_id.clone(), - jobs, - }; - let request_fingerprint = - BlobRef::from_bytes(&serde_json::to_vec(&request).map_err(|error| { - AgentApiError::internal(format!("encode environment job request: {error}")) - })?); - let job_group_id = derived_job_group_id( - &instance.environment_id, - &request_id, - request_fingerprint.as_str(), - ); - let job_ids = request.jobs.iter().map(|job| job.job_id.clone()).collect(); - let start_payload = temporal_workflow::EnvironmentJobStartPayload { request }; + let payload = EnvironmentJobStartPayloadV1::new(request_id.into_string(), jobs); + let encoded = payload.encode().map_err(|error| { + AgentApiError::internal(format!("encode environment job workflow start: {error}")) + })?; + let job_group_id = + derived_job_group_id(&instance.environment_id, &BlobRef::from_bytes(&encoded)); let request_ref = self .store - .put_bytes(serde_json::to_vec(&start_payload).map_err(|error| { - AgentApiError::internal(format!("encode environment job workflow start: {error}")) - })?) + .put_bytes(encoded) .await .map_err(map_blob_store_error)?; let snapshot = self @@ -124,7 +106,7 @@ impl GatewayAgentApi { job_group_id: job_group_id.as_str().to_owned(), request_ref, }, - job_ids, + payload.job_ids(), Vec::new(), ) .await?; @@ -134,22 +116,7 @@ impl GatewayAgentApi { jobs: snapshot .jobs .into_iter() - .map(|summary| SessionJobStartedView { - name: summary.name, - job_id: summary.job_id.as_str().to_owned(), - handle: SessionJobHandleView { - environment_id: instance.environment_id.as_str().to_owned(), - job_id: summary.job_id.as_str().to_owned(), - }, - promise_id: None, - status: api_job_status(summary.status), - dependencies: summary - .dependencies - .into_iter() - .map(|id| id.as_str().to_owned()) - .collect(), - queue_key: summary.queue_key, - }) + .map(|record| api_started_job(&instance.environment_id, record)) .collect(), }) } @@ -157,7 +124,7 @@ impl GatewayAgentApi { async fn start_environment_job_workflow( &self, start: temporal_workflow::EnvironmentJobStartActivityRequest, - job_ids: Vec, + job_ids: Vec, subscriptions: Vec, ) -> Result { let workflow_id = temporal_workflow::compose_environment_job_workflow_id( @@ -230,158 +197,164 @@ impl GatewayAgentApi { &self, params: EnvironmentJobReadParams, ) -> Result { - let jobs = self - .read_jobs( - params.jobs, - params.output_bytes, - params.after_seq, - params.include_artifacts, - ) - .await?; - Ok(EnvironmentJobReadResponse { jobs }) - } - - async fn read_jobs( - &self, - handles: Vec, - output_bytes: Option, - after_seq: Option, - include_artifacts: bool, - ) -> Result, AgentApiError> { - let mut entries = Vec::with_capacity(handles.len()); - for handle in handles { - let resolved = match parse_job_handle(handle) { - Ok(handle) => handle, - Err(error) => { - entries.push(session_job_read_error(None, error)); - continue; - } - }; - let environment_id = match EnvironmentId::try_new(resolved.environment_id.clone()) { - Ok(environment_id) => environment_id, - Err(error) => { - entries.push(session_job_read_error(Some(resolved), error.to_string())); - continue; - } - }; - let job_id = match parse_protocol_job_id(&resolved.job_id) { - Ok(job_id) => job_id, - Err(error) => { - entries.push(session_job_read_error(Some(resolved), error)); - continue; - } - }; - let mut client = match self - .connect_client_for_job_handle(&environment_id, "read") + if params.after.is_some() && params.tail_bytes.is_some() { + return Err(AgentApiError::invalid_request( + "environments/jobs/read takes after or tailBytes, not both", + )); + } + let position = JobReadPosition { + after: params.after, + tail_bytes: params.tail_bytes, + max_bytes: params.max_bytes.unwrap_or(DEFAULT_JOB_OUTPUT_BYTES), + }; + let mut entries: Vec> = vec![None; params.jobs.len()]; + let mut groups: BTreeMap> = + BTreeMap::new(); + for (index, handle) in params.jobs.into_iter().enumerate() { + match parse_job_handle(handle) { + Ok((environment_id, handle, job_id)) => groups + .entry(environment_id) + .or_default() + .push((index, handle, job_id)), + Err(error) => entries[index] = Some(session_job_read_error(None, error)), + } + } + for (environment_id, jobs) in groups { + match self + .read_jobs_in_environment(&environment_id, &jobs, position) .await { - Ok(client) => client, + Ok(read) => { + for ((index, _, _), entry) in jobs.iter().zip(read) { + entries[*index] = Some(entry); + } + } Err(error) => { - entries.push(session_job_read_error(Some(resolved), error)); - continue; + for (index, handle, _) in jobs { + entries[index] = Some(session_job_read_error(Some(handle), error.clone())); + } + } + } + } + Ok(EnvironmentJobReadResponse { + jobs: entries.into_iter().flatten().collect(), + }) + } + + /// One `job/read` entry per job, in order, within the daemon's ceilings. + async fn read_jobs_in_environment( + &self, + environment_id: &EnvironmentId, + jobs: &[(usize, SessionJobHandleView, JobId)], + position: JobReadPosition, + ) -> Result, String> { + let instance = self + .read_job_instance(environment_id) + .await + .map_err(|error| error.message)?; + let client = self + .connect_job_client(&instance) + .await + .map_err(|error| error.message)?; + let limits = client.initialize_response().limits.job.clone(); + let mut entries = Vec::with_capacity(jobs.len()); + let mut result = Ok(()); + for chunk in jobs.chunks((limits.max_request_jobs as usize).max(1)) { + let params = + job_read_params(chunk.iter().map(|(_, _, job_id)| job_id), position, &limits); + match client.read_jobs(¶ms).await { + Ok(response) => { + for ((_, handle, _), entry) in chunk.iter().zip(response.jobs) { + entries.push(session_job_read_entry(handle.clone(), entry)); + } } - }; - match client - .read_jobs(&ProtocolReadJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: vec![job_id], - after_seq, - max_bytes: output_bytes, - include_artifacts, - wait_ms: None, - }) - .await - { - Ok(response) => entries.push(session_job_read_entry_from_response( - resolved, - response.jobs.into_iter().next(), - )), Err(error) => { - entries.push(session_job_read_error(Some(resolved), error.to_string())) + result = Err(error.to_string()); + break; } } } - Ok(entries) + let _ = client.close().await; + result.map(|()| entries) } pub(super) async fn cancel_environment_job_records( &self, params: EnvironmentJobCancelParams, ) -> Result { - let jobs = self - .cancel_jobs(params.jobs, params.scope, params.force) - .await?; - Ok(EnvironmentJobCancelResponse { jobs }) - } - - async fn cancel_jobs( - &self, - handles: Vec, - scope: SessionJobCancelScopeView, - force: bool, - ) -> Result, AgentApiError> { - let mut entries = Vec::with_capacity(handles.len()); - for handle in handles { - let resolved = match parse_job_handle(handle) { - Ok(handle) => handle, - Err(error) => { - entries.push(session_job_cancel_error(None, error)); - continue; - } - }; - let environment_id = match EnvironmentId::try_new(resolved.environment_id.clone()) { - Ok(environment_id) => environment_id, - Err(error) => { - entries.push(session_job_cancel_error(Some(resolved), error.to_string())); - continue; - } - }; - let job_id = match parse_protocol_job_id(&resolved.job_id) { - Ok(job_id) => job_id, - Err(error) => { - entries.push(session_job_cancel_error(Some(resolved), error)); - continue; - } - }; + let scope = match params.scope { + SessionJobCancelScopeView::Job => ProtocolJobCancelScope::Job, + SessionJobCancelScopeView::Dependents => ProtocolJobCancelScope::Dependents, + }; + let mut entries = Vec::with_capacity(params.jobs.len()); + let mut groups: BTreeMap> = + BTreeMap::new(); + for handle in params.jobs { + match parse_job_handle(handle) { + Ok((environment_id, handle, job_id)) => groups + .entry(environment_id) + .or_default() + .push((handle, job_id)), + Err(error) => entries.push(session_job_cancel_error(None, error)), + } + } + for (environment_id, jobs) in groups { match self - .cancel_job_on_provider(&environment_id, job_id, scope, force) + .cancel_jobs_in_environment(&environment_id, &jobs, scope, params.force) .await { - Ok(summary) => entries.push(SessionJobCancelEntryView { - handle: Some(resolved), - summary: Some(api_job_summary(summary)), - error: None, - }), - Err(error) => entries.push(session_job_cancel_error(Some(resolved), error)), + Ok(cancelled) => entries.extend(cancelled), + Err(error) => entries.extend( + jobs.into_iter() + .map(|(handle, _)| session_job_cancel_error(Some(handle), error.clone())), + ), } } - Ok(entries) + Ok(EnvironmentJobCancelResponse { jobs: entries }) } - async fn cancel_job_on_provider( + /// The daemon's entries for the named jobs, then for the dependents a + /// `dependents` cancel reached. + async fn cancel_jobs_in_environment( &self, environment_id: &EnvironmentId, - job_id: JobId, - scope: SessionJobCancelScopeView, + jobs: &[(SessionJobHandleView, JobId)], + scope: ProtocolJobCancelScope, force: bool, - ) -> Result { - let mut client = self - .connect_client_for_job_handle(environment_id, "cancel") - .await?; - let response = client - .cancel_jobs(&ProtocolCancelJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: vec![job_id], - scope: protocol_cancel_scope(scope), - force, - }) + ) -> Result, String> { + let instance = self + .read_job_instance(environment_id) .await - .map_err(|error| error.to_string())?; - response - .jobs - .into_iter() - .next() - .ok_or_else(|| "provider returned no job result".to_owned()) + .map_err(|error| error.message)?; + let client = self + .connect_job_client(&instance) + .await + .map_err(|error| error.message)?; + let per_request = + (client.initialize_response().limits.job.max_request_jobs as usize).max(1); + let mut entries = Vec::new(); + let mut result = Ok(()); + for chunk in jobs.chunks(per_request) { + let params = ProtocolCancelJobsParams { + job_ids: chunk.iter().map(|(_, job_id)| job_id.clone()).collect(), + scope, + force, + }; + match client.cancel_jobs(¶ms).await { + Ok(response) => entries.extend( + response + .jobs + .into_iter() + .map(|entry| session_job_cancel_entry(environment_id, entry)), + ), + Err(error) => { + result = Err(error.to_string()); + break; + } + } + } + let _ = client.close().await; + result.map(|()| entries) } /// Resolve the environment through the shared status-aware resolver so @@ -401,236 +374,269 @@ impl GatewayAgentApi { .map_err(super::environments::map_environment_resolve_error) } - async fn connect_client_for_job_handle( + /// An initialized data client for the environment; its daemon must + /// serve the `job` family. + async fn connect_job_client( &self, - environment_id: &EnvironmentId, - operation: &str, - ) -> Result, String> { - let instance = self - .read_job_instance(environment_id) - .await - .map_err(|error| error.to_string())?; + instance: &EnvironmentRecord, + ) -> Result { let key = self .environment_gateway - .route_key(self.store.config().universe_id, &instance); - let connection = self.environment_gateway.connection(&key); - let (client, capabilities) = connect_initialized_environment_data_client( - &connection, + .route_key(self.store.config().universe_id, instance); + let (client, initialized) = DataClient::connect_initialized( + &self.environment_gateway.endpoint(&key), self.environment_gateway - .connect_options("lightspeed-temporal-runtime"), + .connect_options(JOB_API_USER_AGENT) + .with_request_timeout(JOB_API_REQUEST_TIMEOUT), ) .await - .map_err(|error| error.to_string())?; - let supported = match operation { - "read" => capabilities.job_read, - "cancel" => capabilities.job_cancel, - _ => true, - }; - if !supported { - return Err(format!( - "environment does not support durable job {operation}: {}", - environment_id - )); + .map_err(map_environment_client_api_error)?; + if initialized.capabilities.job.is_none() { + let _ = client.close().await; + return Err(AgentApiError::rejected(format!( + "environment does not serve durable jobs: {}", + instance.environment_id + ))); } Ok(client) } } +/// Where each job of an API read starts and how much it shows. +#[derive(Clone, Copy, Debug)] +struct JobReadPosition { + after: Option, + tail_bytes: Option, + /// Per job; never more than the daemon's read ceiling allows. + max_bytes: u64, +} + +/// One `job/read` for `jobs` at `position`, waiting for nothing. The +/// per-job budget shrinks so the total stays within `maxReadBytes`. +fn job_read_params<'a>( + jobs: impl ExactSizeIterator, + position: JobReadPosition, + limits: &JobLimits, +) -> ProtocolReadJobsParams { + let count = jobs.len().max(1) as u64; + let per_job = position.max_bytes.min(limits.max_read_bytes / count); + ProtocolReadJobsParams { + jobs: jobs + .map(|job_id| ProtocolJobReadRequest { + job_id: job_id.clone(), + after: position.after, + tail_bytes: position.tail_bytes, + max_bytes: Some(per_job), + }) + .collect(), + max_bytes: per_job * count, + wait_ms: 0, + return_on: ProtocolReturnOn::AnyOutput, + } +} + fn parse_environment_job_environment_id(value: String) -> Result { EnvironmentId::try_new(value).map_err(|error| { AgentApiError::invalid_request(format!("invalid environment instance id: {error}")) }) } -fn validate_job_request_id(value: &str) -> Result<(), AgentApiError> { - validate_general_string_id("job request_id", value) - .map_err(|error| AgentApiError::invalid_request(format!("invalid request_id: {error}"))) -} - fn parse_protocol_job_id(value: &str) -> Result { - validate_general_string_id("job_id", value) - .map_err(|error| format!("invalid job_id: {error}"))?; - Ok(JobId::new(value.to_owned())) + JobId::new(value).map_err(|error| format!("invalid job_id: {error}")) } -fn parse_job_handle(handle: SessionJobHandleInput) -> Result { +fn parse_job_handle( + handle: SessionJobHandleInput, +) -> Result<(EnvironmentId, SessionJobHandleView, JobId), String> { let environment_id = EnvironmentId::try_new(handle.environment_id) .map_err(|error| format!("invalid job handle environment_id: {error}"))?; let job_id = parse_protocol_job_id(&handle.job_id)?; - Ok(SessionJobHandleView { + let view = SessionJobHandleView { environment_id: environment_id.as_str().to_owned(), job_id: job_id.as_str().to_owned(), - }) + }; + Ok((environment_id, view, job_id)) } -fn api_start_spec_to_protocol( +/// The job workflow's record of an API job spec, validated against the +/// wire grammar so the start activity cannot refuse it later. +fn api_start_spec_to_workflow( spec: SessionJobStartSpecInput, job_id: JobId, -) -> Result { +) -> Result { if spec.argv.is_empty() { return Err("job argv must not be empty".to_owned()); } - Ok(ProtocolJobStartSpec { - job_id, - name: spec.name, - argv: spec.argv, - cwd: spec - .cwd - .as_deref() - .map(EnvironmentPath::new) - .transpose() - .map_err(|error| format!("invalid job cwd: {error}"))?, - env: spec.env, - secret_env: BTreeMap::new(), - stdin: spec.stdin.map(|stdin| ByteChunk::from(stdin.into_bytes())), - timeout_ms: spec.timeout_ms, + let name = spec + .name + .map(|name| { + JobName::new(name) + .map(JobName::into_string) + .map_err(|error| format!("invalid job name: {error}")) + }) + .transpose()?; + let cwd = spec + .cwd + .map(|cwd| { + EnvironmentPath::new(cwd) + .map(|cwd| cwd.as_str().to_owned()) + .map_err(|error| format!("invalid job cwd: {error}")) + }) + .transpose()?; + let queue_key = spec + .queue_key + .map(|key| { + QueueKey::new(key) + .map(QueueKey::into_string) + .map_err(|error| format!("invalid job queue_key: {error}")) + }) + .transpose()?; + Ok(EnvironmentJobSpec { + job_id: EnvironmentJobId::new(job_id.into_string()), + name, + command: EnvironmentJobCommand { + argv: spec.argv, + cwd, + env: spec.env, + stdin: spec.stdin, + timeout_ms: spec.timeout_ms, + }, depends_on: spec .depends_on .into_iter() - .map(api_dependency_to_protocol) + .map(api_dependency_to_workflow) .collect::, _>>()?, dependency_policy: match spec.dependency_policy { SessionJobDependencyPolicyView::AllSucceeded => { - ProtocolJobDependencyPolicy::AllSucceeded + EnvironmentJobDependencyPolicy::AllSucceeded + } + SessionJobDependencyPolicyView::AllTerminal => { + EnvironmentJobDependencyPolicy::AllTerminal } - SessionJobDependencyPolicyView::AllTerminal => ProtocolJobDependencyPolicy::AllTerminal, }, - queue_key: spec.queue_key, + queue_key, }) } -fn api_dependency_to_protocol( +fn api_dependency_to_workflow( dependency: SessionJobDependencyInput, -) -> Result { +) -> Result { match (dependency.job_id, dependency.name) { - (Some(job_id), None) => Ok(ProtocolJobDependency { - job_id: Some(parse_protocol_job_id(&job_id)?), - name: None, + (Some(job_id), None) => Ok(EnvironmentJobDependency::JobId { + job_id: EnvironmentJobId::new(parse_protocol_job_id(&job_id)?.into_string()), }), - (None, Some(name)) if !name.is_empty() => Ok(ProtocolJobDependency { - job_id: None, - name: Some(name), + (None, Some(name)) => Ok(EnvironmentJobDependency::Name { + name: JobName::new(name) + .map_err(|error| format!("invalid job dependency name: {error}"))? + .into_string(), }), (Some(_), Some(_)) => { Err("job dependency must specify job_id or name, not both".to_owned()) } - _ => Err("job dependency must specify job_id or name".to_owned()), + (None, None) => Err("job dependency must specify job_id or name".to_owned()), } } -fn protocol_cancel_scope(scope: SessionJobCancelScopeView) -> ProtocolJobCancelScope { - match scope { - SessionJobCancelScopeView::Job => ProtocolJobCancelScope::Job, - SessionJobCancelScopeView::Dependents => ProtocolJobCancelScope::Dependents, - } -} - -fn derived_job_id(environment_id: &EnvironmentId, request_id: &str, index: usize) -> JobId { +fn derived_job_id( + environment_id: &EnvironmentId, + request_id: &JobRequestId, + index: usize, +) -> JobId { let hash = BlobRef::from_bytes(format!("{environment_id}:{request_id}:{index}").as_bytes()); - JobId::new(format!("job-{}", &hash.as_str()[7..31])) + JobId::new(format!("job-{}", &hash.as_str()[7..31])).expect("hex job id fits the id grammar") } +/// A group is named by its environment and the content address of its +/// encoded start payload, which is also the payload's CAS `request_ref`. fn derived_job_group_id( environment_id: &EnvironmentId, - request_id: &str, - request_fingerprint: &str, + payload_ref: &BlobRef, ) -> EnvironmentJobGroupId { - let hash = BlobRef::from_bytes( - format!("{environment_id}:{request_id}:{request_fingerprint}").as_bytes(), - ); + let hash = BlobRef::from_bytes(format!("{environment_id}:{payload_ref}").as_bytes()); EnvironmentJobGroupId::new(format!("ejg_{}", &hash.as_str()[7..31])) } -async fn connect_initialized_environment_data_client( - connection: &EnvironmentDataConnection, - options: WebSocketConnectOptions, -) -> Result< - ( - EnvironmentDataClient, - EnvironmentCapabilities, - ), - AgentApiError, -> { - let mut client = connect_environment_data_client(connection, options).await?; - let response = client - .initialize(&DataInitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-temporal-runtime".to_owned(), - scope: connection.scope.clone(), - resume_connection_id: None, - }) - .await - .map_err(map_environment_client_api_error)?; - if response.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(AgentApiError::internal(format!( - "unsupported environment data protocol version {}; expected {CURRENT_PROTOCOL_VERSION}", - response.protocol_version - ))); +fn map_environment_client_api_error(error: ClientError) -> AgentApiError { + let message = error.to_string(); + match error.protocol_kind() { + Some(ProtocolErrorKind::InvalidParams | ProtocolErrorKind::InvalidRequest) => { + AgentApiError::invalid_request(message) + } + Some(ProtocolErrorKind::NotFound) => AgentApiError::not_found(message), + Some(ProtocolErrorKind::AlreadyExists) => AgentApiError::conflict(message), + Some( + ProtocolErrorKind::FailedPrecondition + | ProtocolErrorKind::CapabilityUnavailable + | ProtocolErrorKind::PermissionDenied + | ProtocolErrorKind::VersionMismatch, + ) => AgentApiError::rejected(message), + _ => match error { + ClientError::WrongPlane { .. } | ClientError::Configuration { .. } => { + AgentApiError::rejected(message) + } + _ => AgentApiError::internal(message), + }, } - let capabilities = response.capabilities; - client - .initialized(&DataInitializedParams {}) - .await - .map_err(map_environment_client_api_error)?; - Ok((client, capabilities)) } -pub(crate) async fn connect_environment_data_client( - connection: &EnvironmentDataConnection, - options: WebSocketConnectOptions, -) -> Result, AgentApiError> { - match &connection.transport { - EnvironmentTransport::WebSocket => { - EnvironmentDataClient::connect(&connection.endpoint, options) - .await - .map_err(map_environment_client_api_error) - } - other => Err(AgentApiError::rejected(format!( - "unsupported environment data transport for environment jobs: {other:?}" - ))), +/// The started view of a job the workflow recorded. A job the daemon no +/// longer knows (`lost`) is reported as failed: the API has no such status. +fn api_started_job( + environment_id: &EnvironmentId, + record: EnvironmentJobRecord, +) -> SessionJobStartedView { + SessionJobStartedView { + name: record.name, + job_id: record.job_id.as_str().to_owned(), + handle: SessionJobHandleView { + environment_id: environment_id.as_str().to_owned(), + job_id: record.job_id.as_str().to_owned(), + }, + promise_id: None, + status: api_record_status(record.status), + dependencies: record + .depends_on + .into_iter() + .map(EnvironmentJobId::into_string) + .collect(), + queue_key: record.queue_key, } } -fn map_environment_client_api_error(error: EnvironmentClientError) -> AgentApiError { - match error { - EnvironmentClientError::Protocol(error) => match error.code { - EnvironmentProtocolErrorCode::InvalidRequest => { - AgentApiError::invalid_request(error.message) - } - EnvironmentProtocolErrorCode::NotFound => AgentApiError::not_found(error.message), - EnvironmentProtocolErrorCode::Conflict - | EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable => { - AgentApiError::rejected(error.message) - } - _ => AgentApiError::internal(error.message), - }, - other => AgentApiError::internal(other.to_string()), +fn api_record_status(status: EnvironmentJobStatus) -> SessionJobStatusView { + match status { + EnvironmentJobStatus::Queued => SessionJobStatusView::Queued, + EnvironmentJobStatus::Running => SessionJobStatusView::Running, + EnvironmentJobStatus::Succeeded => SessionJobStatusView::Succeeded, + EnvironmentJobStatus::Failed | EnvironmentJobStatus::Lost => SessionJobStatusView::Failed, + EnvironmentJobStatus::TimedOut => SessionJobStatusView::TimedOut, + EnvironmentJobStatus::Cancelled => SessionJobStatusView::Cancelled, + EnvironmentJobStatus::DependencyFailed => SessionJobStatusView::DependencyFailed, + EnvironmentJobStatus::Interrupted => SessionJobStatusView::Interrupted, } } -fn session_job_read_entry_from_response( +fn session_job_read_entry( handle: SessionJobHandleView, - response: Option, + entry: ProtocolJobReadEntry, ) -> SessionJobReadEntryView { - match response { - Some(response) => SessionJobReadEntryView { + match entry { + ProtocolJobReadEntry::Job { summary, output } => SessionJobReadEntryView { handle: Some(handle), - summary: Some(api_job_summary(response.summary)), - output_chunks: response - .output_chunks + summary: Some(api_job_summary(*summary)), + output_chunks: output + .chunks .into_iter() .map(api_job_output_chunk) .collect(), - output_next_seq: response.output_next_seq, - artifacts: response - .artifacts - .into_iter() - .map(api_job_artifact) - .collect(), + output_next: output.next, + output_end: output.end, + output_gaps: output.gaps.into_iter().map(api_job_output_gap).collect(), error: None, }, - None => session_job_read_error(Some(handle), "provider returned no job result".to_owned()), + ProtocolJobReadEntry::Error { error, .. } => { + session_job_read_error(Some(handle), error.to_string()) + } } } @@ -642,12 +648,33 @@ fn session_job_read_error( handle, summary: None, output_chunks: Vec::new(), - output_next_seq: 0, - artifacts: Vec::new(), + output_next: 0, + output_end: 0, + output_gaps: Vec::new(), error: Some(error), } } +fn session_job_cancel_entry( + environment_id: &EnvironmentId, + entry: ProtocolJobSummaryEntry, +) -> SessionJobCancelEntryView { + let handle = |job_id: &JobId| SessionJobHandleView { + environment_id: environment_id.as_str().to_owned(), + job_id: job_id.as_str().to_owned(), + }; + match entry { + ProtocolJobSummaryEntry::Job { summary } => SessionJobCancelEntryView { + handle: Some(handle(&summary.job_id)), + summary: Some(api_job_summary(*summary)), + error: None, + }, + ProtocolJobSummaryEntry::Error { job_id, error } => { + session_job_cancel_error(Some(handle(&job_id)), error.to_string()) + } + } +} + fn session_job_cancel_error( handle: Option, error: String, @@ -661,57 +688,254 @@ fn session_job_cancel_error( fn api_job_summary(summary: ProtocolJobSummary) -> SessionJobSummaryView { SessionJobSummaryView { - namespace: summary.namespace, - job_id: summary.job_id.as_str().to_owned(), - name: summary.name, + job_id: summary.job_id.into_string(), + request_id: summary.request_id.into_string(), + name: summary.name.map(JobName::into_string), status: api_job_status(summary.status), dependencies: summary - .dependencies + .depends_on .into_iter() - .map(|id| id.as_str().to_owned()) + .map(JobId::into_string) .collect(), created_at_ms: summary.created_at_ms, queued_at_ms: summary.queued_at_ms, started_at_ms: summary.started_at_ms, finished_at_ms: summary.finished_at_ms, - exit_code: summary.exit_code, - orphaned_descendants: summary.orphaned_descendants, - failure: summary.failure, - queue_key: summary.queue_key, + cancel_requested_at_ms: summary.cancel_requested_at_ms, + duration_ms: summary.duration_ms, + exit: summary.exit.map(|exit| match exit { + ProtocolExitStatus::Code { code } => SessionJobExitView::Code { code }, + ProtocolExitStatus::Signal { + signal, + core_dumped, + } => SessionJobExitView::Signal { + signal: signal.as_str().to_owned(), + core_dumped, + }, + }), + stopped_by: summary.stopped_by.map(|stopped_by| match stopped_by { + ProtocolStoppedBy::Timeout => SessionJobStoppedByView::Timeout, + ProtocolStoppedBy::Terminate => SessionJobStoppedByView::Terminate, + ProtocolStoppedBy::Kill => SessionJobStoppedByView::Kill, + }), + orphaned_descendants: summary.leftovers.is_some(), + failure: summary.error.map(|error| error.to_string()), + queue_key: summary.queue_key.map(QueueKey::into_string), } } fn api_job_status(status: ProtocolJobStatus) -> SessionJobStatusView { match status { - ProtocolJobStatus::Accepted => SessionJobStatusView::Accepted, ProtocolJobStatus::Queued => SessionJobStatusView::Queued, ProtocolJobStatus::Running => SessionJobStatusView::Running, ProtocolJobStatus::Succeeded => SessionJobStatusView::Succeeded, ProtocolJobStatus::Failed => SessionJobStatusView::Failed, - ProtocolJobStatus::CancelRequested => SessionJobStatusView::CancelRequested, - ProtocolJobStatus::Cancelled => SessionJobStatusView::Cancelled, ProtocolJobStatus::TimedOut => SessionJobStatusView::TimedOut, + ProtocolJobStatus::Cancelled => SessionJobStatusView::Cancelled, ProtocolJobStatus::DependencyFailed => SessionJobStatusView::DependencyFailed, ProtocolJobStatus::Interrupted => SessionJobStatusView::Interrupted, - ProtocolJobStatus::Lost => SessionJobStatusView::Lost, } } -fn api_job_output_chunk(chunk: ProtocolJobOutputChunk) -> SessionJobOutputChunkView { +fn api_job_output_chunk(chunk: ProtocolOutputChunk) -> SessionJobOutputChunkView { SessionJobOutputChunkView { - seq: chunk.seq, + offset: chunk.offset, stream: match chunk.stream { - ProtocolJobOutputStream::Stdout => SessionJobOutputStreamView::Stdout, - ProtocolJobOutputStream::Stderr => SessionJobOutputStreamView::Stderr, + // Jobs never run on a PTY; a merged stream reads as stdout. + ProtocolOutputStream::Stdout | ProtocolOutputStream::Pty => { + SessionJobOutputStreamView::Stdout + } + ProtocolOutputStream::Stderr => SessionJobOutputStreamView::Stderr, }, - data_base64: BASE64_STANDARD.encode(chunk.chunk.into_inner()), + data_base64: BASE64_STANDARD.encode(chunk.data.as_slice()), } } -fn api_job_artifact(artifact: ProtocolJobArtifact) -> SessionJobArtifactView { - SessionJobArtifactView { - path: artifact.path.as_str().to_owned(), - kind: artifact.kind, - metadata: artifact.metadata, +fn api_job_output_gap(gap: ProtocolOutputGap) -> SessionJobOutputGapView { + SessionJobOutputGapView { + offset: gap.offset, + omitted_bytes: gap.omitted_bytes, + reason: match gap.reason { + ProtocolGapReason::Overflow => SessionJobOutputGapReasonView::Overflow, + ProtocolGapReason::Expired => SessionJobOutputGapReasonView::Expired, + }, + } +} + +#[cfg(test)] +mod tests { + use environment_protocol::{bytes::ByteChunk, data::exec::OutputPage, error::ProtocolError}; + + use super::*; + + fn limits(max_read_bytes: u64) -> JobLimits { + JobLimits { + max_read_bytes, + max_request_jobs: 64, + max_retained_output_bytes: 2 * 1024 * 1024, + record_retention_ms: 1, + cancel_grace_ms: 1, + } + } + + fn job_id(value: &str) -> JobId { + JobId::new(value).unwrap() + } + + #[test] + fn reads_split_the_daemon_ceiling_across_jobs() { + let jobs = [job_id("a"), job_id("b")]; + let params = job_read_params( + jobs.iter(), + JobReadPosition { + after: None, + tail_bytes: Some(100), + max_bytes: 1_000, + }, + &limits(600), + ); + assert_eq!(params.max_bytes, 600); + assert_eq!(params.wait_ms, 0); + for (entry, expected) in params.jobs.iter().zip(&jobs) { + assert_eq!(&entry.job_id, expected); + assert_eq!(entry.tail_bytes, Some(100)); + assert_eq!(entry.after, None); + assert_eq!(entry.max_bytes, Some(300)); + } + } + + #[test] + fn api_specs_are_validated_against_the_wire_grammar() { + let spec = |name: Option<&str>, cwd: Option<&str>| SessionJobStartSpecInput { + name: name.map(str::to_owned), + job_id: None, + argv: vec!["make".to_owned()], + cwd: cwd.map(str::to_owned), + env: BTreeMap::new(), + stdin: Some("input".to_owned()), + timeout_ms: None, + depends_on: vec![SessionJobDependencyInput { + job_id: None, + name: Some("lint".to_owned()), + }], + dependency_policy: SessionJobDependencyPolicyView::AllTerminal, + queue_key: Some("ci".to_owned()), + }; + let stored = + api_start_spec_to_workflow(spec(Some("build"), Some("/workspace")), job_id("build")) + .unwrap(); + assert_eq!(stored.command.cwd.as_deref(), Some("/workspace")); + assert_eq!(stored.command.stdin.as_deref(), Some("input")); + assert_eq!( + stored.depends_on, + vec![EnvironmentJobDependency::Name { + name: "lint".to_owned() + }] + ); + assert!(api_start_spec_to_workflow(spec(None, Some("relative")), job_id("x")).is_err()); + assert!(api_start_spec_to_workflow(spec(Some("a\tb"), None), job_id("x")).is_err()); + assert!(parse_protocol_job_id("not a job id").is_err()); + } + + #[test] + fn read_entries_carry_byte_positions_and_entry_errors() { + let summary: ProtocolJobSummary = serde_json::from_value(serde_json::json!({ + "jobId": "build", + "requestId": "req-1", + "status": "failed", + "createdAtMs": 1, + "exit": {"kind": "signal", "signal": "SIGSEGV", "coreDumped": true}, + "stoppedBy": "kill", + })) + .unwrap(); + let handle = SessionJobHandleView { + environment_id: "environment_1".to_owned(), + job_id: "build".to_owned(), + }; + let entry = session_job_read_entry( + handle.clone(), + ProtocolJobReadEntry::Job { + summary: Box::new(summary), + output: OutputPage { + chunks: vec![ProtocolOutputChunk { + offset: 10, + stream: ProtocolOutputStream::Stderr, + data: ByteChunk::from(b"boom".to_vec()), + }], + next: 14, + end: 40, + gaps: vec![ProtocolOutputGap { + offset: 4, + omitted_bytes: 6, + reason: ProtocolGapReason::Overflow, + }], + }, + }, + ); + assert_eq!((entry.output_next, entry.output_end), (14, 40)); + assert_eq!(entry.output_chunks[0].offset, 10); + assert_eq!( + entry.output_chunks[0].stream, + SessionJobOutputStreamView::Stderr + ); + assert_eq!(entry.output_gaps[0].omitted_bytes, 6); + let summary = entry.summary.unwrap(); + assert_eq!(summary.request_id, "req-1"); + assert_eq!( + summary.exit, + Some(SessionJobExitView::Signal { + signal: "SIGSEGV".to_owned(), + core_dumped: true + }) + ); + assert_eq!(summary.stopped_by, Some(SessionJobStoppedByView::Kill)); + + let missing = session_job_read_entry( + handle, + ProtocolJobReadEntry::Error { + job_id: job_id("build"), + error: ProtocolError::not_found("unknown job"), + }, + ); + assert!(missing.summary.is_none()); + assert!(missing.error.is_some()); + } + + #[test] + fn lost_jobs_are_reported_as_failed() { + assert_eq!( + api_record_status(EnvironmentJobStatus::Lost), + SessionJobStatusView::Failed + ); + assert_eq!( + api_record_status(EnvironmentJobStatus::Queued), + SessionJobStatusView::Queued + ); + } + + #[test] + fn client_refusals_keep_their_api_kind() { + let kind = |error: ProtocolError| { + map_environment_client_api_error(ClientError::Protocol(error)).kind + }; + assert_eq!( + kind(ProtocolError::invalid_params("bad")), + AgentApiErrorKind::InvalidRequest + ); + assert_eq!( + kind(ProtocolError::not_found("gone")), + AgentApiErrorKind::NotFound + ); + assert_eq!( + kind(ProtocolError::failed_precondition( + environment_protocol::error::PreconditionReason::JobIdInUse, + "in use" + )), + AgentApiErrorKind::Rejected + ); + assert_eq!( + map_environment_client_api_error(ClientError::DeadlineExceeded { elapsed_ms: 1 }).kind, + AgentApiErrorKind::Internal + ); } } diff --git a/crates/temporal-runtime/src/gateway/service/tests.rs b/crates/temporal-runtime/src/gateway/service/tests.rs index b872f5af7..d7708431a 100644 --- a/crates/temporal-runtime/src/gateway/service/tests.rs +++ b/crates/temporal-runtime/src/gateway/service/tests.rs @@ -2531,6 +2531,7 @@ async fn skill_list_reads_latest_structured_provenance() { environment.availability = EnvironmentSkillAvailability::Stale; environment.warnings = vec!["scan incomplete".into()]; environment.skills.push(EnvironmentSkill { + root: "/skills/system".into(), skill_id: tools::skills::SkillId::new("environment:review"), name: "current".into(), description: "same named skill".into(), diff --git a/crates/temporal-runtime/src/main.rs b/crates/temporal-runtime/src/main.rs index 3250492d4..8723f02cd 100644 --- a/crates/temporal-runtime/src/main.rs +++ b/crates/temporal-runtime/src/main.rs @@ -3,8 +3,8 @@ use std::{env, net::SocketAddr, sync::Arc, time::Duration}; use clap::{Args, Parser, Subcommand}; use temporal_runtime::{ config::{ - DeploymentStores, TaskQueues, cas_sweep_grace_from_env, environment_public_url_from_env, - gateway_auth_mode_from_env, postgres_pool_from_env, task_queues_from_env, + DeploymentStores, TaskQueues, cas_sweep_grace_from_env, gateway_auth_mode_from_env, + postgres_pool_from_env, task_queues_from_env, }, gateway::{ DEFAULT_GATEWAY_BIND, DEFAULT_MAX_REQUEST_BODY_BYTES, DEFAULT_TEMPORAL_NAMESPACE, @@ -554,6 +554,14 @@ async fn run_roles(args: RunArgs) -> anyhow::Result<()> { .await? .with_blob_cache(temporal_runtime::config::blob_cache_from_env()?); let reaper_stores = stores.clone(); + if roles.has(Role::EnvironmentGateway) { + // Daemons sign the gateway origin they dial; refuse to serve + // registration under an origin the gateway cannot name. + temporal_runtime::gateway::http::registration_origin( + args.public_base_url.as_deref(), + args.bind, + )?; + } let public_base_url = args .public_base_url .clone() @@ -676,10 +684,7 @@ async fn run_roles(args: RunArgs) -> anyhow::Result<()> { api: roles.has(Role::Gateway), environment: roles.has(Role::EnvironmentGateway), }; - let gateway_state = Arc::new( - GatewayState::multi(mode, universes, public_base_url) - .with_environment_public_url(environment_public_url_from_env()?), - ); + let gateway_state = Arc::new(GatewayState::multi(mode, universes, public_base_url)); let app = gateway_router(gateway_state, args.max_request_body_bytes, routes); let listener = tokio::net::TcpListener::bind(args.bind).await?; tracing::info!( diff --git a/crates/temporal-runtime/src/universe.rs b/crates/temporal-runtime/src/universe.rs index 274ad807c..31d5bf5a8 100644 --- a/crates/temporal-runtime/src/universe.rs +++ b/crates/temporal-runtime/src/universe.rs @@ -264,20 +264,50 @@ impl UniverseRuntime { /// Run the one active deployment lifecycle reconciler. Provider calls are /// idempotent, so process restart safely resumes any persisted intent. + /// Each tick is one deployment pass: every provider is dialed at most + /// once, and a provider whose connection fails is skipped for the rest + /// of the pass in every universe. Every + /// [`STEADY_OBSERVATION_INTERVAL`](crate::environments::lifecycle::STEADY_OBSERVATION_INTERVAL) + /// a pass also observes steady environments, and each pass first dials + /// providers whose controller facts are not recorded yet. pub async fn run_environment_reconciler(self: Arc) { + use crate::environments::provider_controllers::{ + ProviderFactsRefresh, ProviderPass, WebSocketProviderControllerConnector, + }; let mut interval = tokio::time::interval(std::time::Duration::from_millis(500)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut failures = crate::gateway::ReconcileFailureLog::default(); + let mut facts = ProviderFactsRefresh::default(); + let provider_store = self.stores.store_for(Uuid::nil()); + let mut steady_at: Option = None; loop { interval.tick().await; + let scope = if steady_at.is_none_or(|at| { + at.elapsed() >= crate::environments::lifecycle::STEADY_OBSERVATION_INTERVAL + }) { + steady_at = Some(std::time::Instant::now()); + ::environments::ReconcileScope::WithSteady + } else { + ::environments::ReconcileScope::Converging + }; + let mut pass = ProviderPass::default(); + facts + .run( + &mut pass, + provider_store.as_ref(), + &WebSocketProviderControllerConnector, + ) + .await; let universe_ids = match store_pg::list_universes_with_pending_environments( self.stores.pool(), + scope, ) .await { Ok(ids) => ids, Err(error) => { tracing::warn!(target: "temporal_runtime", %error, "environment reconciler scan failed"); + pass.close().await; continue; } }; @@ -292,13 +322,14 @@ impl UniverseRuntime { match state .api .environment_service() - .reconcile_environment_lifecycle_once() + .reconcile_environment_lifecycle_pass(&mut pass, scope) .await { Ok(_) => failures.succeeded(universe_id), Err(error) => failures.failed(universe_id, &error), } } + pass.close().await; } } @@ -321,6 +352,9 @@ impl UniverseRuntime { continue; } }; + // Providers whose daemons timed out are skipped for the rest of + // the pass, in every universe. + let mut unhealthy = std::collections::HashSet::new(); for universe_id in universe_ids { let state = match self.state_for(universe_id, false).await { Ok(state) => state, @@ -332,7 +366,7 @@ impl UniverseRuntime { match state .api .environment_service() - .reconcile_idle_power_once() + .reconcile_idle_power_pass(&mut unhealthy) .await { Ok(_) => failures.succeeded(universe_id), diff --git a/crates/temporal-runtime/src/worker/activities/environment_jobs.rs b/crates/temporal-runtime/src/worker/activities/environment_jobs.rs index 03272f9c8..01624d55c 100644 --- a/crates/temporal-runtime/src/worker/activities/environment_jobs.rs +++ b/crates/temporal-runtime/src/worker/activities/environment_jobs.rs @@ -1,34 +1,58 @@ -use std::collections::{BTreeMap, BTreeSet}; +//! Activities of `EnvironmentJobWorkflow`. +//! +//! The workflow keeps its own job records and start payload in history; +//! these activities are the only place that maps them to and from the +//! environment protocol's `job/*` wire types. -use environment_client::{EnvironmentClientError, EnvironmentDataClient}; +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, +}; + +use environment_client::{ClientError, DataClient}; use environment_protocol::{ + bytes::ByteChunk, data::{ - handshake::{InitializeParams, InitializedParams}, - jobs::{CancelJobsParams, JobStatus, ReadJobsParams}, + exec::{CommandSpec, ExecInput, ReturnOn}, + job::{ + CancelJobsParams, JobCancelScope, JobDependency, JobDependencyPolicy, JobReadEntry, + JobReadRequest, JobSpec, JobStatus, JobSummary, JobSummaryEntry, ReadJobsParams, + StartJobsParams, + }, }, - error::EnvironmentProtocolErrorCode, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentDataConnection, EnvironmentTransport}, + error::{ProtocolError, ProtocolErrorKind}, + ids::{JobId, JobName, JobRequestId, QueueKey}, + paths::EnvironmentPath, }; -use environments::{EnvironmentId, EnvironmentJobGroupId}; +use environments::{EnvironmentId, EnvironmentJobGroupId, EnvironmentRecord}; use harness::{BlobRef, PromiseSourceCheckResult, storage::BlobStore}; use temporal_workflow::{ - EnvironmentJobCancelActivityRequest, EnvironmentJobPollActivityRequest, - EnvironmentJobPollActivityResult, EnvironmentJobPrepareWorkflowToolRequest, + EnvironmentJobCancelActivityRequest, EnvironmentJobCancelActivityResult, + EnvironmentJobCancelScope, EnvironmentJobCommand, EnvironmentJobDependency, + EnvironmentJobDependencyPolicy, EnvironmentJobId, EnvironmentJobObservation, + EnvironmentJobPollActivityRequest, EnvironmentJobPollActivityResult, + EnvironmentJobPrepareWorkflowToolRequest, EnvironmentJobRecord, EnvironmentJobSpec, EnvironmentJobStartActivityRequest, EnvironmentJobStartActivityResult, - EnvironmentJobStartPayload, EnvironmentJobSubscription, EnvironmentJobWorkflowArgs, - EnvironmentJobWorkflowToolContext, + EnvironmentJobStartPayloadV1, EnvironmentJobStatus, EnvironmentJobSubscription, + EnvironmentJobWorkflowArgs, EnvironmentJobWorkflowToolContext, }; use temporalio_common::error::ApplicationFailure; use temporalio_sdk::activities::ActivityError; use tools::environment::jobs::{ - JOB_RUN_WORKFLOW_SEMANTIC_TYPE, JOB_RUN_WORKFLOW_TOOL_ID, JOB_SUBMIT_WORKFLOW_SEMANTIC_TYPE, - JOB_SUBMIT_WORKFLOW_TOOL_ID, JobHandle, NormalizeJobResultInput, normalize_job_result, + DEFAULT_JOB_OUTPUT_BYTES, JOB_RUN_WORKFLOW_SEMANTIC_TYPE, JOB_RUN_WORKFLOW_TOOL_ID, + JOB_SUBMIT_WORKFLOW_SEMANTIC_TYPE, JOB_SUBMIT_WORKFLOW_TOOL_ID, JobHandle, + NormalizeJobResultInput, job_output_read, model_job_error, normalize_job_result, store_model_job_result, }; use super::{common::activity_error, state::EnvironmentJobActivityDeps}; -const PROMISE_JOB_OUTPUT_BYTES: usize = 16 * 1024; +/// `User-Agent` of the job activities' worker-route upgrades; `clientInfo` +/// is always the runtime build. +const JOB_ACTIVITY_USER_AGENT: &str = "lightspeed-environment-job-workflow"; + +/// Deadline of every data-plane call these background activities make. +const JOB_REQUEST_TIMEOUT: Duration = Duration::from_secs(60); #[derive(Clone, Copy)] enum WorkflowJobToolKind { @@ -49,11 +73,7 @@ pub(super) async fn prepare_workflow_tool( deps: Option<&EnvironmentJobActivityDeps>, request: EnvironmentJobPrepareWorkflowToolRequest, ) -> Result { - let deps = deps.ok_or_else(|| { - activity_error(anyhow::anyhow!( - "environment job activities are not configured" - )) - })?; + let deps = configured(deps)?; let start = request.start; let expected_holder = temporal_workflow::compose_workflow_id(start.universe_id, &start.invocation.session_id); @@ -112,16 +132,16 @@ pub(super) async fn prepare_workflow_tool( } let environment_id = EnvironmentId::try_new(execution_context.environment_id).map_err(activity_error)?; + let working_directory = execution_context.working_directory.as_deref(); // The executor admitted this call against the session's attachment // grant; the context names the machine it admitted. - let request_id = format!( - "jobreq:{}:{}:{}:{}", + let request_id = workflow_tool_request_id( start.invocation.run_id.as_u64(), start.invocation.turn_id.as_u64(), start.invocation.tool_batch_id.as_u64(), - start.invocation.tool_call_id.as_str() + start.invocation.tool_call_id.as_str(), ); - let mut jobs = match tool_kind { + let wire_jobs = match tool_kind { WorkflowJobToolKind::Submit => { let args: tools::environment::jobs::JobSubmitArgs = serde_json::from_slice(&arguments).map_err(activity_error)?; @@ -134,7 +154,7 @@ pub(super) async fn prepare_workflow_tool( .into_iter() .map(|spec| { let job_id = spec.job_id.clone(); - spec.into_protocol_spec(job_id) + spec.into_job_spec(job_id, working_directory) .map_err(|error| activity_error(anyhow::anyhow!(error.to_string()))) }) .collect::, _>>()? @@ -144,36 +164,17 @@ pub(super) async fn prepare_workflow_tool( serde_json::from_slice(&arguments).map_err(activity_error)?; let job_id = derived_job_run_id(start.invocation.invocation_id.as_str()); vec![ - args.into_protocol_spec(job_id) + args.into_job_spec(job_id, working_directory) .map_err(|error| activity_error(anyhow::anyhow!(error.to_string())))?, ] } }; - if let Some(cwd) = &execution_context.working_directory { - for job in &mut jobs { - let path = tools::environment::sources::absolute( - std::path::Path::new(cwd), - job.cwd.as_ref().map(|path| path.as_str()).unwrap_or("."), - ) - .map_err(|error| activity_error(anyhow::anyhow!(error)))?; - job.cwd = Some( - environment_protocol::shared::EnvironmentPath::new(path.to_string_lossy()) - .map_err(activity_error)?, - ); - } - } - let params = environment_protocol::data::jobs::StartJobsParams { - namespace: environment_id.as_str().to_owned(), - request_id, - jobs, - }; - let request_fingerprint = - BlobRef::from_bytes(&serde_json::to_vec(¶ms).map_err(activity_error)?); - let job_group_id = derived_workflow_tool_job_group_id( - &environment_id, - ¶ms.request_id, - request_fingerprint.as_str(), - ); + let jobs = wire_jobs + .into_iter() + .map(workflow_job_spec) + .collect::, _>>() + .map_err(|error| activity_error(anyhow::anyhow!(error)))?; + let payload = EnvironmentJobStartPayloadV1::new(request_id.into_string(), jobs); let completion_promises = start .invocation .completion_promises @@ -186,12 +187,12 @@ pub(super) async fn prepare_workflow_tool( })?; let subscriptions = match tool_kind { WorkflowJobToolKind::Submit => { - if completion_promises.len() != params.jobs.len() { + if completion_promises.len() != payload.jobs.len() { return Err(activity_error(anyhow::anyhow!( "job_submit completion promise count does not match job count" ))); } - params + payload .jobs .iter() .map(|job| { @@ -232,16 +233,16 @@ pub(super) async fn prepare_workflow_tool( holder_workflow_id: start.holder_workflow_id.clone(), promise_id: promise_id.as_str().to_owned(), completion_key: harness::REPLY_COMPLETION_KEY.to_owned(), - job_id: params.jobs[0].job_id.clone(), + job_id: payload.jobs[0].job_id.clone(), notified: false, }] } }; - let job_ids = params.jobs.iter().map(|job| job.job_id.clone()).collect(); - let payload = EnvironmentJobStartPayload { request: params }; + let encoded = payload.encode().map_err(activity_error)?; + let job_group_id = derived_job_group_id(&environment_id, &BlobRef::from_bytes(&encoded)); let request_ref = deps .blobs - .put_bytes(serde_json::to_vec(&payload).map_err(activity_error)?) + .put_bytes(encoded) .await .map_err(activity_error)?; Ok(EnvironmentJobWorkflowArgs { @@ -252,7 +253,7 @@ pub(super) async fn prepare_workflow_tool( job_group_id: job_group_id.as_str().to_owned(), request_ref, }, - job_ids, + job_ids: payload.job_ids(), subscriptions, started: false, jobs: Vec::new(), @@ -266,67 +267,238 @@ pub(super) async fn prepare_workflow_tool( }) } -fn derived_job_run_id(invocation_id: &str) -> environment_protocol::shared::JobId { +/// The start's idempotency key: one per model tool call, so a retried call +/// replays its jobs and a new call reusing a job id is refused. +fn workflow_tool_request_id(run: u64, turn: u64, batch: u64, call_id: &str) -> JobRequestId { + JobRequestId::new(format!("jobreq:{run}:{turn}:{batch}:{call_id}")).unwrap_or_else(|_| { + // A provider call id outside the id grammar is hashed instead. + let hash = BlobRef::from_bytes(format!("{run}:{turn}:{batch}:{call_id}").as_bytes()); + JobRequestId::new(format!("jobreq-{}", &hash.as_str()[7..39])) + .expect("hex request id fits the id grammar") + }) +} + +fn derived_job_run_id(invocation_id: &str) -> JobId { let hash = BlobRef::from_bytes(invocation_id.as_bytes()); - environment_protocol::shared::JobId::new(format!("job-{}", &hash.as_str()[7..31])) + JobId::new(format!("job-{}", &hash.as_str()[7..31])).expect("hex job id fits the id grammar") } -fn derived_workflow_tool_job_group_id( +/// A group is named by its environment and the content address of its +/// encoded start payload, which is also the payload's CAS `request_ref`. +fn derived_job_group_id( environment_id: &EnvironmentId, - request_id: &str, - request_fingerprint: &str, + payload_ref: &BlobRef, ) -> EnvironmentJobGroupId { - let hash = BlobRef::from_bytes( - format!("{environment_id}:{request_id}:{request_fingerprint}").as_bytes(), - ); + let hash = BlobRef::from_bytes(format!("{environment_id}:{payload_ref}").as_bytes()); EnvironmentJobGroupId::new(format!("ejg_{}", &hash.as_str()[7..31])) } +/// The workflow's record of a wire job spec built by the model-facing +/// arguments. The spec never carries secrets here: the start activity +/// resolves credential-bound variables. +fn workflow_job_spec(spec: JobSpec) -> Result { + let JobSpec { + job_id, + name, + command, + depends_on, + dependency_policy, + queue_key, + } = spec; + let CommandSpec { + argv, + cwd, + env, + secret_env, + input, + timeout_ms, + } = command; + if !secret_env.is_empty() { + return Err(format!( + "job {job_id} carries secret variables before credential resolution" + )); + } + let stdin = match input { + ExecInput::None {} => None, + ExecInput::Bytes { data } => Some( + String::from_utf8(data.into_inner()) + .map_err(|_| format!("job {job_id} stdin is not UTF-8"))?, + ), + ExecInput::Pipe {} => return Err(format!("job {job_id} cannot read a pipe")), + }; + Ok(EnvironmentJobSpec { + job_id: EnvironmentJobId::new(job_id.into_string()), + name: name.map(JobName::into_string), + command: EnvironmentJobCommand { + argv, + cwd: cwd.map(|cwd| cwd.as_str().to_owned()), + env, + stdin, + timeout_ms, + }, + depends_on: depends_on + .into_iter() + .map(|dependency| match dependency { + JobDependency::JobId { job_id } => EnvironmentJobDependency::JobId { + job_id: EnvironmentJobId::new(job_id.into_string()), + }, + JobDependency::Name { name } => EnvironmentJobDependency::Name { + name: name.into_string(), + }, + }) + .collect(), + dependency_policy: match dependency_policy { + JobDependencyPolicy::AllSucceeded => EnvironmentJobDependencyPolicy::AllSucceeded, + JobDependencyPolicy::AllTerminal => EnvironmentJobDependencyPolicy::AllTerminal, + }, + queue_key: queue_key.map(QueueKey::into_string), + }) +} + +/// The wire spec of a stored job, without its secret variables. +fn wire_job_spec(spec: EnvironmentJobSpec) -> Result { + let EnvironmentJobSpec { + job_id, + name, + command, + depends_on, + dependency_policy, + queue_key, + } = spec; + let job_id = wire_job_id(&job_id)?; + Ok(JobSpec { + name: name + .map(|name| JobName::new(name).map_err(|error| format!("job {job_id} name: {error}"))) + .transpose()?, + command: CommandSpec { + argv: command.argv, + cwd: command + .cwd + .map(|cwd| { + EnvironmentPath::new(cwd).map_err(|error| format!("job {job_id} cwd: {error}")) + }) + .transpose()?, + env: command.env, + secret_env: BTreeMap::new(), + input: match command.stdin { + Some(stdin) => ExecInput::Bytes { + data: ByteChunk::from(stdin.into_bytes()), + }, + None => ExecInput::None {}, + }, + timeout_ms: command.timeout_ms, + }, + depends_on: depends_on + .into_iter() + .map(|dependency| match dependency { + EnvironmentJobDependency::JobId { job_id } => { + wire_job_id(&job_id).map(|job_id| JobDependency::JobId { job_id }) + } + EnvironmentJobDependency::Name { name } => JobName::new(name) + .map(|name| JobDependency::Name { name }) + .map_err(|error| format!("job dependency name: {error}")), + }) + .collect::, _>>()?, + dependency_policy: match dependency_policy { + EnvironmentJobDependencyPolicy::AllSucceeded => JobDependencyPolicy::AllSucceeded, + EnvironmentJobDependencyPolicy::AllTerminal => JobDependencyPolicy::AllTerminal, + }, + queue_key: queue_key + .map(|key| { + QueueKey::new(key).map_err(|error| format!("job {job_id} queue key: {error}")) + }) + .transpose()?, + job_id, + }) +} + +fn wire_job_id(job_id: &EnvironmentJobId) -> Result { + JobId::new(job_id.as_str()).map_err(|error| format!("job id {job_id}: {error}")) +} + +/// The workflow's record of a daemon summary. +fn job_record(summary: &JobSummary) -> EnvironmentJobRecord { + EnvironmentJobRecord { + job_id: EnvironmentJobId::new(summary.job_id.as_str()), + name: summary.name.as_ref().map(|name| name.as_str().to_owned()), + status: job_status(summary.status), + depends_on: summary + .depends_on + .iter() + .map(|job_id| EnvironmentJobId::new(job_id.as_str())) + .collect(), + queue_key: summary + .queue_key + .as_ref() + .map(|key| key.as_str().to_owned()), + } +} + +fn job_status(status: JobStatus) -> EnvironmentJobStatus { + match status { + JobStatus::Queued => EnvironmentJobStatus::Queued, + JobStatus::Running => EnvironmentJobStatus::Running, + JobStatus::Succeeded => EnvironmentJobStatus::Succeeded, + JobStatus::Failed => EnvironmentJobStatus::Failed, + JobStatus::TimedOut => EnvironmentJobStatus::TimedOut, + JobStatus::Cancelled => EnvironmentJobStatus::Cancelled, + JobStatus::DependencyFailed => EnvironmentJobStatus::DependencyFailed, + JobStatus::Interrupted => EnvironmentJobStatus::Interrupted, + } +} + +/// A per-entry error: `notFound` is the workflow's `lost` outcome; any +/// other entry error means the daemon broke the contract. +fn entry_error_observation( + job_id: &JobId, + error: ProtocolError, +) -> Result { + if error.kind() == ProtocolErrorKind::NotFound { + Ok(EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new(job_id.as_str()), + }) + } else { + Err(activity_error(anyhow::anyhow!( + "environment answered job {job_id} with {error}" + ))) + } +} + pub(super) async fn start( deps: Option<&EnvironmentJobActivityDeps>, request: EnvironmentJobStartActivityRequest, ) -> Result { - let deps = deps.ok_or_else(|| { - activity_error(anyhow::anyhow!( - "environment job activities are not configured" - )) - })?; + let deps = configured(deps)?; let environment_id = EnvironmentId::try_new(request.environment_id.clone()).map_err(activity_error)?; - let mut payload: EnvironmentJobStartPayload = serde_json::from_slice( + let payload = EnvironmentJobStartPayloadV1::decode( &deps .blobs .read_bytes(&request.request_ref) .await .map_err(activity_error)?, ) - .map_err(activity_error)?; - if payload.request.namespace != environment_id.as_str() { - return Err(activity_error(anyhow::anyhow!( - "environment job start namespace does not match instance {environment_id}" - ))); - } - - for job in &mut payload.request.jobs { + .map_err(non_retryable_activity_error)?; + let request_id = JobRequestId::new(payload.request_id.clone()).map_err(|error| { + non_retryable_activity_error(anyhow::anyhow!("job request id: {error}")) + })?; + let requested_ids = payload.job_ids().into_iter().collect::>(); + let mut jobs = Vec::with_capacity(payload.jobs.len()); + for spec in payload.jobs { + let mut job = wire_job_spec(spec) + .map_err(|error| non_retryable_activity_error(anyhow::anyhow!(error)))?; let secret_env = deps .credentials - .resolve_secret_env(&environment_id, &job.env) + .resolve_secret_env(&environment_id, &job.command.env) .await .map_err(activity_error)?; - job.secret_env.extend(secret_env); + job.command.secret_env.extend(secret_env); + jobs.push(job); } - start_on_provider(deps, &environment_id, &payload).await -} - -async fn start_on_provider( - deps: &EnvironmentJobActivityDeps, - environment_id: &EnvironmentId, - payload: &EnvironmentJobStartPayload, -) -> Result { let instance = deps .environments - .read_environment(environment_id) + .read_environment(&environment_id) .await .map_err(activity_error)?; if matches!( @@ -337,131 +509,152 @@ async fn start_on_provider( "cannot start jobs on closing environment instance {environment_id}" ))); } - let gateway = deps.gateway.as_ref().ok_or_else(|| { - non_retryable_activity_error(anyhow::anyhow!( - "environment gateway is not configured on this worker" - )) - })?; - let connection = gateway.connection_for(deps.universe_id, &instance); - let (mut client, capabilities) = initialized_client(&connection, gateway).await?; - if !capabilities.job_start { - return Err(non_retryable_activity_error(anyhow::anyhow!( - "environment does not support durable job start: {environment_id}" - ))); - } + let client = job_client(deps, &instance).await?; let response = client - .start_jobs(&payload.request) - .await - .map_err(start_environment_activity_error)?; - let requested_ids = payload - .request - .jobs + .start_jobs(&StartJobsParams { request_id, jobs }) + .await; + let _ = client.close().await; + let response = response.map_err(client_activity_error)?; + let records = response.jobs.iter().map(job_record).collect::>(); + let returned_ids = records .iter() - .map(|job| job.job_id.clone()) - .collect::>(); - let returned_ids = response - .jobs - .iter() - .map(|job| job.job_id.clone()) + .map(|record| record.job_id.clone()) .collect::>(); if requested_ids != returned_ids { return Err(non_retryable_activity_error(anyhow::anyhow!( - "environment provider start response job ids do not match the request" + "environment start response job ids do not match the request" ))); } - Ok(EnvironmentJobStartActivityResult { - jobs: response.jobs, - }) -} - -fn start_environment_activity_error(error: EnvironmentClientError) -> ActivityError { - match &error { - EnvironmentClientError::Protocol(error) - if matches!( - error.code, - EnvironmentProtocolErrorCode::InvalidRequest - | EnvironmentProtocolErrorCode::Unauthorized - | EnvironmentProtocolErrorCode::Forbidden - | EnvironmentProtocolErrorCode::NotFound - | EnvironmentProtocolErrorCode::Conflict - | EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable - ) => - { - non_retryable_activity_error(anyhow::anyhow!(error.message.clone())) - } - _ => activity_error(error), - } -} - -fn non_retryable_activity_error(error: impl Into) -> ActivityError { - ActivityError::application(ApplicationFailure::non_retryable(error.into())) + Ok(EnvironmentJobStartActivityResult { jobs: records }) } pub(super) async fn poll( deps: Option<&EnvironmentJobActivityDeps>, request: EnvironmentJobPollActivityRequest, ) -> Result { - let deps = deps.ok_or_else(|| { - activity_error(anyhow::anyhow!( - "environment job activities are not configured" - )) - })?; + let deps = configured(deps)?; let environment_id = EnvironmentId::try_new(request.environment_id).map_err(activity_error)?; + let job_ids = request + .job_ids + .iter() + .map(wire_job_id) + .collect::, _>>() + .map_err(|error| non_retryable_activity_error(anyhow::anyhow!(error)))?; + if job_ids.is_empty() { + return Ok(EnvironmentJobPollActivityResult { + jobs: Vec::new(), + resolutions: BTreeMap::new(), + }); + } let instance = deps .environments .read_environment(&environment_id) .await .map_err(activity_error)?; - let gateway = deps.gateway.as_ref().ok_or_else(|| { - non_retryable_activity_error(anyhow::anyhow!( - "environment gateway is not configured on this worker" - )) - })?; - let connection = gateway.connection_for(deps.universe_id, &instance); - let (mut client, _) = initialized_client(&connection, gateway).await?; - let requested_job_ids = request.job_ids.iter().cloned().collect::>(); - let response = client - .read_jobs(&ReadJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: request.job_ids, - after_seq: None, - max_bytes: Some(PROMISE_JOB_OUTPUT_BYTES), - include_artifacts: true, - wait_ms: None, - }) - .await - .map_err(activity_error)?; - let mut jobs = Vec::with_capacity(response.jobs.len()); + let client = job_client(deps, &instance).await?; + let result = poll_jobs(deps, &client, &environment_id, &job_ids).await; + let _ = client.close().await; + result +} + +async fn poll_jobs( + deps: &EnvironmentJobActivityDeps, + client: &DataClient, + environment_id: &EnvironmentId, + job_ids: &[JobId], +) -> Result { + let limits = client.initialize_response().limits.job.clone(); + let per_request = (limits.max_request_jobs as usize).max(1); + let mut jobs = Vec::with_capacity(job_ids.len()); + let mut terminal = Vec::new(); + // Summaries first, without output: most polls find running jobs. + for chunk in job_ids.chunks(per_request) { + let response = client + .read_jobs(&ReadJobsParams { + jobs: chunk + .iter() + .map(|job_id| JobReadRequest { + job_id: job_id.clone(), + after: None, + tail_bytes: None, + max_bytes: None, + }) + .collect(), + max_bytes: 0, + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + }) + .await + .map_err(client_activity_error)?; + for (job_id, entry) in chunk.iter().zip(response.jobs) { + match entry { + JobReadEntry::Job { summary, .. } => { + let record = job_record(&summary); + if record.status.is_terminal() { + terminal.push((job_id.clone(), record.status)); + } + jobs.push(EnvironmentJobObservation::Job { record }); + } + JobReadEntry::Error { job_id, error } => { + jobs.push(entry_error_observation(&job_id, error)?); + } + } + } + } + let mut resolutions = BTreeMap::new(); - for result in response.jobs { - let summary = result.summary.clone(); - if summary.status.is_terminal() { - let error = (summary.status != JobStatus::Succeeded).then(|| { - summary.failure.clone().unwrap_or_else(|| { - format!( - "environment job {} ended as {:?}", - summary.job_id, summary.status - ) - }) - }); - let normalized = normalize_job_result( - deps.blobs.as_ref(), - NormalizeJobResultInput { - handle: Some(JobHandle { - environment_id: environment_id.as_str().to_owned(), - job_id: summary.job_id.clone(), - }), - summary: Some(result.summary), - output_chunks: result.output_chunks, - output_next_seq: result.output_next_seq, - artifacts: result.artifacts, - error, - output_bytes: Some(PROMISE_JOB_OUTPUT_BYTES), + for observation in &jobs { + if let EnvironmentJobObservation::NotFound { job_id } = observation { + let handle = job_handle(environment_id, job_id)?; + let result = model_job_error( + Some(handle), + format!("environment no longer knows job {job_id}"), + ); + let result_ref = + store_model_job_result(deps.blobs.as_ref(), deps.blob_graph.as_deref(), &result) + .await + .map_err(activity_error)?; + resolutions.insert( + job_id.as_str().to_owned(), + PromiseSourceCheckResult::Failed { + error_ref: Some(result_ref), }, - ) + ); + } + } + + // Each terminal job's result shows the head and the tail of its output. + let per_read = (per_request / 2).max(1); + for chunk in terminal.chunks(per_read) { + let ids = chunk + .iter() + .map(|(job_id, _)| job_id.clone()) + .collect::>(); + let output_bytes = DEFAULT_JOB_OUTPUT_BYTES.min(limits.max_read_bytes / ids.len() as u64); + let response = client + .read_jobs(&job_output_read(&ids, None, output_bytes)) .await - .map_err(activity_error)?; + .map_err(client_activity_error)?; + let mut entries = response.jobs.into_iter(); + for (job_id, status) in chunk { + let handle = job_handle(environment_id, &EnvironmentJobId::new(job_id.as_str()))?; + let mut input = NormalizeJobResultInput::from_entries( + Some(handle), + entries.by_ref().take(2).collect::>(), + ); + if !status.is_success() && input.error.is_none() { + input.error = Some( + match input.summary.as_ref().and_then(|s| s.error.as_ref()) { + Some(error) => { + format!("environment job {job_id} ended {status:?}: {error}") + } + None => format!("environment job {job_id} ended {status:?}"), + }, + ); + } + let normalized = normalize_job_result(deps.blobs.as_ref(), input) + .await + .map_err(activity_error)?; let result_ref = store_model_job_result( deps.blobs.as_ref(), deps.blob_graph.as_deref(), @@ -469,7 +662,7 @@ pub(super) async fn poll( ) .await .map_err(activity_error)?; - let resolution = if summary.status == JobStatus::Succeeded { + let resolution = if status.is_success() { PromiseSourceCheckResult::Resolved { payload_ref: Some(result_ref), } @@ -478,99 +671,311 @@ pub(super) async fn poll( error_ref: Some(result_ref), } }; - resolutions.insert(summary.job_id.as_str().to_owned(), resolution); + resolutions.insert(job_id.as_str().to_owned(), resolution); } - jobs.push(summary); } - let returned_job_ids = jobs - .iter() - .map(|job| job.job_id.clone()) - .collect::>(); - let terminal = returned_job_ids == requested_job_ids - && !jobs.is_empty() - && jobs.iter().all(|job| job.status.is_terminal()); - Ok(EnvironmentJobPollActivityResult { - jobs, - resolutions, - terminal, + Ok(EnvironmentJobPollActivityResult { jobs, resolutions }) +} + +fn job_handle( + environment_id: &EnvironmentId, + job_id: &EnvironmentJobId, +) -> Result { + Ok(JobHandle { + environment_id: environment_id.as_str().to_owned(), + job_id: wire_job_id(job_id) + .map_err(|error| non_retryable_activity_error(anyhow::anyhow!(error)))?, }) } pub(super) async fn cancel( deps: Option<&EnvironmentJobActivityDeps>, request: EnvironmentJobCancelActivityRequest, -) -> Result, ActivityError> { - let deps = deps.ok_or_else(|| { - activity_error(anyhow::anyhow!( - "environment job activities are not configured" - )) - })?; +) -> Result { + let deps = configured(deps)?; let environment_id = EnvironmentId::try_new(request.environment_id).map_err(activity_error)?; + let job_ids = request + .jobs + .iter() + .map(wire_job_id) + .collect::, _>>() + .map_err(|error| non_retryable_activity_error(anyhow::anyhow!(error)))?; + if job_ids.is_empty() { + return Ok(EnvironmentJobCancelActivityResult { jobs: Vec::new() }); + } let instance = deps .environments .read_environment(&environment_id) .await .map_err(activity_error)?; + let client = job_client(deps, &instance).await?; + let result = cancel_jobs(&client, &job_ids, request.scope, request.force).await; + let _ = client.close().await; + result +} + +async fn cancel_jobs( + client: &DataClient, + job_ids: &[JobId], + scope: EnvironmentJobCancelScope, + force: bool, +) -> Result { + let per_request = (client.initialize_response().limits.job.max_request_jobs as usize).max(1); + let scope = match scope { + EnvironmentJobCancelScope::Job => JobCancelScope::Job, + EnvironmentJobCancelScope::Dependents => JobCancelScope::Dependents, + }; + let mut jobs = Vec::new(); + for chunk in job_ids.chunks(per_request) { + let response = client + .cancel_jobs(&CancelJobsParams { + job_ids: chunk.to_vec(), + scope, + force, + }) + .await + .map_err(client_activity_error)?; + for entry in response.jobs { + jobs.push(match entry { + JobSummaryEntry::Job { summary } => EnvironmentJobObservation::Job { + record: job_record(&summary), + }, + JobSummaryEntry::Error { job_id, error } => { + entry_error_observation(&job_id, error)? + } + }); + } + } + Ok(EnvironmentJobCancelActivityResult { jobs }) +} + +fn configured( + deps: Option<&EnvironmentJobActivityDeps>, +) -> Result<&EnvironmentJobActivityDeps, ActivityError> { + deps.ok_or_else(|| { + activity_error(anyhow::anyhow!( + "environment job activities are not configured" + )) + }) +} + +/// An initialized data client for the environment's current incarnation; +/// the daemon must serve the `job` family. +async fn job_client( + deps: &EnvironmentJobActivityDeps, + instance: &EnvironmentRecord, +) -> Result { let gateway = deps.gateway.as_ref().ok_or_else(|| { non_retryable_activity_error(anyhow::anyhow!( "environment gateway is not configured on this worker" )) })?; - let connection = gateway.connection_for(deps.universe_id, &instance); - let (mut client, _) = initialized_client(&connection, gateway).await?; - let response = client - .cancel_jobs(&CancelJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: request.jobs, - scope: request.scope, - force: request.force, - }) - .await - .map_err(activity_error)?; - Ok(response.jobs) + let endpoint = gateway.endpoint_for(deps.universe_id, instance); + let (client, initialized) = DataClient::connect_initialized( + &endpoint, + gateway + .connect_options(JOB_ACTIVITY_USER_AGENT) + .with_request_timeout(JOB_REQUEST_TIMEOUT), + ) + .await + .map_err(client_activity_error)?; + if initialized.capabilities.job.is_none() { + let _ = client.close().await; + return Err(non_retryable_activity_error(anyhow::anyhow!( + "environment does not serve durable jobs: {}", + instance.environment_id + ))); + } + Ok(client) } -async fn initialized_client( - connection: &EnvironmentDataConnection, - gateway: &crate::environments::gateway::EnvironmentGatewayClientConfig, -) -> Result< - ( - EnvironmentDataClient, - environment_protocol::shared::EnvironmentCapabilities, - ), - ActivityError, -> { - if connection.transport != EnvironmentTransport::WebSocket { - return Err(activity_error(anyhow::anyhow!( - "unsupported environment job transport: {:?}", - connection.transport - ))); +/// Retry what may pass on a later attempt (transport trouble, busy or +/// unavailable daemons, server defects); refusals of the request itself, +/// such as `invalidParams` or `failedPrecondition {jobIdInUse}`, and +/// version or plane mismatches fail the activity for good. +fn client_activity_error(error: ClientError) -> ActivityError { + if is_transient(&error) { + activity_error(error) + } else { + non_retryable_activity_error(error) } - let mut client = EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("lightspeed-environment-job-workflow"), - ) - .await - .map_err(activity_error)?; - let response = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-environment-job-workflow".to_owned(), - scope: connection.scope.clone(), - resume_connection_id: None, - }) - .await - .map_err(activity_error)?; - if response.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(activity_error(anyhow::anyhow!( - "unsupported environment protocol version {}", - response.protocol_version +} + +fn is_transient(error: &ClientError) -> bool { + error.is_retryable() + || matches!( + error, + ClientError::Decode(_) | ClientError::InvalidMessage(_) + ) + || error.protocol_kind() == Some(ProtocolErrorKind::Internal) +} + +fn non_retryable_activity_error(error: impl Into) -> ActivityError { + ActivityError::application(ApplicationFailure::non_retryable(error.into())) +} + +#[cfg(test)] +mod tests { + use environment_protocol::error::PreconditionReason; + + use super::*; + + fn submitted_spec() -> JobSpec { + JobSpec { + job_id: JobId::new("build").unwrap(), + name: Some(JobName::new("build & test").unwrap()), + command: CommandSpec { + argv: vec!["make".to_owned(), "test".to_owned()], + cwd: Some(EnvironmentPath::new("/workspace").unwrap()), + env: BTreeMap::from([("CI".to_owned(), "1".to_owned())]), + secret_env: BTreeMap::new(), + input: ExecInput::Bytes { + data: ByteChunk::from(b"input".to_vec()), + }, + timeout_ms: Some(1_000), + }, + depends_on: vec![ + JobDependency::JobId { + job_id: JobId::new("setup").unwrap(), + }, + JobDependency::Name { + name: JobName::new("lint").unwrap(), + }, + ], + dependency_policy: JobDependencyPolicy::AllTerminal, + queue_key: Some(QueueKey::new("ci").unwrap()), + } + } + + #[test] + fn stored_specs_map_back_onto_the_wire_unchanged() { + let spec = submitted_spec(); + let stored = workflow_job_spec(spec.clone()).unwrap(); + assert_eq!(stored.command.stdin.as_deref(), Some("input")); + assert_eq!(stored.command.cwd.as_deref(), Some("/workspace")); + assert_eq!(wire_job_spec(stored).unwrap(), spec); + } + + #[test] + fn stored_specs_never_hold_secrets_or_pipes() { + let mut secret = submitted_spec(); + secret + .command + .secret_env + .insert("TOKEN".to_owned(), "value".into()); + assert!(workflow_job_spec(secret).is_err()); + let mut piped = submitted_spec(); + piped.command.input = ExecInput::Pipe {}; + assert!(workflow_job_spec(piped).is_err()); + } + + #[test] + fn invalid_stored_ids_are_refused_at_the_wire_boundary() { + let mut stored = workflow_job_spec(submitted_spec()).unwrap(); + stored.job_id = EnvironmentJobId::new("not a job id"); + assert!(wire_job_spec(stored).is_err()); + } + + #[test] + fn request_ids_fit_the_id_grammar_for_any_call_id() { + assert_eq!( + workflow_tool_request_id(1, 2, 3, "toolu_01").as_str(), + "jobreq:1:2:3:toolu_01" + ); + let hashed = workflow_tool_request_id(1, 2, 3, "call id/with spaces"); + assert!(hashed.as_str().starts_with("jobreq-")); + assert_eq!( + hashed, + workflow_tool_request_id(1, 2, 3, "call id/with spaces") + ); + assert!( + derived_job_run_id("invocation") + .as_str() + .starts_with("job-") + ); + } + + #[test] + fn job_group_ids_follow_the_encoded_payload() { + let environment_id = EnvironmentId::new("environment_1"); + let payload = EnvironmentJobStartPayloadV1::new( + "jobreq:1:1:1:call", + vec![workflow_job_spec(submitted_spec()).unwrap()], + ); + let encoded = payload.encode().unwrap(); + let first = derived_job_group_id(&environment_id, &BlobRef::from_bytes(&encoded)); + assert_eq!( + first, + derived_job_group_id(&environment_id, &BlobRef::from_bytes(&encoded)) + ); + let other = EnvironmentJobStartPayloadV1::new("jobreq:1:1:1:other", payload.jobs.clone()); + assert_ne!( + first, + derived_job_group_id( + &environment_id, + &BlobRef::from_bytes(&other.encode().unwrap()) + ) + ); + } + + #[test] + fn not_found_entries_become_lost_observations() { + let job_id = JobId::new("build").unwrap(); + assert_eq!( + entry_error_observation(&job_id, ProtocolError::not_found("unknown job")).unwrap(), + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("build") + } + ); + assert!(entry_error_observation(&job_id, ProtocolError::internal("broken")).is_err()); + } + + #[test] + fn refusals_are_terminal_and_transport_trouble_is_retried() { + let retryable = |error: ClientError| is_transient(&error); + assert!(!retryable(ClientError::Protocol( + ProtocolError::failed_precondition(PreconditionReason::JobIdInUse, "job id in use") + ))); + assert!(!retryable(ClientError::Protocol( + ProtocolError::invalid_params("bad") ))); + assert!(retryable(ClientError::Protocol( + ProtocolError::unavailable("daemon restarting") + ))); + assert!(retryable(ClientError::Protocol(ProtocolError::internal( + "bug" + )))); + assert!(retryable(ClientError::Connect { + endpoint: "ws://gateway.test".to_owned(), + message: "refused".to_owned(), + })); + assert!(retryable(ClientError::DeadlineExceeded { + elapsed_ms: 60_000 + })); + } + + #[test] + fn daemon_summaries_map_onto_workflow_records() { + let summary: JobSummary = serde_json::from_value(serde_json::json!({ + "jobId": "build", + "requestId": "jobreq:1:1:1:call", + "name": "build", + "status": "timedOut", + "dependsOn": ["setup"], + "queueKey": "ci", + "createdAtMs": 1, + "stoppedBy": "timeout", + })) + .unwrap(); + assert_eq!( + job_record(&summary), + EnvironmentJobRecord { + job_id: EnvironmentJobId::new("build"), + name: Some("build".to_owned()), + status: EnvironmentJobStatus::TimedOut, + depends_on: vec![EnvironmentJobId::new("setup")], + queue_key: Some("ci".to_owned()), + } + ); } - let capabilities = response.capabilities; - client - .initialized(&InitializedParams {}) - .await - .map_err(activity_error)?; - Ok((client, capabilities)) } diff --git a/crates/temporal-runtime/src/worker/activities/mod.rs b/crates/temporal-runtime/src/worker/activities/mod.rs index 659f12b0b..ebd67a6c8 100644 --- a/crates/temporal-runtime/src/worker/activities/mod.rs +++ b/crates/temporal-runtime/src/worker/activities/mod.rs @@ -708,7 +708,7 @@ impl WorkerActivities { self: Arc, _ctx: ActivityContext, request: EnvironmentJobCancelActivityRequest, - ) -> Result, ActivityError> { + ) -> Result { let state = self.state_for_universe(request.universe_id).await?; environment_jobs::cancel(state.environment_jobs(), request).await } diff --git a/crates/temporal-runtime/src/worker/session_tools.rs b/crates/temporal-runtime/src/worker/session_tools.rs index 0b0369a12..0c9132aa5 100644 --- a/crates/temporal-runtime/src/worker/session_tools.rs +++ b/crates/temporal-runtime/src/worker/session_tools.rs @@ -1,13 +1,10 @@ use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; -use environment_client::{EnvironmentClientError, EnvironmentDataClient, WebSocketConnectOptions}; +use environment_client::{ClientError, DataClient}; use environment_protocol::{ - data::{ - handshake::{InitializeParams, InitializedParams}, - jobs::{JobReadResult as ProtocolJobReadResult, ReadJobsParams}, - }, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentDataConnection, EnvironmentTransport}, + data::fs::{FileKind, GetMetadataParams}, + paths::EnvironmentPath, }; use environments::{EnvironmentId, EnvironmentRecord, EnvironmentRegistryError, EnvironmentStore}; use harness::PromiseIdAllocator; @@ -32,10 +29,10 @@ use tools::{ is_environment_selection_tool, }, environment::jobs::{ - JOB_RUN_WORKFLOW_SEMANTIC_TYPE, JOB_RUN_WORKFLOW_TOOL_ID, + DEFAULT_JOB_OUTPUT_BYTES, JOB_RUN_WORKFLOW_SEMANTIC_TYPE, JOB_RUN_WORKFLOW_TOOL_ID, JOB_SUBMIT_WORKFLOW_SEMANTIC_TYPE, JOB_SUBMIT_WORKFLOW_TOOL_ID, JobHandle, JobHandleArg, JobReadArgs, JobSubmitExecutionContextV1, ModelJobResult, ModelJobResultSet, - NormalizeJobResultInput, normalize_job_result, + NormalizeJobResultInput, job_output_read, model_job_error, normalize_job_result, }, environment_protocol::RemoteEnvironmentConnection, fs::{AttachedVfsFileSystem, FsPath, FsToolContext}, @@ -417,8 +414,7 @@ impl SessionTools { environments, args.jobs, args.output_bytes, - args.after_seq, - args.include_artifacts, + args.after, ) .await?; self.succeeded_job_read_result(call, result.entries).await @@ -442,10 +438,12 @@ impl SessionTools { policy: Option<&harness::EnvironmentsFeature>, environments: &SessionEnvironmentManager, handles: Vec, - output_bytes: Option, - after_seq: Option, - include_artifacts: bool, + output_bytes: Option, + after: Option, ) -> Result { + let output_bytes = output_bytes + .unwrap_or(DEFAULT_JOB_OUTPUT_BYTES) + .min(ToolLimits::default().max_model_visible_output_bytes); let mut entries = Vec::with_capacity(handles.len()); for handle in handles { let resolved = match resolve_job_handle_arg(active_environment_id, policy, handle) { @@ -517,26 +515,26 @@ impl SessionTools { } continue; }; + // The head and the tail of the output, or a page from `after`, + // within the daemon's read ceiling. + let budget = jobs.limits().map_or(output_bytes, |limits| { + output_bytes.min(limits.max_read_bytes) + }); match jobs - .read_jobs(ReadJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: vec![resolved.job_id.clone()], - after_seq, - max_bytes: output_bytes, - include_artifacts, - wait_ms: None, - }) + .read_jobs(job_output_read( + std::slice::from_ref(&resolved.job_id), + after, + budget, + )) .await { Ok(response) => { + let input = + NormalizeJobResultInput::from_entries(Some(resolved), response.jobs); entries.push( - job_read_entry_from_response( - self.blobs.as_ref(), - resolved, - response.jobs.into_iter().next(), - output_bytes, - ) - .await?, + normalize_job_result(self.blobs.as_ref(), input) + .await + .map_err(map_blob_error)?, ); } Err(error) => { @@ -1276,66 +1274,28 @@ impl SessionTools { .environment_gateway .as_ref() .ok_or_else(|| io_error("environment gateway is not configured on this worker"))?; - let connection = gateway.connection_for( + let endpoint = gateway.endpoint_for( self.environment_resolver .as_ref() .map(|resolver| resolver.universe_id()) .unwrap_or_default(), &resource, ); - let mut client = connect_environment_data_client( - &connection, + let (client, _) = DataClient::connect_initialized( + &endpoint, gateway.connect_options("lightspeed-temporal-runtime"), ) - .await?; - let response = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-temporal-runtime".to_owned(), - scope: connection.scope.clone(), - resume_connection_id: None, - }) - .await - .map_err(map_environment_client_error)?; - if response.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(io_error(format!( - "unsupported environment data protocol version {}; expected {CURRENT_PROTOCOL_VERSION}", - response.protocol_version - ))); - } - client - .initialized(&InitializedParams {}) - .await - .map_err(map_environment_client_error)?; - let cwd = if response.capabilities.filesystem_read { - match crate::environments::sources::working_directory( - &mut client, - working_directory, - response.default_cwd.as_deref(), - ) - .await - { - Ok(cwd) => cwd, - Err(error) => { - let _ = client.close().await; - return Err(io_error(error)); - } + .await + .map_err(map_environment_client_error)?; + let cwd = match environment_working_directory(&client, working_directory).await { + Ok(cwd) => cwd, + Err(error) => { + let _ = client.close().await; + return Err(io_error(error)); } - } else { - let cwd = working_directory - .or(response.default_cwd.as_deref()) - .ok_or_else(|| io_error("environment has no working directory"))?; - tools::environment::sources::absolute(std::path::Path::new("/"), cwd) - .map_err(io_error)? - .to_string_lossy() - .into_owned() }; - let cwd = Some(FsPath::new(cwd).map_err(io_error)?); - - let mut remote_connection = RemoteEnvironmentConnection::new(client, response.capabilities); - if let Some(cwd) = cwd { - remote_connection = remote_connection.with_cwd(cwd); - } + let remote_connection = RemoteEnvironmentConnection::new(client) + .with_cwd(FsPath::new(cwd.as_str()).map_err(io_error)?); let (_fs_context, mut environment_context) = remote_connection.clone().into_contexts(self.blobs.clone()); if let Some(credentials) = &self.environment_credentials { @@ -1571,73 +1531,46 @@ fn supplied_environment_policy( .ok_or_else(|| io_error("environment grant is missing from the batch request")) } -async fn job_read_entry_from_response( - blobs: &dyn BlobStore, - handle: JobHandle, - response: Option, - output_bytes: Option, -) -> Result { - match response { - Some(response) => normalize_job_result( - blobs, - NormalizeJobResultInput { - handle: Some(handle), - summary: Some(response.summary), - output_chunks: response.output_chunks, - output_next_seq: response.output_next_seq, - artifacts: response.artifacts, - output_bytes, - ..Default::default() - }, - ) - .await - .map_err(map_blob_error), - None => Ok(model_job_error( - Some(handle), - "provider returned no job result".to_owned(), - )), - } -} - -fn model_job_error(handle: Option, error: String) -> ModelJobResult { - ModelJobResult { - handle: handle.map(|mut handle| { - handle.environment_id = - tools::environment::handles::environment_handle(&handle.environment_id); - handle - }), - summary: None, - output: Vec::new(), - output_next_seq: 0, - truncated: false, - artifacts: Vec::new(), - error: Some(error), - } -} - -async fn connect_environment_data_client( - connection: &EnvironmentDataConnection, - options: WebSocketConnectOptions, -) -> Result, CoreAgentIoError> { - match &connection.transport { - EnvironmentTransport::WebSocket => { - EnvironmentDataClient::connect(&connection.endpoint, options) - .await - .map_err(map_environment_client_error) +/// The session's working directory in the environment: the attachment's +/// configured one, else the daemon's `defaultCwd`. It must be a directory, +/// a symlink to one inside the fs root included; a daemon without the +/// filesystem family cannot be asked, so its directory is taken as given. +async fn environment_working_directory( + client: &DataClient, + configured: Option<&str>, +) -> Result { + let initialized = client.initialize_response(); + let cwd = match configured { + Some(value) => { + if !value.starts_with('/') { + return Err(format!( + "environment working directory must be absolute: {value}" + )); + } + let normalized = + tools::environment::sources::absolute(std::path::Path::new("/"), value)?; + EnvironmentPath::new(normalized.to_string_lossy()) + .map_err(|error| format!("environment working directory {value}: {error}"))? } - EnvironmentTransport::Http => Err(unsupported_environment_data_transport("http")), - EnvironmentTransport::Stdio => Err(unsupported_environment_data_transport("stdio")), - EnvironmentTransport::Ssh => Err(unsupported_environment_data_transport("ssh")), - EnvironmentTransport::Provider { provider_type } => Err( - unsupported_environment_data_transport(format!("provider:{provider_type}")), - ), + None => initialized.default_cwd.clone(), + }; + if initialized.capabilities.filesystem.is_none() { + return Ok(cwd); } -} - -fn unsupported_environment_data_transport(transport: impl std::fmt::Display) -> CoreAgentIoError { - io_error(format!( - "environment data transport is not supported by this worker: {transport}" - )) + let response = client + .get_metadata(&GetMetadataParams { + path: cwd.clone(), + follow_symlinks: true, + }) + .await + .map_err(|error| format!("working directory {}: {error}", cwd.as_str()))?; + if response.metadata.kind != FileKind::Directory { + return Err(format!( + "working directory is not a directory: {}", + cwd.as_str() + )); + } + Ok(cwd) } #[derive(Clone, Copy)] @@ -2302,7 +2235,7 @@ fn map_environments_error(error: EnvironmentRegistryError) -> CoreAgentIoError { io_error(format!("load session environment bindings: {error}")) } -fn map_environment_client_error(error: EnvironmentClientError) -> CoreAgentIoError { +fn map_environment_client_error(error: ClientError) -> CoreAgentIoError { io_error(format!("environment data-plane call failed: {error}")) } @@ -2335,9 +2268,9 @@ mod tests { use tools::environment::control::ENVIRONMENT_LIST_TOOL_NAME; use crate::environments::runtime::RuntimeEnvironment; - use environment_protocol::shared::{EnvironmentTransport, ProviderTargetId}; + use environment_protocol::ids::{JobId, ProviderTargetId}; use environments::{ - CreateEnvironment, EnvironmentConnectionSpec, EnvironmentIncarnationId, + CreateEnvironment, EnvironmentEndpoint, EnvironmentIncarnationId, EnvironmentIncarnationRecord, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, EnvironmentProvisionRequestId, EnvironmentSource, @@ -4256,7 +4189,7 @@ mod tests { blobs: Arc, process: Arc, ) -> RuntimeEnvironment { - let target_id = ProviderTargetId::new("test"); + let target_id = ProviderTargetId::new("test").unwrap(); let resource = environments::EnvironmentRecord { environment_id: harness::EnvironmentId::new("test"), request_id: EnvironmentProvisionRequestId::new("request-test"), @@ -4282,6 +4215,7 @@ mod tests { public_endpoint: None, metadata: BTreeMap::new(), last_seen_at_ms: None, + lifecycle_error: None, created_at_ms: 1, updated_at_ms: 1, }; @@ -4298,23 +4232,31 @@ mod tests { #[cfg(any(target_os = "linux", target_os = "macos"))] mod transfers { use super::*; - use environment_protocol::data::transfer_session::{TransferRequest, TransferResponse}; - - struct LocalTransfer(environment_daemon::filesystem::LocalFileSystem); - - #[async_trait] - impl tools::transfer::EnvironmentTransfer for LocalTransfer { - async fn request( - &self, - request: TransferRequest, - ) -> tools::ToolResult { - self.0 - .transfer(request) - .await - .map_err(|error| tools::ToolError::InvalidRequest { - message: error.message, - }) - } + use environment_protocol::handshake::ImplementationInfo; + + /// envd's data plane over an in-process socket, rooted at `root` + /// with its state dir beside it. + async fn in_process_envd(root: &std::path::Path, state: &std::path::Path) -> DataClient { + let runtime = + environment_daemon::DaemonRuntime::new(environment_daemon::config::DaemonConfig { + listen: None, + cwd: root.into(), + fs_root: root.into(), + state_dir: state.into(), + read_only_fs: false, + registration: None, + scrubbed_env: Vec::new(), + }) + .expect("envd runtime"); + let (client_io, server_io) = tokio::io::duplex(1 << 20); + tokio::spawn(environment_daemon::server::serve_stream(runtime, server_io)); + let options = environment_client::ConnectOptions::new( + ImplementationInfo::new("session-tools-test").expect("client info"), + ); + DataClient::connect_initialized_over(client_io, options) + .await + .expect("connect to in-process envd") + .0 } async fn dispatch( @@ -4368,19 +4310,19 @@ mod tests { ] { let (blobs, tools, session_id, workspace_attachments) = session_tools_with_readme_attachment().await; - let root = tempfile::tempdir().unwrap(); - let fs = environment_daemon::filesystem::LocalFileSystem::new( - root.path().into(), - root.path().into(), - true, - ); + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap().join("root"); + std::fs::create_dir_all(&root).unwrap(); + let client = + in_process_envd(&root, &temp.path().canonicalize().unwrap().join("state")) + .await; let environment = test_environment( blobs.clone(), Arc::new(RecordingProcessExecutor::default()), ); let mut context = environment.tool_context().clone(); - context.process_cwd = Some(FsPath::new(root.path().to_string_lossy()).unwrap()); - context.transfer = Some(Arc::new(LocalTransfer(fs))); + context.process_cwd = Some(FsPath::new(root.to_string_lossy()).unwrap()); + context.transfer = Some(Arc::new(client)); let environment = RuntimeEnvironment::from_resource(environment.resource().clone(), context); let tools = tools @@ -4403,20 +4345,17 @@ mod tests { let result = dispatch(&tools, request.clone(), batch).await; succeeded(&blobs, &result).await; assert_eq!( - std::fs::read(root.path().join("created/nested/README.md")).unwrap(), + std::fs::read(root.join("created/nested/README.md")).unwrap(), b"hello\n" ); // A completed retry through the hosted path must keep its operation // identity and preserve edits made after publication. - std::fs::write( - root.path().join("created/nested/README.md"), - b"environment edit", - ) - .unwrap(); + std::fs::write(root.join("created/nested/README.md"), b"environment edit") + .unwrap(); succeeded(&blobs, &dispatch(&tools, request.clone(), batch).await).await; assert_eq!( - std::fs::read(root.path().join("created/nested/README.md")).unwrap(), + std::fs::read(root.join("created/nested/README.md")).unwrap(), b"environment edit" ); @@ -4511,11 +4450,10 @@ mod tests { .put_provider(PutEnvironmentProvider { provider_id: EnvironmentProviderId::new(provider_id), display_name: None, - controller_connection: EnvironmentConnectionSpec::new( - "http://controller.test", - EnvironmentTransport::Http, - ), + controller_endpoint: EnvironmentEndpoint::parse("ws://controller.test/control") + .unwrap(), metadata: BTreeMap::new(), + credential: Some(auth::SecretValue::new("provider-credential")), updated_at_ms: 10, }) .await @@ -4540,7 +4478,7 @@ mod tests { provider_id: &str, observed_at_ms: i64, ) { - let target_id = ProviderTargetId::new(format!("target-{environment_id}")); + let target_id = ProviderTargetId::new(format!("target-{environment_id}")).unwrap(); let environment_id = EnvironmentId::new(environment_id); store .create_environment(CreateEnvironment { @@ -4562,9 +4500,10 @@ mod tests { store .observe_provisioned_environment(ObserveProvisionedEnvironment { environment_id, - provider_target_id: target_id, + provider_target_id: Some(target_id), status: EnvironmentStatus::Offline, power_states: Vec::new(), + daemon_build: None, observed_at_ms, }) .await @@ -4619,9 +4558,12 @@ mod tests { registry .observe_provisioned_environment(ObserveProvisionedEnvironment { environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-environment-allowed-1"), + provider_target_id: Some( + ProviderTargetId::new("target-environment-allowed-1").unwrap(), + ), status, power_states, + daemon_build: None, observed_at_ms: 11, }) .await @@ -4723,7 +4665,7 @@ mod tests { Some(&policy), JobHandleArg { environment_id: Some(handle.into()), - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), }, ) .unwrap(); @@ -4744,7 +4686,7 @@ mod tests { Some(&test_environment_policy(&["other"])), JobHandleArg { environment_id: Some(handle.into()), - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), } ) .is_err() @@ -4901,9 +4843,10 @@ mod tests { registry .observe_provisioned_environment(ObserveProvisionedEnvironment { environment_id: EnvironmentId::new("environment-pending"), - provider_target_id: ProviderTargetId::new("target-pending"), + provider_target_id: Some(ProviderTargetId::new("target-pending").unwrap()), status: EnvironmentStatus::Ready, power_states: Vec::new(), + daemon_build: None, observed_at_ms: 20, }) .await @@ -4937,6 +4880,7 @@ mod tests { .fail_environment_lifecycle(environments::FailEnvironmentLifecycle { environment_id: EnvironmentId::new("environment-failed"), message: "no capacity".to_owned(), + daemon_build: None, observed_at_ms: 11, }) .await @@ -5742,7 +5686,7 @@ mod tests { ])), JobHandleArg { environment_id: None, - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), }, ) .expect("defaults to the active environment"); @@ -5757,7 +5701,7 @@ mod tests { ])), JobHandleArg { environment_id: Some("environment_other".to_owned()), - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), }, ) .expect("explicit environment wins"); @@ -5769,7 +5713,7 @@ mod tests { None, JobHandleArg { environment_id: None, - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), }, ) .is_err(), diff --git a/crates/temporal-runtime/tests/code_capabilities_live.rs b/crates/temporal-runtime/tests/code_capabilities_live.rs index 328f22ff1..1ac73f4b4 100644 --- a/crates/temporal-runtime/tests/code_capabilities_live.rs +++ b/crates/temporal-runtime/tests/code_capabilities_live.rs @@ -313,6 +313,7 @@ where let mut tasks = Tasks(vec![task.abort_handle()]); let directory = tempfile::tempdir()?; let root = directory.path().canonicalize()?; + let daemon_state = tempfile::tempdir()?; let context = support::live::local_request_context_for(api::AccessScope::Universe { universe_id: universe, }) @@ -326,11 +327,11 @@ where display_name:"Code capability test".into(), identity_mode:api::EnvironmentIdentityModeView::Ephemeral, max_active_environments:Some(1), ephemeral_disconnect_grace_ms:None, expires_at_ms:None, }).await?.result; - let connect_url = format!("ws://{address}/environment-gateway/connect"); + let connect_url = format!("ws://{address}{}", environment_protocol::routes::CONNECT_PATH); let mut registration = environment_daemon::config::RegistrationConfig::new(connect_url.clone(), environment_daemon::upgrade::resolve_discovery_url(Some(&connect_url), None)?); - registration.registration_key = Some(environment_protocol::shared::SecretString::new(key.secret.0)); + registration.registration_key = Some(environment_protocol::bytes::SecretString::new(key.secret.0)); let daemon = environment_daemon::DaemonRuntime::new(environment_daemon::config::DaemonConfig { - listen:None, cwd:root.clone(), fs_root:root.clone(), state_dir:root.join(".envd"), read_only_fs:false, registration:Some(registration), scrubbed_env:vec![], + listen:None, cwd:root.clone(), fs_root:root.clone(), state_dir:daemon_state.path().into(), read_only_fs:false, registration:Some(registration), scrubbed_env:vec![], })?; let task = tokio::spawn(environment_daemon::server::run(daemon)); tasks.0.push(task.abort_handle()); @@ -643,9 +644,11 @@ async fn jobs_and_subagents_case(fixture: Fixture) -> anyhow::Result<()> { .as_ref() .is_some_and(|handle| handle.environment_id == tools::environment::handles::environment_handle(environment)) - && actual.summary.as_ref().is_some_and(|summary| summary.status - == environment_protocol::data::jobs::JobStatus::Succeeded - && &summary.namespace == environment), + && actual + .summary + .as_ref() + .is_some_and(|summary| summary.status + == tools::environment::jobs::ModelJobStatus::Succeeded), "job must finish on the admitted environment: {actual:?}" ); } diff --git a/crates/temporal-runtime/tests/environment_provider_live.rs b/crates/temporal-runtime/tests/environment_provider_live.rs index 6f2c35b1b..ba421c6a8 100644 --- a/crates/temporal-runtime/tests/environment_provider_live.rs +++ b/crates/temporal-runtime/tests/environment_provider_live.rs @@ -1,9 +1,10 @@ //! Minimal live environment control-plane acceptance test. //! -//! This deliberately uses the in-process provider with real Postgres and the -//! real deployment reconciler. It covers the durable provider and ingress seams -//! without requiring an Incus installation; the separate Incus live smoke test -//! owns backend connectivity and topology. +//! This deliberately uses the fake controller, served on loopback and +//! registered like any provider, with real Postgres and the real deployment +//! reconciler. It covers the durable provider and ingress seams without +//! requiring an Incus installation; the separate Incus live smoke test owns +//! backend connectivity and topology. mod support; @@ -11,8 +12,7 @@ use std::{collections::BTreeMap, sync::Arc, time::Duration}; use api::{ AgentApiService, DeploymentApiService, DeploymentEnvironmentAdoptParams, - DeploymentEnvironmentProviderConnection, DeploymentEnvironmentProviderDeleteParams, - DeploymentEnvironmentProviderPutParams, DeploymentEnvironmentProviderTransport, + DeploymentEnvironmentProviderDeleteParams, DeploymentEnvironmentProviderPutParams, DeploymentProviderBindingPutParams, DeploymentUniverseCreateParams, EnvironmentCloseParams, EnvironmentCreateParams, EnvironmentIngressPutParams, EnvironmentLifecycleStatusView, EnvironmentListParams, EnvironmentProviderBindingStatusView, EnvironmentTemplateListParams, @@ -26,6 +26,7 @@ use support::live::{ }; use temporal_runtime::{ DeploymentStores, UniverseRuntime, + environments::fake_controller::{FAKE_INGRESS_ENDPOINT, FAKE_TEMPLATE_ID, FakeController}, gateway::{GatewayAgentApi, GatewayDeploymentApi}, pg_store_from_env, }; @@ -57,6 +58,7 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res stores, )?); let operator = GatewayDeploymentApi::new(runtime.clone()); + let controller = FakeController::start().await?; let reconciler = tokio::spawn(runtime.clone().run_environment_reconciler()); // Direct service calls carry an explicit local request context. use temporal_runtime::gateway::request_context::{ @@ -80,16 +82,46 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res operator.put_environment_provider(DeploymentEnvironmentProviderPutParams { provider_id: provider_id.clone(), display_name: Some("Live fake provider".to_owned()), - controller_connection: DeploymentEnvironmentProviderConnection { - endpoint: "in-process".to_owned(), - transport: DeploymentEnvironmentProviderTransport::Provider { - provider_type: "fake".to_owned(), - }, - }, + controller_endpoint: controller.controller_url().to_owned(), metadata: BTreeMap::new(), + credential: Some(controller.credential().to_owned()), }), ) .await?; + // Registration ran the controller handshake and recorded its facts. + let registered = calling( + deployment_caller.clone(), + operator.read_environment_provider(api::DeploymentEnvironmentProviderReadParams { + provider_id: provider_id.clone(), + }), + ) + .await? + .result + .provider; + assert_eq!( + registered.controller_features, + Some(api::DeploymentEnvironmentProviderFeaturesView { + adopt_target: true, + ingress: true, + }) + ); + assert!(registered.data_route_base.is_some()); + // A controller URL that names no controller is refused outright. + let mispathed = calling( + deployment_caller.clone(), + operator.put_environment_provider(DeploymentEnvironmentProviderPutParams { + provider_id: format!("{provider_id}-mispathed"), + display_name: None, + controller_endpoint: controller + .controller_url() + .replace("/control", "/elsewhere"), + metadata: BTreeMap::new(), + credential: Some(controller.credential().to_owned()), + }), + ) + .await + .expect_err("a mis-pathed controller URL is refused"); + assert_eq!(mispathed.kind, api::AgentApiErrorKind::InvalidRequest); for (universe_id, binding_id) in [(universe_a, "primary-a"), (universe_b, "primary-b")] { calling( deployment_caller.clone(), @@ -123,14 +155,15 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res ) .await?; assert_eq!(templates.result.templates.len(), 1); - assert_eq!(templates.result.templates[0].template_id, "rust-v1"); + assert_eq!(templates.result.templates[0].template_id, FAKE_TEMPLATE_ID); + assert!(templates.result.templates[0].resources.is_some()); let created = calling( caller_a.clone(), state_a.api.create_environment(EnvironmentCreateParams { request_id: format!("create-{suffix}"), binding_id: "primary-a".to_owned(), - template_id: "rust-v1".to_owned(), + template_id: FAKE_TEMPLATE_ID.to_owned(), display_name: Some("Provisioned live VM".to_owned()), metadata: BTreeMap::new(), idle_policy: None, @@ -144,7 +177,7 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res state_a.api.create_environment(EnvironmentCreateParams { request_id: format!("create-{suffix}"), binding_id: "primary-a".to_owned(), - template_id: "rust-v1".to_owned(), + template_id: FAKE_TEMPLATE_ID.to_owned(), display_name: None, metadata: BTreeMap::new(), idle_policy: None, @@ -208,6 +241,19 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res .await?; assert!(created_ready.incarnation.provider_target_id.is_some()); assert!(adopted_ready.incarnation.provider_target_id.is_some()); + // The Ready transition ran the data-plane handshake and recorded the + // daemon's protocol version. + assert_eq!( + created_ready + .metadata + .get(environments::ENVD_PROTOCOL_VERSION_METADATA_KEY) + .map(String::as_str), + Some( + environment_protocol::ENVD_PROTOCOL_VERSION + .to_string() + .as_str() + ) + ); let listed_a = calling( caller_a.clone(), @@ -236,10 +282,17 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res }), ) .await?; - assert_eq!( - ingress.result.environment.public_endpoint.as_deref(), - Some("https://fake.env.test") - ); + // The put records the intent; the reconciler realizes it. + assert!(ingress.result.environment.public_ingress_enabled); + calling( + caller_a.clone(), + wait_for_public_endpoint( + state_a.api.as_ref(), + created.environment_id.as_str(), + Some(FAKE_INGRESS_ENDPOINT), + ), + ) + .await?; calling( caller_a.clone(), state_a @@ -250,6 +303,11 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res }), ) .await?; + calling( + caller_a.clone(), + wait_for_public_endpoint(state_a.api.as_ref(), created.environment_id.as_str(), None), + ) + .await?; for (api, caller, environment_id) in [ ( @@ -278,6 +336,7 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res reconciler.abort(); let _ = reconciler.await; + drop(controller); runtime.evict(universe_a).await; runtime.evict(universe_b).await; for universe_id in [universe_a, universe_b] { @@ -319,6 +378,33 @@ async fn wait_for_status( } } +async fn wait_for_public_endpoint( + api: &temporal_runtime::gateway::GatewayAgentApi, + environment_id: &str, + expected: Option<&str>, +) -> anyhow::Result { + let started = std::time::Instant::now(); + loop { + let environment = api + .read_environment(api::EnvironmentReadParams { + environment_id: environment_id.to_owned(), + }) + .await? + .result + .environment; + if environment.public_endpoint.as_deref() == expected { + return Ok(environment); + } + if started.elapsed() > Duration::from_secs(10) { + anyhow::bail!( + "timed out waiting for environment {environment_id} to report public endpoint {expected:?}; it reports {:?}", + environment.public_endpoint + ); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + fn require_live_env() -> anyhow::Result<()> { if std::env::var("LIGHTSPEED_TEST_POSTGRES_URL").is_err() && std::env::var("LIGHTSPEED_POSTGRES_URL").is_err() @@ -353,9 +439,8 @@ async fn run_environment_power_live_client( ) -> anyhow::Result<()> { use std::collections::BTreeMap; - use environment_protocol::shared::EnvironmentTransport; use environments::{ - EnvironmentConnectionSpec, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, + EnvironmentEndpoint, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, PutEnvironmentProvider, PutEnvironmentProviderBinding, }; @@ -369,17 +454,14 @@ async fn run_environment_power_live_client( let suffix = uuid::Uuid::new_v4().simple().to_string(); let provider_id = format!("fake-power-{suffix}"); let binding_id = format!("binding-power-{suffix}"); + let controller = FakeController::start().await?; store .put_provider(PutEnvironmentProvider { provider_id: EnvironmentProviderId::new(provider_id.clone()), display_name: Some("Live fake provider (power)".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "in-process", - EnvironmentTransport::Provider { - provider_type: "fake".to_owned(), - }, - ), + controller_endpoint: EnvironmentEndpoint::parse(controller.controller_url())?, metadata: BTreeMap::new(), + credential: Some(auth::SecretValue::new(controller.credential())), updated_at_ms: 1, }) .await?; @@ -405,7 +487,7 @@ async fn run_environment_power_live_client( .create_environment(api::EnvironmentCreateParams { request_id: format!("power-{suffix}"), binding_id: binding_id.clone(), - template_id: "rust-v1".to_owned(), + template_id: FAKE_TEMPLATE_ID.to_owned(), display_name: Some("Power VM".to_owned()), metadata: BTreeMap::new(), idle_policy: Some(policy.clone()), @@ -480,8 +562,8 @@ async fn run_environment_power_live_client( .environment; assert_eq!(restored.idle_policy.as_ref(), Some(&policy)); - // The reaper sees the candidate but cannot reach a daemon through the - // fake provider, so it leaves the environment alone. + // The reaper sees the candidate but no gateway serves its route here, so + // it cannot read an idle report and leaves the environment alone. let stats = api.reap_idle_environments_once().await?; assert_eq!(stats.candidates, 1); assert_eq!(stats.unreachable, 1); @@ -660,10 +742,7 @@ async fn run_environment_power_live_client( let external = api .create_external_environment(api::EnvironmentExternalCreateParams { request_id: format!("power-external-{suffix}"), - connection: api::EnvironmentConnectionView { - endpoint: format!("ws://127.0.0.1:1/{suffix}"), - transport: api::EnvironmentConnectionTransportView::WebSocket, - }, + endpoint: format!("ws://127.0.0.1:1/{suffix}"), display_name: None, metadata: BTreeMap::new(), }) diff --git a/crates/temporal-runtime/tests/environment_registration_live.rs b/crates/temporal-runtime/tests/environment_registration_live.rs index 8a45ff53c..bf84c0c17 100644 --- a/crates/temporal-runtime/tests/environment_registration_live.rs +++ b/crates/temporal-runtime/tests/environment_registration_live.rs @@ -1,10 +1,11 @@ //! Key-based outbound registration, end to end against the real gateway: //! an in-process `lightspeed-envd` dials the environment gateway with a //! registration key, becomes a `Ready` registered environment, serves a -//! worker route through a reverse-dialed data socket, goes `Offline` when -//! its control connection drops, reconnects by identity alone, keeps -//! reconnecting after its key is revoked, and is refused for good once its -//! environment is closed. +//! worker route through a reverse-dialed stream socket, goes `Offline` when +//! its control connection drops, reconnects by identity alone, loses to a +//! newer process with the same identity, keeps reconnecting after its key +//! is revoked, closes its ephemeral environment at once when it says +//! goodbye, and is told to exit for good once its environment is closed. mod support; @@ -17,22 +18,31 @@ use api::{ EnvironmentRegistrationKeyReadParams, EnvironmentRegistrationKeyRevokeParams, EnvironmentSourceView, EnvironmentView, }; -use environment_client::{EnvironmentDataClient, JsonRpcTransport}; +use environment_client::DataClient; use environment_daemon::{ DaemonRuntime, config::{DaemonConfig, RegistrationConfig}, + server::DaemonExit, }; use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + bytes::{ByteChunk, SecretString}, data::{ + exec::{ + CommandSpec, ExecInput, ExecState, ExitStatus, OutputChunk, PtySize, ReturnOn, + StoppedBy, + }, fs::{ReadDirectoryParams, WriteFileParams}, - handshake::InitializeParams, - idle::IdleParams, + idle::GetIdleParams, process::{ - ProcessOutputChunk, ProcessSignal, ReadProcessParams, StartProcessParams, + ProcessSignal, ReadProcessParams, ReadProcessResponse, StartProcessParams, TerminateProcessParams, WriteProcessParams, }, }, - shared::{ByteChunk, CURRENT_PROTOCOL_VERSION, EnvironmentPath, ProcessId, SecretString}, + error::ProtocolErrorKind, + ids::ProcessId, + metadata::{DisplayName, Metadata}, + paths::EnvironmentPath, }; use environments::EnvironmentStore as _; use support::live::{LIVE_TEST_LOCK, require_storage_live_env}; @@ -207,15 +217,15 @@ async fn scenario( assert_eq!(minted.registration_key.active_environment_count, 0); // First daemon: a fresh identity admitted by the key. - let root_a = sandbox.join("a"); - std::fs::create_dir_all(&root_a)?; - let receipt_a = root_a.join("receipt.json"); + let root_a = workspace(sandbox, "a")?; + let receipt_a = sandbox.join("receipt-a.json"); let daemon_a = spawn_daemon(daemon_config( &root_a, + &sandbox.join("state-a"), connect_url, Some(&secret), Some(&receipt_a), - )?); + )?)?; let environment_a = wait_for_registered(api, &key_id, 1, EnvironmentLifecycleStatusView::Ready).await?; let EnvironmentSourceView::Registered { @@ -233,51 +243,40 @@ async fn scenario( assert_eq!(*identity_mode, EnvironmentIdentityModeView::Ephemeral); assert!(daemon_id.starts_with("daemon_")); assert!(environment_a.last_seen_at_ms.is_some()); + let metadata = &environment_a.metadata; assert_eq!( - environment_a - .metadata - .get("harbor.trialId") - .map(String::as_str), + metadata.get("harbor.trialId").map(String::as_str), Some("trial-1") ); assert_eq!( - environment_a - .metadata - .get("lightspeed.envd.version") - .map(String::as_str), + metadata.get("lightspeed.envd.version").map(String::as_str), Some(release_info::VERSION) ); assert_eq!( - environment_a - .metadata - .get("lightspeed.envd.gitSha") - .map(String::as_str), + metadata.get("lightspeed.envd.gitSha").map(String::as_str), Some(release_info::GIT_SHA) ); assert_eq!( - environment_a - .metadata - .get("lightspeed.envd.protocolVersion") - .map(String::as_str), - Some(CURRENT_PROTOCOL_VERSION.to_string().as_str()) + metadata.get("lightspeed.envd.protocolVersion"), + Some(&ENVD_PROTOCOL_VERSION.to_string()) ); - let receipt: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&receipt_a)?)?; + let receipt = read_receipt(&receipt_a)?; assert_eq!(receipt["environmentId"], environment_a.environment_id); assert_eq!(receipt["daemonId"], daemon_id.as_str()); assert_eq!(receipt["identityMode"], "ephemeral"); + assert_eq!(receipt["admission"], "created"); + assert!( + receipt["controlConnectionId"] + .as_str() + .is_some_and(|id| id.starts_with("connection_")) + ); assert!(receipt.get("registrationKey").is_none()); - let key_view = api - .read_environment_registration_key(EnvironmentRegistrationKeyReadParams { - registration_key_id: key_id.clone(), - }) - .await? - .result - .registration_key; + let key_view = read_key(api, &key_id).await?; assert_eq!(key_view.active_environment_count, 1); assert_eq!(key_view.registered_environment_count, 1); - // The worker route reaches the daemon through a reverse-dialed data - // socket; two concurrent routes each get their own socket. + // The worker route reaches the daemon through a reverse-dialed stream + // socket; concurrent routes each get their own socket. let record = runtime .state_for(universe_id, false) .await? @@ -293,81 +292,76 @@ async fn scenario( base_url, runtime.environment_gateway().deployment_token(), ); - let connection = gateway.connection_for(universe_id, &record); - let mut first = EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("registration-live"), - ) - .await?; - let mut second = EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("registration-live"), - ) - .await?; - let mut third = EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("registration-live"), - ) - .await?; - let mut capabilities = None; - for client in [&mut first, &mut second, &mut third] { - let initialized = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "registration-live".to_owned(), - scope: environment_protocol::shared::EnvironmentScope::Default, - resume_connection_id: None, - }) - .await?; - assert_eq!(initialized.protocol_version, CURRENT_PROTOCOL_VERSION); - capabilities = Some(initialized.capabilities); - } - let capabilities = capabilities.expect("negotiated capabilities"); + let key = gateway.route_key(universe_id, &record); + let options = || gateway.connect_options("registration-live"); + let (first, initialized) = gateway.connect(&key, options()).await?; + let (second, _) = gateway.connect(&key, options()).await?; + let (third, _) = gateway.connect(&key, options()).await?; + assert_eq!(initialized.protocol_version, ENVD_PROTOCOL_VERSION); + assert_eq!( + initialized.default_cwd.as_str(), + root_a.to_str().expect("utf-8 root") + ); + let written = path(&root_a, "registered.txt")?; first .write_file(&WriteFileParams { - path: EnvironmentPath::new("registered.txt")?, + path: written.clone(), data: ByteChunk::new(b"through the reverse dial".to_vec()), + create: Default::default(), + if_match: None, + executable: false, }) .await?; let listing = second .read_directory(&ReadDirectoryParams { - path: EnvironmentPath::new(".")?, + path: initialized.default_cwd.clone(), + limit: 100, + cursor: None, }) .await?; assert!( listing .entries .iter() - .any(|entry| entry.file_name == "registered.txt"), + .any(|entry| entry.name.as_str() == "registered.txt"), "second data socket sees the file the first wrote: {listing:?}" ); assert_eq!( std::fs::read_to_string(root_a.join("registered.txt"))?, "through the reverse dial" ); - process_scenario(&mut first, &mut second, &root_a).await?; - executor_scenario(third, capabilities).await?; + process_scenario(&first, &second, &root_a).await?; + executor_scenario(third).await?; first.close().await?; second.close().await?; // Dropping the control connection makes the environment Offline; the // same identity, now without any key, reconnects the same environment. - daemon_a.abort(); + daemon_a.exit.abort(); wait_for_status( api, &environment_a.environment_id, EnvironmentLifecycleStatusView::Offline, ) .await?; - let daemon_a = spawn_daemon(daemon_config(&root_a, connect_url, None, Some(&receipt_a))?); + let daemon_a = spawn_daemon(daemon_config( + &root_a, + &sandbox.join("state-a"), + connect_url, + None, + Some(&receipt_a), + )?)?; wait_for_status( api, &environment_a.environment_id, EnvironmentLifecycleStatusView::Ready, ) .await?; - let receipt: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&receipt_a)?)?; - assert_eq!(receipt["environmentId"], environment_a.environment_id); + wait_for_receipt_admission(&receipt_a, "reconnected").await?; + assert_eq!( + read_receipt(&receipt_a)?["environmentId"], + environment_a.environment_id + ); assert_eq!( wait_for_registered(api, &key_id, 1, EnvironmentLifecycleStatusView::Ready) .await? @@ -376,10 +370,49 @@ async fn scenario( "reconnect must not create a second environment" ); + // A second process with the same identity (a cloned state directory) + // wins; the older one is told it was superseded and exits instead of + // fighting for the identity. + let twin_state = sandbox.join("state-a-twin"); + std::fs::create_dir_all(&twin_state)?; + std::fs::copy( + sandbox + .join("state-a") + .join(environment_daemon::identity::DAEMON_KEY_FILE), + twin_state.join(environment_daemon::identity::DAEMON_KEY_FILE), + )?; + let twin_a = spawn_daemon(daemon_config( + &root_a, + &twin_state, + connect_url, + None, + None, + )?)?; + let superseded = tokio::time::timeout(SETTLE, daemon_a.exit).await??; + assert!( + matches!(superseded, DaemonExit::Superseded { .. }), + "unexpected exit: {superseded:?}" + ); + assert_eq!(superseded.status(), 5); + wait_for_status( + api, + &environment_a.environment_id, + EnvironmentLifecycleStatusView::Ready, + ) + .await?; + let (after_supersede, _) = gateway.connect(&key, options()).await?; + after_supersede.get_idle(&GetIdleParams {}).await?; + after_supersede.close().await?; + // A second identity with the same key is a second environment. - let root_b = sandbox.join("b"); - std::fs::create_dir_all(&root_b)?; - let daemon_b = spawn_daemon(daemon_config(&root_b, connect_url, Some(&secret), None)?); + let root_b = workspace(sandbox, "b")?; + let daemon_b = spawn_daemon(daemon_config( + &root_b, + &sandbox.join("state-b"), + connect_url, + Some(&secret), + None, + )?)?; let environments = wait_for_registered_count(api, &key_id, 2).await?; let environment_b = environments .iter() @@ -394,74 +427,78 @@ async fn scenario( close_environments: false, }) .await?; - let root_c = sandbox.join("c"); - std::fs::create_dir_all(&root_c)?; - let daemon_c = spawn_daemon(daemon_config(&root_c, connect_url, Some(&secret), None)?); - let refused = tokio::time::timeout(SETTLE, daemon_c).await??; - let refused = refused.expect_err("a revoked key is a terminal refusal for a new daemon"); + let root_c = workspace(sandbox, "c")?; + let daemon_c = spawn_daemon(daemon_config( + &root_c, + &sandbox.join("state-c"), + connect_url, + Some(&secret), + None, + )?)?; + let refused = tokio::time::timeout(SETTLE, daemon_c.exit).await??; assert!( - refused.to_string().contains("RegistrationKeyRevoked"), - "unexpected refusal: {refused:#}" + matches!(refused, DaemonExit::Refused(_)), + "a revoked key is a terminal refusal for a new daemon: {refused:?}" ); - daemon_b.abort(); - wait_for_status( - api, - &environment_b.environment_id, - EnvironmentLifecycleStatusView::Offline, - ) - .await?; - let daemon_b = spawn_daemon(daemon_config(&root_b, connect_url, None, None)?); + assert_eq!(refused.status(), 3); + + // A deliberate stop says goodbye and the daemon exits cleanly. The + // sandbox is finished, so its ephemeral environment closes at once + // instead of sitting Offline for the key's disconnect grace. + daemon_b.runtime.shutdown().cancel(); + let stopped = tokio::time::timeout(SETTLE, daemon_b.exit).await??; + assert!( + matches!(stopped, DaemonExit::Shutdown), + "unexpected exit: {stopped:?}" + ); + assert_eq!(stopped.status(), 0); wait_for_status( api, &environment_b.environment_id, - EnvironmentLifecycleStatusView::Ready, + EnvironmentLifecycleStatusView::Closed, ) .await?; + // The departed identity is spent: a restart is refused at register. + let restarted_b = spawn_daemon(daemon_config( + &root_b, + &sandbox.join("state-b"), + connect_url, + None, + None, + )?)?; + let refused_b = tokio::time::timeout(SETTLE, restarted_b.exit).await??; + assert!( + matches!(refused_b, DaemonExit::EnvironmentClosed), + "a departed ephemeral identity cannot reconnect: {refused_b:?}" + ); + assert_eq!(refused_b.status(), 4); - // Closing the environment spends the identity: the daemon is told to go - // and its reconnect is refused for good. + // Closing the environment spends the identity: the daemon is told the + // environment closed and exits for good. api.close_environment(EnvironmentCloseParams { environment_id: environment_a.environment_id.clone(), }) .await?; - let spent = tokio::time::timeout(SETTLE, daemon_a).await??; - let spent = spent.expect_err("a closed environment is a terminal refusal"); + let spent = tokio::time::timeout(SETTLE, twin_a.exit).await??; assert!( - spent.to_string().contains("EnvironmentClosed"), - "unexpected refusal: {spent:#}" + matches!(spent, DaemonExit::EnvironmentClosed), + "a closed environment ends its daemon: {spent:?}" ); + assert_eq!(spent.status(), 4); wait_for_status( api, &environment_a.environment_id, EnvironmentLifecycleStatusView::Closed, ) .await?; - let key_view = api - .read_environment_registration_key(EnvironmentRegistrationKeyReadParams { - registration_key_id: key_id.clone(), - }) - .await? - .result - .registration_key; + let key_view = read_key(api, &key_id).await?; assert_eq!(key_view.registered_environment_count, 2); - assert_eq!(key_view.active_environment_count, 1); + assert_eq!(key_view.active_environment_count, 0); assert_eq!( key_view.status, api::EnvironmentRegistrationKeyStatusView::Revoked ); - daemon_b.abort(); - api.close_environment(EnvironmentCloseParams { - environment_id: environment_b.environment_id.clone(), - }) - .await?; - wait_for_status( - api, - &environment_b.environment_id, - EnvironmentLifecycleStatusView::Closed, - ) - .await?; - // Ephemeral grace: a key with a short grace closes an environment whose // daemon stays away, through the lifecycle reconciler alone. let short = api @@ -475,30 +512,24 @@ async fn scenario( .await? .result; let short_key_id = short.registration_key.registration_key_id.clone(); - let root_d = sandbox.join("d"); - std::fs::create_dir_all(&root_d)?; + let root_d = workspace(sandbox, "d")?; let daemon_d = spawn_daemon(daemon_config( &root_d, + &sandbox.join("state-d"), connect_url, Some(&short.secret.0), None, - )?); + )?)?; let environment_d = wait_for_registered(api, &short_key_id, 1, EnvironmentLifecycleStatusView::Ready).await?; - daemon_d.abort(); + daemon_d.exit.abort(); wait_for_status( api, &environment_d.environment_id, EnvironmentLifecycleStatusView::Closed, ) .await?; - let short_view = api - .read_environment_registration_key(EnvironmentRegistrationKeyReadParams { - registration_key_id: short_key_id, - }) - .await? - .result - .registration_key; + let short_view = read_key(api, &short_key_id).await?; assert_eq!(short_view.active_environment_count, 0); assert_eq!(short_view.registered_environment_count, 1); Ok(()) @@ -508,88 +539,115 @@ async fn scenario( /// at once: a normal exit leaves its background child running, reported /// and counted by the idle report and stoppable through the handle; the /// read cursor is the daemon's, so the second socket continues where the -/// first stopped and an empty write is a wait; a PTY takes input; and an -/// interrupt reaches the group. -async fn process_scenario( - first: &mut EnvironmentDataClient, - second: &mut EnvironmentDataClient, +/// first stopped while positioned reads consume nothing; a PTY takes input; +/// an interrupt reaches the group; and terminate escalates gracefully. +async fn process_scenario( + first: &DataClient, + second: &DataClient, root: &Path, -) -> anyhow::Result<()> -where - T: JsonRpcTransport + Send + 'static, -{ +) -> anyhow::Result<()> { // A service the command left behind survives its exit. first .start_process(&shell_process( "live-leftover", "nohup sleep 60 >/dev/null 2>&1 & echo $! > leftover.pid; echo started; exit 0", false, - )) + )?) .await?; let leftover_pid = wait_for_pid_file(&root.join("leftover.pid")).await?; - tokio::time::sleep(Duration::from_millis(100)).await; let read = first - .read_process(&read_params("live-leftover", Some(10_000))) + .read_process(&read_until_exit("live-leftover")?) .await?; - assert!(read.exited); - assert_eq!(read.exit_code, Some(0)); - assert!(read.pid.is_some(), "the response names the root pid"); - assert_eq!(chunk_text(&read.chunks), "started\n"); + assert_eq!(read.state, ExecState::Exited); + assert_eq!(read.exit, Some(ExitStatus::Code { code: 0 })); + assert!(read.pid > 0, "the response names the root pid"); + assert_eq!(page_text(&read.output.chunks), "started\n"); assert!( - read.leftover_processes + read.leftovers.as_ref().is_some_and(|leftovers| leftovers + .processes .iter() - .any(|member| i32::try_from(member.pid) == Ok(leftover_pid)), + .any(|member| i32::try_from(member.pid) == Ok(leftover_pid))), "the leftover is reported with its pid: {:?}", - read.leftover_processes + read.leftovers ); assert!( process_alive(leftover_pid), "the leftover survives the call that started it" ); - let idle = second.idle(&IdleParams {}).await?; - assert_eq!(idle.running_processes, 0); - assert_eq!(idle.leftover_process_groups, 1); - assert!(idle.is_quiescent(), "a leftover is not running work"); + let idle = second.get_idle(&GetIdleParams {}).await?; + assert_eq!(idle.running_process_count, 0); + assert_eq!(idle.leftover_process_group_count, 1); second .terminate_process(&TerminateProcessParams { - process_id: ProcessId::new("live-leftover"), + process_id: process_id("live-leftover")?, signal: ProcessSignal::Kill, }) .await?; wait_for_process_gone(leftover_pid).await?; - assert_eq!(first.idle(&IdleParams {}).await?.leftover_process_groups, 0); + assert_eq!( + first + .get_idle(&GetIdleParams {}) + .await? + .leftover_process_group_count, + 0 + ); // The daemon owns the cursor: the second socket reads only what the - // first did not, and an empty write after exit is a wait. + // first did not, a positioned read consumes nothing, and writing to a + // process without input is refused. first .start_process(&shell_process( "live-cursor", "echo one; sleep 0.4; echo two; sleep 0.4; echo three", false, - )) + )?) .await?; let head = first - .read_process(&read_params("live-cursor", Some(150))) + .read_process(&ReadProcessParams { + return_on: ReturnOn::AnyOutput, + wait_ms: 10_000, + ..read_until_exit("live-cursor")? + }) .await?; - assert!(!head.exited); - assert_eq!(chunk_text(&head.chunks), "one\n"); + assert_eq!(head.state, ExecState::Running); + assert_eq!(page_text(&head.output.chunks), "one\n"); let rest = second - .read_process(&read_params("live-cursor", Some(10_000))) + .read_process(&read_until_exit("live-cursor")?) .await?; - assert!(rest.exited); - assert_eq!(chunk_text(&rest.chunks), "two\nthree\n"); - let last_head_seq = head.chunks.last().expect("head chunk").seq; - assert!(rest.chunks.iter().all(|chunk| chunk.seq > last_head_seq)); - let wait = first - .write_process(&WriteProcessParams { - process_id: ProcessId::new("live-cursor"), - chunk: None, - close_stdin: false, + assert_eq!(rest.state, ExecState::Exited); + assert_eq!(page_text(&rest.output.chunks), "two\nthree\n"); + assert!( + rest.output + .chunks + .iter() + .all(|chunk| chunk.offset >= head.output.next) + ); + let whole = first + .read_process(&ReadProcessParams { + after: Some(0), + ..read_until_exit("live-cursor")? }) .await?; + assert_eq!(page_text(&whole.output.chunks), "one\ntwo\nthree\n"); + assert_eq!(whole.output.next, whole.output.end); + let tail = second + .read_process(&ReadProcessParams { + tail_bytes: Some(6), + ..read_until_exit("live-cursor")? + }) + .await?; + assert_eq!(page_text(&tail.output.chunks), "three\n"); + let refused = first + .write_process(&WriteProcessParams { + process_id: process_id("live-cursor")?, + data: ByteChunk::new(b"late".to_vec()), + close_input: false, + }) + .await + .expect_err("an exited process takes no input"); assert_eq!( - wait.status, - environment_protocol::data::process::WriteProcessStatus::Accepted + refused.protocol_kind(), + Some(ProtocolErrorKind::FailedPrecondition) ); // A PTY takes input written from another socket. @@ -598,27 +656,22 @@ where "live-pty", "read line; echo got:$line", true, - )) + )?) .await?; let write = second .write_process(&WriteProcessParams { - process_id: ProcessId::new("live-pty"), - chunk: Some(ByteChunk::new(b"hello\n".to_vec())), - close_stdin: false, + process_id: process_id("live-pty")?, + data: ByteChunk::new(b"hello\n".to_vec()), + close_input: false, }) .await?; - assert_eq!( - write.status, - environment_protocol::data::process::WriteProcessStatus::Accepted - ); - let pty = first - .read_process(&read_params("live-pty", Some(10_000))) - .await?; - assert!(pty.exited); + assert_eq!(write.accepted_bytes, 6); + let pty = first.read_process(&read_until_exit("live-pty")?).await?; + assert_eq!(pty.state, ExecState::Exited); assert!( - chunk_text(&pty.chunks).contains("got:hello"), + page_text(&pty.output.chunks).contains("got:hello"), "{:?}", - chunk_text(&pty.chunks) + page_text(&pty.output.chunks) ); // An interrupt reaches the group; the next read observes the trap. @@ -627,39 +680,63 @@ where "live-interrupt", "trap 'echo caught; exit 3' INT; sleep 100 & wait $!", false, - )) + )?) .await?; tokio::time::sleep(Duration::from_millis(300)).await; let signalled = first .terminate_process(&TerminateProcessParams { - process_id: ProcessId::new("live-interrupt"), + process_id: process_id("live-interrupt")?, signal: ProcessSignal::Interrupt, }) .await?; assert!(signalled.running); let caught = second - .read_process(&read_params("live-interrupt", Some(10_000))) + .read_process(&read_until_exit("live-interrupt")?) .await?; - assert!(caught.exited); - assert_eq!(caught.exit_code, Some(3)); - assert_eq!(chunk_text(&caught.chunks), "caught\n"); + assert_exited_with(&caught, 3, None, "caught\n"); + + // Terminate is graceful first: the process may clean up on SIGTERM. + first + .start_process(&shell_process( + "live-terminate", + "trap 'echo stopping; exit 7' TERM; sleep 100 & wait $!", + false, + )?) + .await?; + tokio::time::sleep(Duration::from_millis(300)).await; + second + .terminate_process(&TerminateProcessParams { + process_id: process_id("live-terminate")?, + signal: ProcessSignal::Terminate, + }) + .await?; + let terminated = first + .read_process(&read_until_exit("live-terminate")?) + .await?; + assert_exited_with(&terminated, 7, Some(StoppedBy::Terminate), "stopping\n"); Ok(()) } +fn assert_exited_with( + read: &ReadProcessResponse, + code: i32, + stopped_by: Option, + output: &str, +) { + assert_eq!(read.state, ExecState::Exited); + assert_eq!(read.exit, Some(ExitStatus::Code { code })); + assert_eq!(read.stopped_by, stopped_by); + assert_eq!(page_text(&read.output.chunks), output); +} + /// The substrate executor over the real transport: a yielded run returns /// a handle and pid, a continue reads only new output, and a kill through /// the handle ends the group. -async fn executor_scenario( - client: EnvironmentDataClient, - capabilities: environment_protocol::shared::EnvironmentCapabilities, -) -> anyhow::Result<()> -where - T: JsonRpcTransport + Send + 'static, -{ - let connection = RemoteEnvironmentConnection::new(client, capabilities); +async fn executor_scenario(client: DataClient) -> anyhow::Result<()> { + let connection = RemoteEnvironmentConnection::new(client); let executor = connection .process_executor() - .expect("the daemon negotiates process execution"); + .expect("the daemon serves the process family"); let mut request = ProcessRequest::argv(["/bin/sh", "-c", "echo hi; sleep 60"]); request.yield_ms = Some(300); request.max_output_bytes = Some(4096); @@ -679,7 +756,7 @@ where more.stdout.text_lossy() ); - let root_pid = running.pid.expect("pid") as i32; + let root_pid = i32::try_from(running.pid.expect("pid"))?; let killed = executor .continue_process(ContinueProcessRequest { handle, @@ -697,32 +774,50 @@ where Ok(()) } -fn shell_process(process_id: &str, script: &str, tty: bool) -> StartProcessParams { - StartProcessParams { - process_id: ProcessId::new(process_id), - argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(60_000), - tty, - } +fn process_id(id: &str) -> anyhow::Result { + Ok(ProcessId::new(id)?) } -fn read_params(process_id: &str, wait_ms: Option) -> ReadProcessParams { - ReadProcessParams { - process_id: ProcessId::new(process_id), - after_seq: None, - max_bytes: None, - wait_ms, - } +fn path(root: &Path, name: &str) -> anyhow::Result { + let path = root.join(name); + Ok(EnvironmentPath::new(path.to_str().ok_or_else(|| { + anyhow::anyhow!("non-utf-8 path {}", path.display()) + })?)?) +} + +fn shell_process(process_id: &str, script: &str, pty: bool) -> anyhow::Result { + Ok(StartProcessParams { + process_id: self::process_id(process_id)?, + command: CommandSpec { + argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], + cwd: None, + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input: ExecInput::None {}, + timeout_ms: Some(60_000), + }, + pty: pty.then_some(PtySize { rows: 24, cols: 80 }), + }) } -fn chunk_text(chunks: &[ProcessOutputChunk]) -> String { +/// A cursor read that returns once the process exited and its output +/// settled. +fn read_until_exit(process_id: &str) -> anyhow::Result { + Ok(ReadProcessParams { + process_id: self::process_id(process_id)?, + after: None, + tail_bytes: None, + ack: None, + max_bytes: 65_536, + wait_ms: 10_000, + return_on: ReturnOn::Exit, + }) +} + +fn page_text(chunks: &[OutputChunk]) -> String { let bytes = chunks .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) + .flat_map(|chunk| chunk.data.as_slice().to_vec()) .collect::>(); String::from_utf8_lossy(&bytes).into_owned() } @@ -764,41 +859,83 @@ async fn wait_for_process_gone(pid: i32) -> anyhow::Result<()> { } } +fn read_receipt(path: &Path) -> anyhow::Result { + Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?) +} + +async fn wait_for_receipt_admission(path: &Path, admission: &str) -> anyhow::Result<()> { + let deadline = tokio::time::Instant::now() + SETTLE; + loop { + if read_receipt(path).is_ok_and(|receipt| receipt["admission"] == admission) { + return Ok(()); + } + if tokio::time::Instant::now() > deadline { + anyhow::bail!("receipt {} never said {admission}", path.display()); + } + tokio::time::sleep(POLL).await; + } +} + +/// A canonical, empty working tree for one daemon. +fn workspace(sandbox: &Path, name: &str) -> anyhow::Result { + let root = sandbox.join(name); + std::fs::create_dir_all(&root)?; + Ok(root.canonicalize()?) +} + +/// A daemon serving `root`, with its state outside the tree it serves. fn daemon_config( root: &Path, + state_dir: &Path, connect_url: &str, key: Option<&str>, receipt: Option<&Path>, ) -> anyhow::Result { - let root = root.canonicalize()?; let mut metadata = BTreeMap::new(); metadata.insert("harbor.trialId".to_owned(), "trial-1".to_owned()); + let mut registration = RegistrationConfig::new( + connect_url.to_owned(), + environment_daemon::upgrade::resolve_discovery_url(Some(connect_url), None)?, + ); + registration.registration_key = key.map(SecretString::new); + registration.display_name = Some(DisplayName::new("registration live daemon")?); + registration.metadata = Metadata::new(metadata)?; + registration.receipt_path = receipt.map(Path::to_path_buf); Ok(DaemonConfig { listen: None, - cwd: root.clone(), - fs_root: root.clone(), - state_dir: root.join(".lightspeed-envd"), + cwd: root.to_path_buf(), + fs_root: root.to_path_buf(), + state_dir: state_dir.to_path_buf(), read_only_fs: false, - registration: Some({ - let mut registration = RegistrationConfig::new( - connect_url.to_owned(), - environment_daemon::upgrade::resolve_discovery_url(Some(connect_url), None)?, - ); - registration.registration_key = key.map(SecretString::new); - registration.display_name = Some("registration live daemon".to_owned()); - registration.metadata = metadata; - registration.receipt_path = receipt.map(Path::to_path_buf); - registration - }), + registration: Some(registration), scrubbed_env: Vec::new(), }) } -fn spawn_daemon(config: DaemonConfig) -> tokio::task::JoinHandle> { - tokio::spawn(async move { - let runtime = DaemonRuntime::new(config)?; - environment_daemon::server::run(runtime).await - }) +/// One running daemon: how it ends, and the runtime whose shutdown token +/// stops it deliberately. +struct Daemon { + exit: tokio::task::JoinHandle, + runtime: DaemonRuntime, +} + +fn spawn_daemon(config: DaemonConfig) -> anyhow::Result { + let runtime = DaemonRuntime::new(config)?; + let exit = tokio::spawn(environment_daemon::server::run(runtime.clone())); + Ok(Daemon { exit, runtime }) +} + +async fn read_key( + api: &GatewayAgentApi, + key_id: &str, +) -> anyhow::Result { + Ok(api + .read_environment_registration_key(EnvironmentRegistrationKeyReadParams { + registration_key_id: key_id.to_owned(), + }) + .await? + .result + .registration_key) } async fn wait_for_registered( diff --git a/crates/temporal-runtime/tests/profiles_live.rs b/crates/temporal-runtime/tests/profiles_live.rs index 88bdfdd1c..57522cf9e 100644 --- a/crates/temporal-runtime/tests/profiles_live.rs +++ b/crates/temporal-runtime/tests/profiles_live.rs @@ -50,12 +50,12 @@ async fn run_profile_environment_selection_live_client( ) -> anyhow::Result<()> { use std::collections::BTreeMap; - use environment_protocol::shared::EnvironmentTransport; use environments::{ - EnvironmentConnectionSpec, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, + EnvironmentEndpoint, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, PutEnvironmentProvider, PutEnvironmentProviderBinding, }; + use temporal_runtime::environments::fake_controller::{FAKE_TEMPLATE_ID, FakeController}; let store = pg_store_from_env().await?; let model = support::live::openai_live_model(); @@ -68,20 +68,17 @@ async fn run_profile_environment_selection_live_client( let binding_id = format!("binding-profile-{suffix}"); let profile_id = ProfileId::new(format!("live_selection_{suffix}")); - // Register the in-process fake provider and bind it to this universe - // directly through the store: the deployment API is deployment-scoped and - // this test drives one universe's gateway. + // Serve a fake controller on loopback, register it and bind it to this + // universe directly through the store: the deployment API is + // deployment-scoped and this test drives one universe's gateway. + let controller = FakeController::start().await?; store .put_provider(PutEnvironmentProvider { provider_id: EnvironmentProviderId::new(provider_id.clone()), display_name: Some("Live fake provider".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "in-process", - EnvironmentTransport::Provider { - provider_type: "fake".to_owned(), - }, - ), + controller_endpoint: EnvironmentEndpoint::parse(controller.controller_url())?, metadata: BTreeMap::new(), + credential: Some(auth::SecretValue::new(controller.credential())), updated_at_ms: 1, }) .await?; @@ -101,7 +98,7 @@ async fn run_profile_environment_selection_live_client( .create_environment(api::EnvironmentCreateParams { request_id: format!("independent-{suffix}"), binding_id: binding_id.clone(), - template_id: "rust-v1".into(), + template_id: FAKE_TEMPLATE_ID.into(), display_name: None, metadata: BTreeMap::new(), idle_policy: None, diff --git a/crates/temporal-runtime/tests/subagents_live.rs b/crates/temporal-runtime/tests/subagents_live.rs index 41ae3566f..4f7881ae3 100644 --- a/crates/temporal-runtime/tests/subagents_live.rs +++ b/crates/temporal-runtime/tests/subagents_live.rs @@ -1314,28 +1314,26 @@ async fn run_agent_run_inherit_environment_live_client( ) -> anyhow::Result<()> { use std::collections::BTreeMap; - use environment_protocol::shared::EnvironmentTransport; use environments::{ - EnvironmentConnectionSpec, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, + EnvironmentEndpoint, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, PutEnvironmentProvider, PutEnvironmentProviderBinding, }; + use temporal_runtime::environments::fake_controller::FakeController; let store = pg_store_from_env().await?; let suffix = uuid::Uuid::new_v4().simple().to_string(); let provider_id = format!("fake-inherit-{suffix}"); let binding_id = format!("binding-inherit-{suffix}"); + // Serves the controller and data routes until the test returns. + let controller = FakeController::start().await?; store .put_provider(PutEnvironmentProvider { provider_id: EnvironmentProviderId::new(provider_id.clone()), display_name: Some("Live fake provider".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "in-process", - EnvironmentTransport::Provider { - provider_type: "fake".to_owned(), - }, - ), + controller_endpoint: EnvironmentEndpoint::parse(controller.controller_url())?, metadata: BTreeMap::new(), + credential: Some(auth::SecretValue::new(controller.credential())), updated_at_ms: 1, }) .await?; diff --git a/crates/temporal-runtime/tests/vfs_transfer_live.rs b/crates/temporal-runtime/tests/vfs_transfer_live.rs index 2faf3451a..e1a0c5ef2 100644 --- a/crates/temporal-runtime/tests/vfs_transfer_live.rs +++ b/crates/temporal-runtime/tests/vfs_transfer_live.rs @@ -13,11 +13,13 @@ use environment_daemon::{ config::{DaemonConfig, RegistrationConfig}, }; use environment_protocol::{ + bytes::SecretString, data::{ - handshake::InitializeParams, - inventory::{ScanContent, ScanDigestAlgorithm, ScanParams}, + fs::{TreeContent, TreeFilter}, + scan::{ScanLimits, ScanParams, ScanRoot, ScanRootResult, ScanSkip, ScanSkipReason}, }, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentPath, SecretString}, + paths::EnvironmentPath, + routes::CONNECT_PATH, }; use environments::EnvironmentStore; use harness::{ @@ -112,97 +114,224 @@ async fn temporal_live_vfs_transfers_follow_profile_grants_and_publish_large_fil let mut tasks = Tasks(vec![gateway.abort_handle()]); let sandbox = tempfile::tempdir()?; let root = sandbox.path().canonicalize()?; + let daemon_state = tempfile::tempdir()?; let context = support::live::local_request_context_for(api::AccessScope::Universe { universe_id }) .await?; // Keep the large transfer scenario off the current-thread test stack. - let result = temporal_runtime::gateway::request_context::with_request_context(context, Box::pin(async { - let state = runtime.state_for(universe_id, false).await?; - let key = state.api.create_environment_registration_key(api::EnvironmentRegistrationKeyCreateParams { - display_name: "VFS transfer live".into(), - identity_mode: api::EnvironmentIdentityModeView::Ephemeral, - max_active_environments: Some(1), - ephemeral_disconnect_grace_ms: None, - expires_at_ms: None, - }).await?.result; - let connect_url = format!("ws://{address}/environment-gateway/connect"); - let mut registration = RegistrationConfig::new(connect_url.clone(), - environment_daemon::upgrade::resolve_discovery_url(Some(&connect_url), None)?); - registration.registration_key = Some(SecretString::new(key.secret.0)); - let daemon = DaemonRuntime::new(DaemonConfig { - listen: None, cwd: root.clone(), fs_root: root.clone(), state_dir: root.join(".envd"), - read_only_fs: false, registration: Some(registration), scrubbed_env: Vec::new(), - })?; - let daemon = tokio::spawn(environment_daemon::server::run(daemon)); - tasks.0.push(daemon.abort_handle()); - let environment = tokio::time::timeout(Duration::from_secs(30), async { - loop { - let environments = state.api.list_environments(api::EnvironmentListParams::default()).await?.result.environments; - if let Some(environment) = environments.into_iter().find(|e| e.status == api::EnvironmentLifecycleStatusView::Ready) { - return anyhow::Ok(environment); + let result = temporal_runtime::gateway::request_context::with_request_context( + context, + Box::pin(async { + let state = runtime.state_for(universe_id, false).await?; + let key = state + .api + .create_environment_registration_key(api::EnvironmentRegistrationKeyCreateParams { + display_name: "VFS transfer live".into(), + identity_mode: api::EnvironmentIdentityModeView::Ephemeral, + max_active_environments: Some(1), + ephemeral_disconnect_grace_ms: None, + expires_at_ms: None, + }) + .await? + .result; + let connect_url = format!("ws://{address}{CONNECT_PATH}"); + let mut registration = RegistrationConfig::new( + connect_url.clone(), + environment_daemon::upgrade::resolve_discovery_url(Some(&connect_url), None)?, + ); + registration.registration_key = Some(SecretString::new(key.secret.0)); + let daemon = DaemonRuntime::new(DaemonConfig { + listen: None, + cwd: root.clone(), + fs_root: root.clone(), + state_dir: daemon_state.path().into(), + read_only_fs: false, + registration: Some(registration), + scrubbed_env: Vec::new(), + })?; + let daemon = tokio::spawn(environment_daemon::server::run(daemon)); + tasks.0.push(daemon.abort_handle()); + let environment = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let environments = state + .api + .list_environments(api::EnvironmentListParams::default()) + .await? + .result + .environments; + if let Some(environment) = environments + .into_iter() + .find(|e| e.status == api::EnvironmentLifecycleStatusView::Ready) + { + return anyhow::Ok(environment); + } + tokio::time::sleep(Duration::from_millis(100)).await; } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }).await??; - let hosted = Arc::new(SessionTools::from_pg_store(state.store.clone()) - .with_environment_gateway(gateway_config.clone())); - let activities = WorkerActivities::for_universe(universe_id, - ActivityState::from_pg_store(state.store.clone(), Arc::new(TransferLlm { blobs: state.store.clone() }), hosted.clone()) - .with_hosted_tools(hosted).with_environment_gateway(gateway_config.clone())); - let gateway_config = gateway_config.clone(); - let store = state.store.clone(); - run_with_live_worker(activities, |client, queue, base_session| async move { - let api = GatewayAgentApi::builder(client.clone(), store.clone()) - .with_task_queue(queue).with_environment_gateway(gateway_config.clone()).build(); - // All provider surfaces execute both directions. Additional sessions - // test the public grant boundary using the same profile path. - for (index, (provider, mode)) in [ - (ProviderApiKind::OpenAiResponses, "edit"), - (ProviderApiKind::AnthropicMessages, "edit"), - (ProviderApiKind::OpenAiCompletions, "edit"), - (ProviderApiKind::OpenAiResponses, "readonly"), - (ProviderApiKind::OpenAiResponses, "sourcing"), - (ProviderApiKind::OpenAiResponses, "noenv"), - (ProviderApiKind::OpenAiResponses, "content"), - (ProviderApiKind::AnthropicMessages, "content"), - (ProviderApiKind::OpenAiCompletions, "content"), - (ProviderApiKind::OpenAiResponses, "contentread"), - ].into_iter().enumerate() { - let session = SessionId::new(format!("{base_session}_{index}_{mode}")); - let case = Box::pin(run_case(&api, store.as_ref(), &session, provider, mode, index, &root, &environment.environment_id)).await; - let handle = client.get_workflow_handle::( - temporal_workflow::compose_workflow_id(universe_id, &session)); - let _ = handle.terminate(WorkflowTerminateOptions::builder().reason("transfer live cleanup").build()).await; - case?; - } - // Scan negotiation and conditional observations use the same reverse route. - let record = store.read_environment(&harness::EnvironmentId::new(&environment.environment_id)).await?; - let connection = gateway_config.connection_for(universe_id, &record); - let mut remote = environment_client::EnvironmentDataClient::connect(&connection.endpoint, gateway_config.connect_options("transfer-live-scan")).await?; - let initialized = remote.initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, client_name: "transfer-live-scan".into(), - scope: connection.scope, resume_connection_id: None, - }).await?; - anyhow::ensure!(initialized.capabilities.filesystem_scan && initialized.capabilities.filesystem_transfer); - let mut query = ScanParams { - follow_symlinks: false, - roots: vec![EnvironmentPath::new("./capture-source")?], include_patterns: vec!["new.bin".into()], - read_content: false, digest_algorithm: Some(ScanDigestAlgorithm::Sha256), limits: Default::default(), if_none_match: None, - }; - let scan = remote.scan(&query).await?; - anyhow::ensure!(scan.complete && scan.diagnostics.is_empty()); - assert!(scan.entries.iter().any(|entry| matches!(&entry.content, ScanContent::File { digest: Some(digest), .. } - if digest == BlobRef::from_bytes(&std::fs::read(root.join("capture-source/new.bin")).unwrap()).as_str()))); - query.if_none_match = Some(scan.fingerprint.expect("complete scan fingerprint")); - let unchanged = remote.scan(&query).await?; - anyhow::ensure!(unchanged.unchanged && unchanged.entries.is_empty()); - std::fs::write(root.join("capture-source/new.bin"), b"changed after observation")?; - let changed = remote.scan(&query).await?; - anyhow::ensure!(changed.complete && !changed.unchanged && changed.fingerprint != query.if_none_match); - remote.close().await?; - anyhow::Ok(()) - }).await - })).await; + }) + .await??; + let hosted = Arc::new( + SessionTools::from_pg_store(state.store.clone()) + .with_environment_gateway(gateway_config.clone()), + ); + let activities = WorkerActivities::for_universe( + universe_id, + ActivityState::from_pg_store( + state.store.clone(), + Arc::new(TransferLlm { + blobs: state.store.clone(), + }), + hosted.clone(), + ) + .with_hosted_tools(hosted) + .with_environment_gateway(gateway_config.clone()), + ); + let gateway_config = gateway_config.clone(); + let store = state.store.clone(); + run_with_live_worker(activities, |client, queue, base_session| async move { + let api = GatewayAgentApi::builder(client.clone(), store.clone()) + .with_task_queue(queue) + .with_environment_gateway(gateway_config.clone()) + .build(); + // All provider surfaces execute both directions. Additional sessions + // test the public grant boundary using the same profile path. + for (index, (provider, mode)) in [ + (ProviderApiKind::OpenAiResponses, "edit"), + (ProviderApiKind::AnthropicMessages, "edit"), + (ProviderApiKind::OpenAiCompletions, "edit"), + (ProviderApiKind::OpenAiResponses, "readonly"), + (ProviderApiKind::OpenAiResponses, "sourcing"), + (ProviderApiKind::OpenAiResponses, "noenv"), + (ProviderApiKind::OpenAiResponses, "content"), + (ProviderApiKind::AnthropicMessages, "content"), + (ProviderApiKind::OpenAiCompletions, "content"), + (ProviderApiKind::OpenAiResponses, "contentread"), + ] + .into_iter() + .enumerate() + { + let session = SessionId::new(format!("{base_session}_{index}_{mode}")); + let case = Box::pin(run_case( + &api, + store.as_ref(), + &session, + provider, + mode, + index, + &root, + &environment.environment_id, + )) + .await; + let handle = client + .get_workflow_handle::( + temporal_workflow::compose_workflow_id(universe_id, &session), + ); + let _ = handle + .terminate( + WorkflowTerminateOptions::builder() + .reason("transfer live cleanup") + .build(), + ) + .await; + case?; + } + // Scan negotiation and conditional observations use the same reverse route. + let record = store + .read_environment(&harness::EnvironmentId::new(&environment.environment_id)) + .await?; + let (remote, initialized) = environment_client::DataClient::connect_initialized( + &gateway_config.endpoint_for(universe_id, &record), + gateway_config.connect_options("transfer-live-scan"), + ) + .await?; + anyhow::ensure!(initialized.capabilities.filesystem.is_some()); + let ceilings = initialized.limits.filesystem.scan; + // Scans observe small files such as skills and prompts: the + // large captured file is listed as skipped, never hashed, + // while a small sibling is observed with its digest. + std::fs::write(root.join("capture-source/observed.txt"), b"observed")?; + let mut query = ScanParams { + roots: vec![ScanRoot { + path: EnvironmentPath::new(root.join("capture-source").to_string_lossy())?, + if_none_match: None, + max_depth: None, + }], + filter: TreeFilter { + include: vec!["new.bin".into(), "observed.txt".into()], + ..TreeFilter::default() + }, + read_content: false, + include_digests: true, + follow_symlinks: false, + limits: ScanLimits { + max_entries: ceilings.max_entries, + max_file_bytes: ceilings.max_file_bytes, + max_total_bytes: ceilings.max_total_bytes, + max_duration_ms: ceilings.max_duration_ms, + }, + }; + anyhow::ensure!( + std::fs::metadata(root.join("capture-source/new.bin"))?.len() + > ceilings.max_file_bytes, + "the captured file must exceed the scan's per-file ceiling" + ); + let expected = + BlobRef::from_bytes(&std::fs::read(root.join("capture-source/observed.txt"))?); + let scan = remote.scan(&query).await?; + let fingerprint = match scan.roots.as_slice() { + [ + ScanRootResult::Complete { + fingerprint, + entries, + skipped, + skipped_entry_count, + .. + }, + ] => { + anyhow::ensure!( + *skipped_entry_count == 1 + && matches!(skipped.as_slice(), [ScanSkip { + path: Some(path), + reason: ScanSkipReason::FileTooLarge, + }] if path.as_str() == "new.bin"), + "{scan:?}" + ); + anyhow::ensure!( + matches!(entries.as_slice(), [entry] + if entry.path.as_str() == "observed.txt" + && matches!(&entry.content, TreeContent::File { digest: Some(digest), .. } + if format!("sha256:{}", digest.hex()) == expected.as_str())), + "{scan:?}" + ); + fingerprint.clone() + } + other => anyhow::bail!("capture-source scan did not complete: {other:?}"), + }; + query.roots[0].if_none_match = Some(fingerprint.clone()); + let unchanged = remote.scan(&query).await?; + anyhow::ensure!( + matches!( + unchanged.roots.as_slice(), + [ScanRootResult::Unchanged { .. }] + ), + "{unchanged:?}" + ); + std::fs::write( + root.join("capture-source/observed.txt"), + b"changed after observation", + )?; + let changed = remote.scan(&query).await?; + anyhow::ensure!( + matches!(changed.roots.as_slice(), + [ScanRootResult::Complete { fingerprint: changed, .. }] if *changed != fingerprint), + "{changed:?}" + ); + remote.close().await?; + anyhow::Ok(()) + }) + .await + }), + ) + .await; drop(tasks); let store = runtime.state_for(universe_id, false).await?.store.clone(); runtime.evict(universe_id).await; diff --git a/crates/temporal-workflow/Cargo.toml b/crates/temporal-workflow/Cargo.toml index b6d5bfce0..49f5fac0b 100644 --- a/crates/temporal-workflow/Cargo.toml +++ b/crates/temporal-workflow/Cargo.toml @@ -11,7 +11,6 @@ harness = { path = "../harness", features = ["contract"] } schemars = { version = "1", features = ["uuid1"] } sha2 = "0.11" hex = "0.4" -environment-protocol = { path = "../environment-protocol" } anyhow = "1" chrono = { version = "0.4", default-features = false, features = ["std"] } futures = "0.3" diff --git a/crates/temporal-workflow/src/activities.rs b/crates/temporal-workflow/src/activities.rs index 603c7762a..f468a63d8 100644 --- a/crates/temporal-workflow/src/activities.rs +++ b/crates/temporal-workflow/src/activities.rs @@ -9,10 +9,11 @@ use crate::{ AppendEventsRequest, AwaitEnvironmentReadyActivityRequest, AwaitEnvironmentReadyActivityResult, AwaitMaterializationRequest, AwaitMaterializationResult, ContextCompactActivityRequest, CreateOrLoadSessionRequest, CreateOrLoadSessionResult, EnvironmentJobCancelActivityRequest, - EnvironmentJobPollActivityRequest, EnvironmentJobPollActivityResult, - EnvironmentJobPrepareWorkflowToolRequest, EnvironmentJobStartActivityRequest, - EnvironmentJobStartActivityResult, JoinedContextPreparationRequest, LlmGenerateActivityRequest, - PutBlobRequest, ReadBlobRequest, ReadBlobResult, RuntimeProjectionRefreshActivityRequest, + EnvironmentJobCancelActivityResult, EnvironmentJobPollActivityRequest, + EnvironmentJobPollActivityResult, EnvironmentJobPrepareWorkflowToolRequest, + EnvironmentJobStartActivityRequest, EnvironmentJobStartActivityResult, + JoinedContextPreparationRequest, LlmGenerateActivityRequest, PutBlobRequest, ReadBlobRequest, + ReadBlobResult, RuntimeProjectionRefreshActivityRequest, RuntimeProjectionRefreshActivityResult, SubagentCloseActivityRequest, SubagentPrepareActivityRequest, SubagentPrepareActivityResult, SubagentResolveActivityRequest, ToolInvokeBatchActivityRequest, ToolInvokeCallActivityRequest, ToolInvokeCallActivityResult, @@ -270,7 +271,7 @@ impl WorkflowActivities { pub async fn environment_job_cancel( _ctx: ActivityContext, _request: EnvironmentJobCancelActivityRequest, - ) -> Result, ActivityError> { + ) -> Result { unimplemented!("workflow activity definition only") } diff --git a/crates/temporal-workflow/src/environment_jobs.rs b/crates/temporal-workflow/src/environment_jobs.rs new file mode 100644 index 000000000..09ab7b321 --- /dev/null +++ b/crates/temporal-workflow/src/environment_jobs.rs @@ -0,0 +1,495 @@ +//! Records and activity contracts of `EnvironmentJobWorkflow`. +//! +//! The workflow owns every type it persists in history (arguments, +//! continue-as-new state, local-activity results, signals and the start +//! payload). None of them is an environment protocol wire type: the job +//! activities map between these records and `job/*` at their boundary, so a +//! protocol change never changes a history shape. + +use std::collections::BTreeMap; + +use harness::{BlobRef, PromiseSourceCheckResult}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::WorkflowToolStartArgs; + +/// A job id on the environment's daemon (`jobId` on the wire). The start +/// activity validates it when it maps the payload onto `job/start`. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct EnvironmentJobId(String); + +impl EnvironmentJobId { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +impl std::fmt::Display for EnvironmentJobId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// Lifecycle of one job as the workflow records it. Every daemon status has +/// a variant; `lost` is the workflow's own outcome for a job the daemon no +/// longer knows. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentJobStatus { + /// Waiting for dependencies or its queue. + Queued, + Running, + /// Exited with code 0. + Succeeded, + /// Exited otherwise, or could not start. + Failed, + /// Its timeout expired and the daemon stopped it. + TimedOut, + /// Stopped or never started because of a cancel. + Cancelled, + /// A dependency did not finish as the dependency policy requires. + DependencyFailed, + /// The daemon stopped while the job was queued or running. + Interrupted, + /// The daemon answered `notFound` for the job: it never existed there or + /// its record was garbage-collected. + Lost, +} + +impl EnvironmentJobStatus { + /// `queued` and `running` are live; every other status is final. + pub fn is_terminal(self) -> bool { + !matches!(self, Self::Queued | Self::Running) + } + + pub fn is_success(self) -> bool { + self == Self::Succeeded + } +} + +/// What the workflow keeps about one job. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentJobRecord { + pub job_id: EnvironmentJobId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub status: EnvironmentJobStatus, + /// Resolved ids of the jobs this one depends on. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub depends_on: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, +} + +impl EnvironmentJobRecord { + /// The record of a job the daemon answered `notFound` for and the + /// workflow holds nothing else about. + pub fn lost(job_id: EnvironmentJobId) -> Self { + Self { + job_id, + name: None, + status: EnvironmentJobStatus::Lost, + depends_on: Vec::new(), + queue_key: None, + } + } +} + +/// One job as a poll or cancel activity observed it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind", deny_unknown_fields)] +pub enum EnvironmentJobObservation { + /// The daemon's summary, mapped onto the workflow's record. + Job { record: EnvironmentJobRecord }, + /// The daemon answered this entry with `notFound`. The workflow records + /// the job as `lost`, a terminal failure. + NotFound { job_id: EnvironmentJobId }, +} + +impl EnvironmentJobObservation { + pub fn job_id(&self) -> &EnvironmentJobId { + match self { + Self::Job { record } => &record.job_id, + Self::NotFound { job_id } => job_id, + } + } +} + +/// Which jobs a cancel affects. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentJobCancelScope { + /// Only the named jobs. + #[default] + Job, + /// The named jobs and every job that transitively depends on them. + Dependents, +} + +/// Start payload of a job group, stored in CAS and named by +/// [`EnvironmentJobStartActivityRequest::request_ref`]. Written once by the +/// starter, read by the start activity, which resolves credential-bound +/// secret variables and maps it onto `job/start`. It never holds secret +/// values. Its content address is the stable fingerprint of the request, so +/// a starter derives the job-group id (and with it the workflow id) from the +/// encoded payload. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentJobStartPayloadV1 { + /// [`Self::VERSION`]. + pub version: u32, + /// Idempotency key of the start (`requestId` on the wire): a retried + /// start with the same id and jobs replays the existing jobs. + pub request_id: String, + /// Never empty. + pub jobs: Vec, +} + +impl EnvironmentJobStartPayloadV1 { + pub const VERSION: u32 = 1; + + pub fn new(request_id: impl Into, jobs: Vec) -> Self { + Self { + version: Self::VERSION, + request_id: request_id.into(), + jobs, + } + } + + /// The bytes a starter stores in CAS. + pub fn encode(&self) -> Result, serde_json::Error> { + serde_json::to_vec(self) + } + + /// Decode a stored payload, checking its version before its shape. + pub fn decode(bytes: &[u8]) -> Result { + #[derive(Deserialize)] + struct VersionProbe { + version: u32, + } + let probe: VersionProbe = + serde_json::from_slice(bytes).map_err(EnvironmentJobStartPayloadError::Decode)?; + if probe.version != Self::VERSION { + return Err(EnvironmentJobStartPayloadError::UnsupportedVersion { + version: probe.version, + }); + } + serde_json::from_slice(bytes).map_err(EnvironmentJobStartPayloadError::Decode) + } + + pub fn job_ids(&self) -> Vec { + self.jobs.iter().map(|job| job.job_id.clone()).collect() + } +} + +/// Why a stored start payload could not be read. +#[derive(Debug)] +pub enum EnvironmentJobStartPayloadError { + Decode(serde_json::Error), + UnsupportedVersion { version: u32 }, +} + +impl std::fmt::Display for EnvironmentJobStartPayloadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Decode(error) => write!(f, "decode environment job start payload: {error}"), + Self::UnsupportedVersion { version } => write!( + f, + "unsupported environment job start payload version {version} (expected {})", + EnvironmentJobStartPayloadV1::VERSION + ), + } + } +} + +impl std::error::Error for EnvironmentJobStartPayloadError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Decode(error) => Some(error), + Self::UnsupportedVersion { .. } => None, + } + } +} + +/// One job of a start payload. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentJobSpec { + pub job_id: EnvironmentJobId, + /// Local name other jobs of the same start can depend on. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub command: EnvironmentJobCommand, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub depends_on: Vec, + #[serde( + default, + skip_serializing_if = "EnvironmentJobDependencyPolicy::is_default" + )] + pub dependency_policy: EnvironmentJobDependencyPolicy, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, +} + +/// What a job runs. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentJobCommand { + /// Program and arguments. Not empty. + pub argv: Vec, + /// Absolute working directory in the environment; the daemon's default + /// when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cwd: Option, + /// Plain variables. Credential-bound variables are resolved by the start + /// activity and never stored here. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, + /// Text written to standard input, then end of file; `/dev/null` when + /// absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stdin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + +/// A dependency of a job. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind", deny_unknown_fields)] +pub enum EnvironmentJobDependency { + /// An existing job on the daemon, or one in the same start. + JobId { job_id: EnvironmentJobId }, + /// The `name` of a job in the same start. + Name { name: String }, +} + +/// What a job waits for from its dependencies. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentJobDependencyPolicy { + /// Run once every dependency succeeded; otherwise `dependency_failed`. + #[default] + AllSucceeded, + /// Run once every dependency is terminal, whatever its status. + AllTerminal, +} + +impl EnvironmentJobDependencyPolicy { + pub fn is_default(&self) -> bool { + *self == Self::AllSucceeded + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobStartActivityRequest { + pub universe_id: Uuid, + pub environment_id: String, + pub job_group_id: String, + /// CAS reference of the encoded [`EnvironmentJobStartPayloadV1`]. + pub request_ref: BlobRef, +} + +/// The started jobs, one record per payload job. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobStartActivityResult { + pub jobs: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobSubscription { + pub holder_workflow_id: String, + pub promise_id: String, + pub completion_key: String, + pub job_id: EnvironmentJobId, + #[serde(default)] + pub notified: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobWorkflowArgs { + pub universe_id: Uuid, + pub start: EnvironmentJobStartActivityRequest, + pub job_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub subscriptions: Vec, + #[serde(default)] + pub started: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub jobs: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub resolutions: BTreeMap, + #[serde(default = "default_environment_job_poll_ms")] + pub poll_ms: u64, + #[serde(default)] + pub poll_attempt: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workflow_tool: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobWorkflowToolContext { + pub execution_id: String, + pub invocation_id: harness::WorkflowToolInvocationId, +} + +/// Accepted inputs for the core job workflow. Bare public starts arrive as +/// fully prepared job arguments; internally supervised starts use the fixed +/// internally supervised input and are normalized by an activity before +/// provider work begins. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum EnvironmentJobWorkflowInput { + WorkflowTool(WorkflowToolStartArgs), + Job(EnvironmentJobWorkflowArgs), +} + +impl From for EnvironmentJobWorkflowInput { + fn from(value: EnvironmentJobWorkflowArgs) -> Self { + Self::Job(value) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobPrepareWorkflowToolRequest { + pub start: WorkflowToolStartArgs, +} + +fn default_environment_job_poll_ms() -> u64 { + 2_000 +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobWorkflowSnapshot { + pub environment_id: String, + pub job_group_id: String, + #[serde(default)] + pub started: bool, + #[serde(default)] + pub jobs: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub resolutions: BTreeMap, + /// Every job of the group has a terminal record and a recorded + /// resolution. + pub terminal: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, +} + +/// Poll of the group's jobs that are not settled yet. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobPollActivityRequest { + pub universe_id: Uuid, + pub environment_id: String, + pub job_group_id: String, + pub job_ids: Vec, +} + +/// One observation per polled job. `resolutions` is keyed by job id and +/// holds the promise result of every job observed terminal, `NotFound` +/// included (a `Failed` resolution carrying the model-facing result); the +/// workflow falls back to a `Failed` resolution without a payload for a +/// `NotFound` job that has none. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobPollActivityResult { + pub jobs: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub resolutions: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobCancelSignal { + pub jobs: Vec, + #[serde(default)] + pub scope: EnvironmentJobCancelScope, + #[serde(default)] + pub force: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobCancelActivityRequest { + pub universe_id: Uuid, + pub environment_id: String, + pub jobs: Vec, + pub scope: EnvironmentJobCancelScope, + pub force: bool, +} + +/// One observation per job the cancel listed (named jobs first, then +/// dependents). Observations of jobs outside the group are ignored. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobCancelActivityResult { + pub jobs: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_payload_round_trips_and_checks_its_version() { + let payload = EnvironmentJobStartPayloadV1::new( + "jobreq:1", + vec![EnvironmentJobSpec { + job_id: EnvironmentJobId::new("build"), + name: Some("build".to_owned()), + command: EnvironmentJobCommand { + argv: vec!["make".to_owned()], + cwd: Some("/workspace".to_owned()), + env: BTreeMap::new(), + stdin: None, + timeout_ms: Some(60_000), + }, + depends_on: vec![EnvironmentJobDependency::Name { + name: "fetch".to_owned(), + }], + dependency_policy: EnvironmentJobDependencyPolicy::AllTerminal, + queue_key: None, + }], + ); + let bytes = payload.encode().expect("encode payload"); + assert_eq!( + EnvironmentJobStartPayloadV1::decode(&bytes).expect("decode payload"), + payload + ); + + let mut future: serde_json::Value = serde_json::from_slice(&bytes).expect("json"); + future["version"] = serde_json::json!(2); + future["extra"] = serde_json::json!(true); + assert!(matches!( + EnvironmentJobStartPayloadV1::decode(&serde_json::to_vec(&future).expect("encode")), + Err(EnvironmentJobStartPayloadError::UnsupportedVersion { version: 2 }) + )); + } + + #[test] + fn only_live_statuses_are_not_terminal() { + use EnvironmentJobStatus::*; + for status in [Queued, Running] { + assert!(!status.is_terminal()); + } + for status in [ + Succeeded, + Failed, + TimedOut, + Cancelled, + DependencyFailed, + Interrupted, + Lost, + ] { + assert!(status.is_terminal()); + } + assert!(Succeeded.is_success()); + assert!(!Lost.is_success()); + } +} diff --git a/crates/temporal-workflow/src/lib.rs b/crates/temporal-workflow/src/lib.rs index 57ce0f75d..cef7a3453 100644 --- a/crates/temporal-workflow/src/lib.rs +++ b/crates/temporal-workflow/src/lib.rs @@ -5,6 +5,18 @@ mod code_execution; mod code_tools; pub use code_tools::*; mod config; +mod environment_jobs; +pub use environment_jobs::{ + EnvironmentJobCancelActivityRequest, EnvironmentJobCancelActivityResult, + EnvironmentJobCancelScope, EnvironmentJobCancelSignal, EnvironmentJobCommand, + EnvironmentJobDependency, EnvironmentJobDependencyPolicy, EnvironmentJobId, + EnvironmentJobObservation, EnvironmentJobPollActivityRequest, EnvironmentJobPollActivityResult, + EnvironmentJobPrepareWorkflowToolRequest, EnvironmentJobRecord, EnvironmentJobSpec, + EnvironmentJobStartActivityRequest, EnvironmentJobStartActivityResult, + EnvironmentJobStartPayloadError, EnvironmentJobStartPayloadV1, EnvironmentJobStatus, + EnvironmentJobSubscription, EnvironmentJobWorkflowArgs, EnvironmentJobWorkflowInput, + EnvironmentJobWorkflowSnapshot, EnvironmentJobWorkflowToolContext, +}; mod rehydrate; mod session_preparation; mod temporal_helpers; @@ -62,28 +74,22 @@ pub use types::{ AwaitEnvironmentReadyActivityRequest, AwaitEnvironmentReadyActivityResult, AwaitMaterializationRequest, AwaitMaterializationResult, AwaitOutcome, AwaitPromiseResult, CancellingWatchdog, ContextCompactActivityRequest, CreateOrLoadSessionRequest, - CreateOrLoadSessionResult, EnvironmentJobCancelActivityRequest, EnvironmentJobCancelSignal, - EnvironmentJobPollActivityRequest, EnvironmentJobPollActivityResult, - EnvironmentJobPrepareWorkflowToolRequest, EnvironmentJobStartActivityRequest, - EnvironmentJobStartActivityResult, EnvironmentJobStartPayload, EnvironmentJobSubscription, - EnvironmentJobWorkflowArgs, EnvironmentJobWorkflowInput, EnvironmentJobWorkflowSnapshot, - EnvironmentJobWorkflowToolContext, JoinedContextPreparationRequest, - LLM_PROVIDER_TRANSIENT_ERROR_TYPE, LLM_TRANSIENT_FAILURE_DETAILS_VERSION, - LlmGenerateActivityRequest, LlmTransientFailureDetails, PendingEmission, - PendingPromiseCancellation, PendingSourceResolution, PendingToolBatchResume, PromiseSourcePoll, - PutBlobRequest, ReadBlobRequest, ReadBlobResult, RuntimeProjectionRefreshActivityRequest, - RuntimeProjectionRefreshActivityResult, SessionBootstrapPayloadTooLarge, SubagentChildRef, - SubagentCloseActivityRequest, SubagentExecutionPhase, SubagentExecutionSnapshot, - SubagentPrepareActivityRequest, SubagentPrepareActivityResult, SubagentResolveActivityRequest, - SubagentTerminal, ToolInvokeBatchActivityRequest, ToolInvokeCallActivityRequest, - ToolInvokeCallActivityResult, ToolPreparePromiseControlsActivityRequest, - WORKFLOW_TOOL_RECIPE_FINGERPRINT_PREFIX, WORKFLOW_TOOL_RECIPE_FORMAT_V1, - WORKFLOW_TOOL_RECOVERY_QUERY, WorkflowToolExecutionCancelRequest, - WorkflowToolExecutionCheckRequest, WorkflowToolRecipeV1, WorkflowToolRecoveryResult, - WorkflowToolReplyValidationRequest, WorkflowToolReplyValidationResult, - WorkflowToolStartActivityRequest, WorkflowToolStartActivityResult, WorkflowToolStartArgs, - compose_environment_job_workflow_id, compose_workflow_id, split_workflow_id, - workflow_tool_recipe_fingerprint, + CreateOrLoadSessionResult, JoinedContextPreparationRequest, LLM_PROVIDER_TRANSIENT_ERROR_TYPE, + LLM_TRANSIENT_FAILURE_DETAILS_VERSION, LlmGenerateActivityRequest, LlmTransientFailureDetails, + PendingEmission, PendingPromiseCancellation, PendingSourceResolution, PendingToolBatchResume, + PromiseSourcePoll, PutBlobRequest, ReadBlobRequest, ReadBlobResult, + RuntimeProjectionRefreshActivityRequest, RuntimeProjectionRefreshActivityResult, + SessionBootstrapPayloadTooLarge, SubagentChildRef, SubagentCloseActivityRequest, + SubagentExecutionPhase, SubagentExecutionSnapshot, SubagentPrepareActivityRequest, + SubagentPrepareActivityResult, SubagentResolveActivityRequest, SubagentTerminal, + ToolInvokeBatchActivityRequest, ToolInvokeCallActivityRequest, ToolInvokeCallActivityResult, + ToolPreparePromiseControlsActivityRequest, WORKFLOW_TOOL_RECIPE_FINGERPRINT_PREFIX, + WORKFLOW_TOOL_RECIPE_FORMAT_V1, WORKFLOW_TOOL_RECOVERY_QUERY, + WorkflowToolExecutionCancelRequest, WorkflowToolExecutionCheckRequest, WorkflowToolRecipeV1, + WorkflowToolRecoveryResult, WorkflowToolReplyValidationRequest, + WorkflowToolReplyValidationResult, WorkflowToolStartActivityRequest, + WorkflowToolStartActivityResult, WorkflowToolStartArgs, compose_environment_job_workflow_id, + compose_workflow_id, split_workflow_id, workflow_tool_recipe_fingerprint, }; pub use workflows::bots; pub use workflows::channels; diff --git a/crates/temporal-workflow/src/types.rs b/crates/temporal-workflow/src/types.rs index 1994cc1ee..d24f73735 100644 --- a/crates/temporal-workflow/src/types.rs +++ b/crates/temporal-workflow/src/types.rs @@ -400,79 +400,6 @@ pub struct CancellingWatchdog { pub since_ms: u64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobStartActivityRequest { - pub universe_id: Uuid, - pub environment_id: String, - pub job_group_id: String, - pub request_ref: BlobRef, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobStartPayload { - pub request: environment_protocol::data::jobs::StartJobsParams, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobStartActivityResult { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobSubscription { - pub holder_workflow_id: String, - pub promise_id: String, - pub completion_key: String, - pub job_id: environment_protocol::shared::JobId, - #[serde(default)] - pub notified: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobWorkflowArgs { - pub universe_id: Uuid, - pub start: EnvironmentJobStartActivityRequest, - pub job_ids: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub subscriptions: Vec, - #[serde(default)] - pub started: bool, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub jobs: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub resolutions: BTreeMap, - #[serde(default = "default_environment_job_poll_ms")] - pub poll_ms: u64, - #[serde(default)] - pub poll_attempt: u32, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub workflow_tool: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobWorkflowToolContext { - pub execution_id: String, - pub invocation_id: harness::WorkflowToolInvocationId, -} - -/// Accepted inputs for the core job workflow. Bare public starts arrive as -/// fully prepared job arguments; internally supervised starts use the fixed -/// internally supervised input and are normalized by an activity before -/// provider work begins. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum EnvironmentJobWorkflowInput { - WorkflowTool(WorkflowToolStartArgs), - Job(EnvironmentJobWorkflowArgs), -} - -impl From for EnvironmentJobWorkflowInput { - fn from(value: EnvironmentJobWorkflowArgs) -> Self { - Self::Job(value) - } -} - -/// Sub-agent execution: one delegation supervised by /// `SubagentExecutionWorkflow`, started on call by the parent session. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct SubagentPrepareActivityRequest { @@ -547,62 +474,6 @@ pub struct SubagentExecutionSnapshot { pub resolution: Option, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobPrepareWorkflowToolRequest { - pub start: WorkflowToolStartArgs, -} - -fn default_environment_job_poll_ms() -> u64 { - 2_000 -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobWorkflowSnapshot { - pub environment_id: String, - pub job_group_id: String, - #[serde(default)] - pub started: bool, - #[serde(default)] - pub jobs: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub resolutions: BTreeMap, - pub terminal: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub last_error: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobPollActivityRequest { - pub universe_id: Uuid, - pub environment_id: String, - pub job_group_id: String, - pub job_ids: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobPollActivityResult { - pub jobs: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub resolutions: BTreeMap, - pub terminal: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobCancelSignal { - pub jobs: Vec, - pub scope: environment_protocol::data::jobs::JobCancelScope, - pub force: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobCancelActivityRequest { - pub universe_id: Uuid, - pub environment_id: String, - pub jobs: Vec, - pub scope: environment_protocol::data::jobs::JobCancelScope, - pub force: bool, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct CreateOrLoadSessionRequest { pub session_id: SessionId, diff --git a/crates/temporal-workflow/src/workflows/environment_job.rs b/crates/temporal-workflow/src/workflows/environment_job.rs index d948e7fb5..79927389d 100644 --- a/crates/temporal-workflow/src/workflows/environment_job.rs +++ b/crates/temporal-workflow/src/workflows/environment_job.rs @@ -10,10 +10,12 @@ use temporalio_sdk::{ use crate::{ AgentSessionWorkflow, DEFAULT_CONTINUE_AS_NEW_HISTORY_THRESHOLD, - EnvironmentJobCancelActivityRequest, EnvironmentJobCancelSignal, - EnvironmentJobPollActivityRequest, EnvironmentJobSubscription, EnvironmentJobWorkflowInput, - EnvironmentJobWorkflowSnapshot, EnvironmentJobWorkflowToolContext, WorkflowActivities, - WorkflowToolRecoveryResult, compose_environment_job_workflow_id, + EnvironmentJobCancelActivityRequest, EnvironmentJobCancelScope, EnvironmentJobCancelSignal, + EnvironmentJobId, EnvironmentJobObservation, EnvironmentJobPollActivityRequest, + EnvironmentJobPollActivityResult, EnvironmentJobRecord, EnvironmentJobStatus, + EnvironmentJobSubscription, EnvironmentJobWorkflowInput, EnvironmentJobWorkflowSnapshot, + EnvironmentJobWorkflowToolContext, WorkflowActivities, WorkflowToolRecoveryResult, + compose_environment_job_workflow_id, }; #[workflow] @@ -82,6 +84,7 @@ impl EnvironmentJobWorkflow { state.snapshot.started = args.started; state.snapshot.jobs = args.jobs.clone(); state.snapshot.resolutions = args.resolutions.clone(); + refresh_terminal(&mut state.snapshot, &args.job_ids); state.workflow_tool = args.workflow_tool.clone(); for subscription in args.subscriptions.drain(..) { if !state.subscriptions.iter().any(|existing| { @@ -137,17 +140,9 @@ impl EnvironmentJobWorkflow { ) .await { - Ok(jobs) => ctx.state_mut(|state| { - for job in jobs { - if let Some(existing) = state - .snapshot - .jobs - .iter_mut() - .find(|existing| existing.job_id == job.job_id) - { - *existing = job; - } - } + Ok(result) => ctx.state_mut(|state| { + apply_observations(&mut state.snapshot, &args.job_ids, result.jobs); + refresh_terminal(&mut state.snapshot, &args.job_ids); state.snapshot.last_error = None; }), Err(error) => { @@ -156,7 +151,8 @@ impl EnvironmentJobWorkflow { } } - if !ctx.state(|state| state.snapshot.terminal) { + let unsettled = ctx.state(|state| unsettled_job_ids(&state.snapshot, &args.job_ids)); + if !unsettled.is_empty() { match ctx .execute_local_activity( WorkflowActivities::environment_job_poll, @@ -164,7 +160,7 @@ impl EnvironmentJobWorkflow { universe_id: args.universe_id, environment_id: args.start.environment_id.clone(), job_group_id: args.start.job_group_id.clone(), - job_ids: args.job_ids.clone(), + job_ids: unsettled, }, environment_job_activity_options(), ) @@ -172,9 +168,7 @@ impl EnvironmentJobWorkflow { { Ok(result) => { ctx.state_mut(|state| { - state.snapshot.jobs = result.jobs; - state.snapshot.resolutions.extend(result.resolutions); - state.snapshot.terminal = result.terminal; + apply_poll_result(&mut state.snapshot, &args.job_ids, result); state.snapshot.last_error = None; }); } @@ -298,7 +292,7 @@ async fn cancel_workflow_jobs( universe_id: args.universe_id, environment_id: args.start.environment_id.clone(), jobs, - scope: environment_protocol::data::jobs::JobCancelScope::Job, + scope: EnvironmentJobCancelScope::Job, force: false, }, environment_job_activity_options(), @@ -306,6 +300,86 @@ async fn cancel_workflow_jobs( .await; } +/// Apply a poll: record the resolutions, then the observations. A job the +/// daemon answered `notFound` for is `lost`; when the activity returned no +/// resolution for it, the job still settles as a failure without a payload, +/// so its subscribers are released. +fn apply_poll_result( + snapshot: &mut EnvironmentJobWorkflowSnapshot, + job_ids: &[EnvironmentJobId], + result: EnvironmentJobPollActivityResult, +) { + snapshot.resolutions.extend( + result + .resolutions + .into_iter() + .filter(|(job_id, _)| job_ids.iter().any(|id| id.as_str() == job_id)), + ); + for observation in &result.jobs { + if let EnvironmentJobObservation::NotFound { job_id } = observation + && job_ids.contains(job_id) + { + snapshot + .resolutions + .entry(job_id.as_str().to_owned()) + .or_insert(harness::PromiseSourceCheckResult::Failed { error_ref: None }); + } + } + apply_observations(snapshot, job_ids, result.jobs); + refresh_terminal(snapshot, job_ids); +} + +/// A summary replaces the job's record; `notFound` marks the job `lost`, +/// keeping what the workflow already knew about it. Observations of jobs +/// outside the group (such as dependents a cancel reached) are ignored. +fn apply_observations( + snapshot: &mut EnvironmentJobWorkflowSnapshot, + job_ids: &[EnvironmentJobId], + observations: Vec, +) { + for observation in observations { + if !job_ids.contains(observation.job_id()) { + continue; + } + let existing = snapshot + .jobs + .iter_mut() + .find(|existing| &existing.job_id == observation.job_id()); + match (observation, existing) { + (EnvironmentJobObservation::Job { record }, Some(existing)) => *existing = record, + (EnvironmentJobObservation::Job { record }, None) => snapshot.jobs.push(record), + (EnvironmentJobObservation::NotFound { .. }, Some(existing)) => { + existing.status = EnvironmentJobStatus::Lost; + } + (EnvironmentJobObservation::NotFound { job_id }, None) => { + snapshot.jobs.push(EnvironmentJobRecord::lost(job_id)); + } + } + } +} + +/// Jobs of the group without both a terminal record and a resolution. +fn unsettled_job_ids( + snapshot: &EnvironmentJobWorkflowSnapshot, + job_ids: &[EnvironmentJobId], +) -> Vec { + job_ids + .iter() + .filter(|job_id| { + let terminal = snapshot + .jobs + .iter() + .any(|job| &job.job_id == *job_id && job.status.is_terminal()); + !(terminal && snapshot.resolutions.contains_key(job_id.as_str())) + }) + .cloned() + .collect() +} + +fn refresh_terminal(snapshot: &mut EnvironmentJobWorkflowSnapshot, job_ids: &[EnvironmentJobId]) { + snapshot.terminal = !job_ids.is_empty() && unsettled_job_ids(snapshot, job_ids).is_empty(); +} + /// Environment jobs deliberately remain co-located with the core environment /// registry and environment adapter. Their short, idempotent adapter calls therefore /// run as local activities: Temporal still records completion and retries, but @@ -358,7 +432,7 @@ fn queue_workflow_tool_cancellation( { state.pending_cancels.push(EnvironmentJobCancelSignal { jobs: vec![subscription.job_id.clone()], - scope: environment_protocol::data::jobs::JobCancelScope::Job, + scope: EnvironmentJobCancelScope::Job, force: false, }); } @@ -456,7 +530,8 @@ fn workflow_tool_recovery(state: &EnvironmentJobWorkflow) -> WorkflowToolRecover #[cfg(test)] mod tests { - use environment_protocol::shared::JobId; + use std::collections::BTreeMap; + use harness::{BlobRef, PromiseSourceCheckResult}; use super::*; @@ -466,7 +541,7 @@ mod tests { holder_workflow_id: "universe/session_1".to_owned(), promise_id: "promise_1".to_owned(), completion_key: "job_1".to_owned(), - job_id: JobId::new("job_1"), + job_id: EnvironmentJobId::new("job_1"), notified: false, } } @@ -515,6 +590,143 @@ mod tests { assert!(second.is_empty()); } + fn record(job_id: &str, status: EnvironmentJobStatus) -> EnvironmentJobRecord { + EnvironmentJobRecord { + job_id: EnvironmentJobId::new(job_id), + name: Some(job_id.to_owned()), + status, + depends_on: Vec::new(), + queue_key: None, + } + } + + #[test] + fn not_found_settles_the_job_as_lost() { + let job_ids = vec![ + EnvironmentJobId::new("job_1"), + EnvironmentJobId::new("job_2"), + ]; + let mut snapshot = EnvironmentJobWorkflowSnapshot { + started: true, + jobs: vec![ + record("job_1", EnvironmentJobStatus::Running), + record("job_2", EnvironmentJobStatus::Queued), + ], + ..Default::default() + }; + assert_eq!(unsettled_job_ids(&snapshot, &job_ids), job_ids); + + let done = BlobRef::from_bytes(b"done"); + apply_poll_result( + &mut snapshot, + &job_ids, + EnvironmentJobPollActivityResult { + jobs: vec![ + EnvironmentJobObservation::Job { + record: record("job_1", EnvironmentJobStatus::Succeeded), + }, + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("job_2"), + }, + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("elsewhere"), + }, + ], + resolutions: BTreeMap::from([( + "job_1".to_owned(), + PromiseSourceCheckResult::Resolved { + payload_ref: Some(done.clone()), + }, + )]), + }, + ); + + assert!(snapshot.terminal); + assert!(unsettled_job_ids(&snapshot, &job_ids).is_empty()); + assert_eq!(snapshot.jobs.len(), 2); + assert_eq!(snapshot.jobs[1].status, EnvironmentJobStatus::Lost); + // The record keeps what the workflow knew before the job was lost. + assert_eq!(snapshot.jobs[1].name.as_deref(), Some("job_2")); + assert_eq!( + snapshot.resolutions.get("job_2"), + Some(&PromiseSourceCheckResult::Failed { error_ref: None }) + ); + assert!(!snapshot.resolutions.contains_key("elsewhere")); + } + + #[test] + fn activity_resolution_of_a_lost_job_wins_and_running_jobs_stay_unsettled() { + let job_ids = vec![ + EnvironmentJobId::new("job_1"), + EnvironmentJobId::new("job_2"), + ]; + let mut snapshot = EnvironmentJobWorkflowSnapshot::default(); + let lost = BlobRef::from_bytes(b"lost"); + apply_poll_result( + &mut snapshot, + &job_ids, + EnvironmentJobPollActivityResult { + jobs: vec![ + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("job_1"), + }, + EnvironmentJobObservation::Job { + record: record("job_2", EnvironmentJobStatus::Running), + }, + ], + resolutions: BTreeMap::from([( + "job_1".to_owned(), + PromiseSourceCheckResult::Failed { + error_ref: Some(lost.clone()), + }, + )]), + }, + ); + + assert!(!snapshot.terminal); + assert_eq!( + snapshot.jobs[0], + EnvironmentJobRecord::lost(EnvironmentJobId::new("job_1")) + ); + assert_eq!( + snapshot.resolutions.get("job_1"), + Some(&PromiseSourceCheckResult::Failed { + error_ref: Some(lost) + }) + ); + assert_eq!( + unsettled_job_ids(&snapshot, &job_ids), + vec![EnvironmentJobId::new("job_2")] + ); + } + + #[test] + fn cancel_not_found_marks_lost_but_waits_for_the_poll_resolution() { + let job_ids = vec![EnvironmentJobId::new("job_1")]; + let mut snapshot = EnvironmentJobWorkflowSnapshot { + jobs: vec![record("job_1", EnvironmentJobStatus::Running)], + ..Default::default() + }; + apply_observations( + &mut snapshot, + &job_ids, + vec![ + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("job_1"), + }, + EnvironmentJobObservation::Job { + record: record("dependent", EnvironmentJobStatus::Cancelled), + }, + ], + ); + refresh_terminal(&mut snapshot, &job_ids); + + assert_eq!(snapshot.jobs.len(), 1); + assert_eq!(snapshot.jobs[0].status, EnvironmentJobStatus::Lost); + assert!(!snapshot.terminal); + assert_eq!(unsettled_job_ids(&snapshot, &job_ids), job_ids); + } + #[test] fn workflow_tool_cancellation_targets_only_the_matching_job() { let invocation_id = @@ -545,7 +757,10 @@ mod tests { ); assert_eq!(workflow.pending_cancels.len(), 1); - assert_eq!(workflow.pending_cancels[0].jobs, vec![JobId::new("job_1")]); + assert_eq!( + workflow.pending_cancels[0].jobs, + vec![EnvironmentJobId::new("job_1")] + ); assert!(workflow.nudged); queue_workflow_tool_cancellation( diff --git a/crates/temporal-workflow/src/workflows/session/preparation.rs b/crates/temporal-workflow/src/workflows/session/preparation.rs index c84d39480..f9d6d6c87 100644 --- a/crates/temporal-workflow/src/workflows/session/preparation.rs +++ b/crates/temporal-workflow/src/workflows/session/preparation.rs @@ -380,8 +380,8 @@ async fn prepare_operation( "session configuration revision changed", )); } - summary.config_changed = drive.state().lifecycle.config.as_ref() != Some(&config); configure(ctx, drive, &mut candidate, config, expected_revision).await?; + summary.config_changed = config_changed(drive, &candidate); } SessionOperation::ApplyProfile { profile, @@ -418,6 +418,14 @@ async fn prepare_operation( Ok((candidate, summary)) } +/// Whether the candidate's admitted configuration differs from the live one. +/// Admission materializes defaults (such as the compaction policy) into the +/// requested document, so comparing the request itself would report a +/// change for a replacement the harness admits as a no-op. +fn config_changed(live: &CoreAgentDrive, candidate: &PreparationCandidate) -> bool { + live.state().lifecycle.config != candidate.state().lifecycle.config +} + async fn configure( ctx: &mut WorkflowContext, drive: &mut CoreAgentDrive, @@ -504,7 +512,6 @@ async fn apply_profile( workflow_time_ms(ctx), )?; if profile.config.is_some() { - summary.config_changed = drive.state().lifecycle.config.as_ref() != Some(&config); candidate.push( CoreAgentCommand::ReplaceSessionConfig { expected_revision, @@ -512,6 +519,7 @@ async fn apply_profile( }, workflow_time_ms(ctx), )?; + summary.config_changed = config_changed(drive, candidate); } // The default fills an empty pointer and never overrides a live // selection. The configuration replacement above has already cleared a @@ -690,6 +698,68 @@ pub(super) fn abandon_pending_run(state: &mut AgentSessionWorkflow) { mod tests { use super::*; + /// A live session opened with the default configuration, whose admission + /// materialized the default compaction policy. + fn opened_live() -> CoreAgentDrive { + let empty = CoreAgentDrive::from_replayed( + SessionId::new("config-changed"), + CoreAgentState::new(), + None, + ); + let mut opening = PreparationCandidate::new(&empty); + opening + .push( + CoreAgentCommand::OpenSession { + config: crate::default_session_config(harness::ModelSelection { + api_kind: harness::ProviderApiKind::OpenAiResponses, + provider_id: "openai".into(), + model: "gpt-test".into(), + }), + }, + 1, + ) + .unwrap(); + let state = opening.state().clone(); + let head = state.reduced_to.clone(); + CoreAgentDrive::from_replayed(empty.session_id().clone(), state, head) + } + + #[test] + fn config_change_compares_the_admitted_configuration() { + let live = opened_live(); + let current = live.state().lifecycle.config.clone().unwrap(); + assert!(current.context.compaction.is_some()); + // A profile document omits the defaults admission fills in: the same + // configuration is no change. + let mut same = current.clone(); + same.context.compaction = None; + let mut candidate = PreparationCandidate::new(&live); + candidate + .push( + CoreAgentCommand::ReplaceSessionConfig { + expected_revision: None, + config: same, + }, + 2, + ) + .unwrap(); + assert!(!config_changed(&live, &candidate)); + + let mut other = current; + other.model.model = "gpt-other".into(); + let mut candidate = PreparationCandidate::new(&live); + candidate + .push( + CoreAgentCommand::ReplaceSessionConfig { + expected_revision: None, + config: other, + }, + 2, + ) + .unwrap(); + assert!(config_changed(&live, &candidate)); + } + fn pending() -> PendingRunPreparation { let config = crate::default_session_config(harness::ModelSelection { api_kind: harness::ProviderApiKind::OpenAiResponses, diff --git a/crates/tools/Cargo.toml b/crates/tools/Cargo.toml index 7428e7895..0d2da8f64 100644 --- a/crates/tools/Cargo.toml +++ b/crates/tools/Cargo.toml @@ -8,9 +8,10 @@ api = { path = "../api" } async-trait = "0.1" harness = { path = "../harness", features = ["contract"] } futures-util = "0.3" -glob = "0.3" +globset = "0.4" +ignore = "0.4" environment-client = { path = "../environment-client" } -environment-protocol = { path = "../environment-protocol", features = ["schema"] } +environment-protocol = { path = "../environment-protocol" } jsonschema = { version = "0.58", default-features = false } regex = "1" schemars = "1" diff --git a/crates/tools/src/builtin/canonical.rs b/crates/tools/src/builtin/canonical.rs index c3087217d..db334315b 100644 --- a/crates/tools/src/builtin/canonical.rs +++ b/crates/tools/src/builtin/canonical.rs @@ -498,7 +498,7 @@ fn job_submit_schema() -> Value { "job_id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", - "description": "Your name for the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read." + "description": "Your name for this run of the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read. An id names one run: to run a command again, choose a new id." }, "argv": { "type": "array", "items": { "type": "string" } }, "cwd": { "type": ["string", "null"] }, @@ -578,9 +578,16 @@ fn job_read_schema() -> Value { "type": "object", "properties": { "jobs": { "type": "array", "items": job_handle_schema() }, - "output_bytes": { "type": ["integer", "null"], "minimum": 0 }, - "after_seq": { "type": ["integer", "null"], "minimum": 0 }, - "include_artifacts": { "type": "boolean" } + "output_bytes": { + "type": ["integer", "null"], + "minimum": 0, + "description": "Output bytes to show per job. Without `after`, half come from the start of the output and half from its end." + }, + "after": { + "type": ["integer", "null"], + "minimum": 0, + "description": "Byte offset to read forward from, such as a result's `output_next` or its omitted range's `offset`." + } }, "required": ["jobs"], "additionalProperties": false, diff --git a/crates/tools/src/builtin/output_schema_tests.rs b/crates/tools/src/builtin/output_schema_tests.rs index 776aaf8d3..520f2ac58 100644 --- a/crates/tools/src/builtin/output_schema_tests.rs +++ b/crates/tools/src/builtin/output_schema_tests.rs @@ -289,24 +289,22 @@ async fn process_schema_requires_one_typed_encoding_per_stream() { fn durable_job_and_agent_results_preserve_their_exact_serialized_names() { use crate::environment::jobs::{ JobHandle, JobSubmitResult, JobSubmitted, ModelJobOutputSegment, ModelJobResult, - ModelJobResultSet, + ModelJobResultSet, ModelJobStatus, ModelJobSummary, ModelOutputOmission, ModelOutputStream, + ModelStoppedBy, }; use crate::subagents::{SubagentResultEnvelope, SubagentResultStatus, SubagentToolKind}; - use environment_protocol::{ - data::jobs::{JobArtifact, JobOutputStream, JobStatus, JobSummary}, - shared::{EnvironmentPath, JobId}, - }; + use environment_protocol::ids::JobId; let handle = JobHandle { environment_id: "env-1".into(), - job_id: JobId::new("build"), + job_id: JobId::new("build").unwrap(), }; let submitted = JobSubmitResult { jobs: vec![JobSubmitted { name: Some("Build".into()), job_id: handle.job_id.clone(), handle: Some(handle.clone()), - status: JobStatus::Queued, + status: ModelJobStatus::Queued, dependencies: vec![], queue_key: None, promise: Some("promise_1".into()), @@ -319,35 +317,48 @@ fn durable_job_and_agent_results_preserve_their_exact_serialized_names() { .expect("submitted jobs"); let result = ModelJobResult { handle: Some(handle), - summary: Some(JobSummary { - namespace: "session".into(), - job_id: JobId::new("build"), + summary: Some(ModelJobSummary { + job_id: JobId::new("build").unwrap(), name: None, - status: JobStatus::TimedOut, - dependencies: vec![], + status: ModelJobStatus::TimedOut, + depends_on: vec![], + queue_key: None, created_at_ms: 1, - queued_at_ms: None, started_at_ms: Some(2), finished_at_ms: Some(3), + duration_ms: Some(1), exit_code: None, - orphaned_descendants: true, - failure: None, - queue_key: None, + signal: Some("SIGTERM".into()), + core_dumped: false, + stopped_by: Some(ModelStoppedBy::Timeout), + swept_processes: vec![crate::environment::process::LeftoverProcess { + pid: 7, + command: "sleep 100".into(), + }], + error: None, }), output: vec![ModelJobOutputSegment { - stream: JobOutputStream::Stdout, + stream: ModelOutputStream::Stdout, + text: Some("start".into()), + blob_ref: None, + media_type: None, + byte_len: None, + }], + omitted: Some(ModelOutputOmission { + offset: 5, + bytes: 90, + lost_bytes: 0, + }), + tail: vec![ModelJobOutputSegment { + stream: ModelOutputStream::Stderr, text: Some("done".into()), blob_ref: None, media_type: None, byte_len: Some(4), }], - output_next_seq: 9, + output_next: 99, + output_end: 99, truncated: false, - artifacts: vec![JobArtifact { - path: EnvironmentPath::new("/out").unwrap(), - kind: None, - metadata: Default::default(), - }], error: None, }; let schema = validator(BuiltinTool::environment_canonical( @@ -355,8 +366,8 @@ fn durable_job_and_agent_results_preserve_their_exact_serialized_names() { )); schema.validate(&serialized(&result)).expect("joined job"); let mut invalid = serialized(&result); - invalid["summary"]["status"] = json!("timed_out"); - assert!(!schema.is_valid(&invalid), "job statuses use camelCase"); + invalid["summary"]["status"] = json!("timedOut"); + assert!(!schema.is_valid(&invalid), "job statuses use snake_case"); validator(BuiltinTool::environment_canonical( BuiltinToolOperation::JobRead, )) diff --git a/crates/tools/src/builtin/shared.rs b/crates/tools/src/builtin/shared.rs index bb57517e1..cd4bb2b66 100644 --- a/crates/tools/src/builtin/shared.rs +++ b/crates/tools/src/builtin/shared.rs @@ -31,6 +31,9 @@ pub(super) fn visible_with_search_stop( crate::fs::FsSearchStop::TimeLimit => { "[truncated: time budget exhausted — narrow the path or add an include filter]" } + crate::fs::FsSearchStop::ResponseLimit => { + "[truncated: response size limit reached — narrow the pattern or the path]" + } }; if !visible.is_empty() { visible.push('\n'); diff --git a/crates/tools/src/environment.rs b/crates/tools/src/environment.rs index c2ca34c59..248c871bd 100644 --- a/crates/tools/src/environment.rs +++ b/crates/tools/src/environment.rs @@ -15,6 +15,7 @@ pub(crate) mod catalog_text; pub mod control; pub mod handles; pub mod jobs; +pub mod output; pub mod process; pub mod projection; pub mod sources; diff --git a/crates/tools/src/environment/jobs.rs b/crates/tools/src/environment/jobs.rs index 4662f0923..4d68531d4 100644 --- a/crates/tools/src/environment/jobs.rs +++ b/crates/tools/src/environment/jobs.rs @@ -1,16 +1,26 @@ //! Durable environment job capability boundary. +//! +//! The model-facing argument and result types here are this crate's own: +//! they map to and from the environment protocol's job wire types at this +//! boundary, so a protocol change never silently reshapes what the model +//! reads or writes. use std::collections::BTreeMap; use async_trait::async_trait; use environment_protocol::{ - data::jobs::{ - CancelJobsParams, CancelJobsResponse, JobArtifact, JobCancelScope, - JobDependency as ProtocolJobDependency, JobDependencyPolicy, JobOutputChunk, JobStartSpec, - JobStatus, JobSummary, ReadJobsParams, ReadJobsResponse, StartJobsParams, - StartJobsResponse, + bytes::ByteChunk, + data::{ + exec::{CommandSpec, ExecInput, ExitStatus, StoppedBy}, + handshake::JobLimits, + job::{ + CancelJobsParams, CancelJobsResponse, JobDependency, JobDependencyPolicy, JobReadEntry, + JobReadRequest, JobSpec, JobStatus, JobSummary, ReadJobsParams, ReadJobsResponse, + StartJobsParams, StartJobsResponse, + }, }, - shared::{ByteChunk, EnvironmentPath, JobId}, + ids::{JobId, JobName, QueueKey}, + paths::EnvironmentPath, }; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -20,7 +30,13 @@ use harness::{ storage::{BlobEdge, BlobGraphStore, BlobStore, BlobStoreError}, }; -use crate::fs::FsPath; +use crate::{ + environment::{ + output::{ShownOutput, is_stderr}, + process::LeftoverProcess, + }, + fs::FsPath, +}; pub const JOB_SUBMIT_TOOL_NAME: &str = "job_submit"; pub const JOB_RUN_TOOL_NAME: &str = "job_run"; @@ -32,11 +48,19 @@ pub const JOB_RUN_WORKFLOW_SEMANTIC_TYPE: &str = "lightspeed.environment.job.run pub const JOB_RUN_DEFAULT_TIMEOUT_MS: u64 = 30 * 60 * 1_000; pub const JOB_RUN_MAX_TIMEOUT_MS: u64 = 60 * 60 * 1_000; pub const JOB_RUN_DEADLINE_AFTER_MS: u64 = 65 * 60 * 1_000; +/// Output a job result shows when the reader names no budget: the first +/// half from the start of the output, the second half from its end. +pub const DEFAULT_JOB_OUTPUT_BYTES: u64 = 16 * 1024; pub type JobExecResult = Result; #[async_trait] pub trait JobExecutor: Send + Sync { + /// The endpoint's job ceilings, when known; reads stay within them. + fn limits(&self) -> Option { + None + } + async fn start_jobs(&self, request: StartJobsParams) -> JobExecResult; async fn read_jobs(&self, request: ReadJobsParams) -> JobExecResult; @@ -52,6 +76,10 @@ pub enum JobError { #[error("invalid environment job request: {message}")] InvalidRequest { message: String }, + /// A job id is taken by another run; the model must choose a new id. + #[error("environment job id in use: {message}")] + JobIdInUse { message: String }, + #[error("environment job execution failed: {message}")] Failed { message: String }, } @@ -69,6 +97,7 @@ pub struct JobHandleArg { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] pub struct JobHandle { pub environment_id: String, + #[schemars(with = "String")] pub job_id: JobId, } @@ -97,7 +126,13 @@ pub struct JobRunArgs { } impl JobRunArgs { - pub fn into_protocol_spec(self, job_id: JobId) -> JobExecResult { + /// The wire spec of this single job. A relative `cwd` resolves against + /// `working_directory`, which is also the default `cwd`. + pub fn into_job_spec( + self, + job_id: JobId, + working_directory: Option<&str>, + ) -> JobExecResult { let timeout_ms = self.timeout_ms.unwrap_or(JOB_RUN_DEFAULT_TIMEOUT_MS); if timeout_ms > JOB_RUN_MAX_TIMEOUT_MS { return Err(JobError::InvalidRequest { @@ -113,10 +148,10 @@ impl JobRunArgs { stdin: self.stdin, timeout_ms: Some(timeout_ms), depends_on: Vec::new(), - dependency_policy: JobDependencyPolicy::AllSucceeded, + dependency_policy: JobDependencyPolicyArg::AllSucceeded, queue_key: self.queue_key, } - .into_protocol_spec(job_id) + .into_job_spec(job_id, working_directory) } } @@ -150,22 +185,18 @@ impl JobSubmitExecutionContextV1 { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct JobDependencyArg { - #[serde(default, alias = "jobId", skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub job_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, } impl JobDependencyArg { - fn into_protocol_dependency(self) -> JobExecResult { + fn into_job_dependency(self) -> JobExecResult { match (self.job_id, self.name) { - (Some(job_id), None) => Ok(ProtocolJobDependency { - job_id: Some(job_id), - name: None, - }), - (None, Some(name)) if !name.is_empty() => Ok(ProtocolJobDependency { - job_id: None, - name: Some(name), + (Some(job_id), None) => Ok(JobDependency::JobId { job_id }), + (None, Some(name)) if !name.is_empty() => Ok(JobDependency::Name { + name: job_name(name)?, }), (Some(_), Some(_)) => Err(JobError::InvalidRequest { message: "job dependency must specify job_id or name, not both".to_owned(), @@ -177,6 +208,26 @@ impl JobDependencyArg { } } +/// What a job waits for from its `depends_on` jobs. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum JobDependencyPolicyArg { + /// Run once every dependency succeeded. + #[default] + AllSucceeded, + /// Run once every dependency finished, whatever its outcome. + AllTerminal, +} + +impl From for JobDependencyPolicy { + fn from(policy: JobDependencyPolicyArg) -> Self { + match policy { + JobDependencyPolicyArg::AllSucceeded => Self::AllSucceeded, + JobDependencyPolicyArg::AllTerminal => Self::AllTerminal, + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct JobSubmitSpecArgs { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -194,13 +245,21 @@ pub struct JobSubmitSpecArgs { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub depends_on: Vec, #[serde(default)] - pub dependency_policy: JobDependencyPolicy, + pub dependency_policy: JobDependencyPolicyArg, #[serde(default, skip_serializing_if = "Option::is_none")] pub queue_key: Option, } impl JobSubmitSpecArgs { - pub fn into_protocol_spec(self, job_id: JobId) -> JobExecResult { + /// The wire spec of this job. A relative `cwd` resolves against + /// `working_directory`, which is also the default `cwd`; without one a + /// relative `cwd` is refused and an absent one means the environment's + /// default. + pub fn into_job_spec( + self, + job_id: JobId, + working_directory: Option<&str>, + ) -> JobExecResult { if self.argv.is_empty() { return Err(JobError::InvalidRequest { message: "job argv must not be empty".to_owned(), @@ -217,42 +276,50 @@ impl JobSubmitSpecArgs { let depends_on = self .depends_on .into_iter() - .map(JobDependencyArg::into_protocol_dependency) + .map(JobDependencyArg::into_job_dependency) .collect::>>()?; - Ok(JobStartSpec { + Ok(JobSpec { job_id, - name: self.name, - argv: self.argv, - cwd: self.cwd.as_ref().map(environment_path).transpose()?, - env: self.env, - secret_env: BTreeMap::new(), - stdin: self.stdin.map(|value| ByteChunk::from(value.into_bytes())), - timeout_ms: self.timeout_ms, + name: self.name.map(job_name).transpose()?, + command: CommandSpec { + argv: self.argv, + cwd: job_cwd(self.cwd.as_ref(), working_directory)?, + env: self.env, + secret_env: BTreeMap::new(), + input: match self.stdin { + Some(stdin) => ExecInput::Bytes { + data: ByteChunk::from(stdin.into_bytes()), + }, + None => ExecInput::None {}, + }, + timeout_ms: self.timeout_ms, + }, depends_on, - dependency_policy: self.dependency_policy, - queue_key: self.queue_key, + dependency_policy: self.dependency_policy.into(), + queue_key: self + .queue_key + .map(|key| { + QueueKey::new(key).map_err(|error| JobError::InvalidRequest { + message: format!("queue_key is invalid: {error}"), + }) + }) + .transpose()?, }) } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct JobReadArgs { pub jobs: Vec, + /// Output bytes to show per job. Without `after`, half come from the + /// start of the output and half from its end. #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_bytes: Option, + pub output_bytes: Option, + /// Byte offset to page forward from, such as a previous result's + /// `output_next` or its omitted range's `offset`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub after_seq: Option, - #[serde(default)] - pub include_artifacts: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct JobCancelArgs { - pub jobs: Vec, - #[serde(default)] - pub scope: JobCancelScope, - #[serde(default)] - pub force: bool, + pub after: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] @@ -264,10 +331,12 @@ pub struct JobSubmitResult { pub struct JobSubmitted { #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, + #[schemars(with = "String")] pub job_id: JobId, #[serde(default, skip_serializing_if = "Option::is_none")] pub handle: Option, - pub status: JobStatus, + pub status: ModelJobStatus, + #[schemars(with = "Vec")] pub dependencies: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub queue_key: Option, @@ -276,33 +345,216 @@ pub struct JobSubmitted { pub promise: Option, } +impl JobSubmitted { + pub fn from_summary(summary: &JobSummary, handle: Option) -> Self { + Self { + name: summary.name.as_ref().map(|name| name.as_str().to_owned()), + job_id: summary.job_id.clone(), + handle, + status: summary.status.into(), + dependencies: summary.depends_on.clone(), + queue_key: summary + .queue_key + .as_ref() + .map(|key| key.as_str().to_owned()), + promise: None, + } + } +} + +/// Where a job stands. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ModelJobStatus { + Queued, + Running, + Succeeded, + Failed, + TimedOut, + Cancelled, + DependencyFailed, + Interrupted, +} + +impl ModelJobStatus { + pub fn is_terminal(self) -> bool { + !matches!(self, Self::Queued | Self::Running) + } +} + +impl From for ModelJobStatus { + fn from(status: JobStatus) -> Self { + match status { + JobStatus::Queued => Self::Queued, + JobStatus::Running => Self::Running, + JobStatus::Succeeded => Self::Succeeded, + JobStatus::Failed => Self::Failed, + JobStatus::TimedOut => Self::TimedOut, + JobStatus::Cancelled => Self::Cancelled, + JobStatus::DependencyFailed => Self::DependencyFailed, + JobStatus::Interrupted => Self::Interrupted, + } + } +} + +/// Why the environment stopped a job that did not end on its own. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ModelStoppedBy { + Timeout, + Terminate, + Kill, +} + +impl From for ModelStoppedBy { + fn from(stopped_by: StoppedBy) -> Self { + match stopped_by { + StoppedBy::Timeout => Self::Timeout, + StoppedBy::Terminate => Self::Terminate, + StoppedBy::Kill => Self::Kill, + } + } +} + +/// What a job is and how it ended. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct ModelJobSummary { + #[schemars(with = "String")] + pub job_id: JobId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub status: ModelJobStatus, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[schemars(with = "Vec")] + pub depends_on: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, + pub created_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_code: Option, + /// The signal that killed the job's command, such as `SIGSEGV`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signal: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub core_dumped: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stopped_by: Option, + /// Processes the job left running, which the environment stopped when + /// the job finished; the job's script should shut them down itself. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub swept_processes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl From for ModelJobSummary { + fn from(summary: JobSummary) -> Self { + let (exit_code, signal, core_dumped) = match summary.exit { + Some(ExitStatus::Code { code }) => (Some(code), None, false), + Some(ExitStatus::Signal { + signal, + core_dumped, + }) => (None, Some(signal.as_str().to_owned()), core_dumped), + None => (None, None, false), + }; + Self { + job_id: summary.job_id, + name: summary.name.map(JobName::into_string), + status: summary.status.into(), + depends_on: summary.depends_on, + queue_key: summary.queue_key.map(QueueKey::into_string), + created_at_ms: summary.created_at_ms, + started_at_ms: summary.started_at_ms, + finished_at_ms: summary.finished_at_ms, + duration_ms: summary.duration_ms, + exit_code, + signal, + core_dumped, + stopped_by: summary.stopped_by.map(Into::into), + swept_processes: summary + .leftovers + .map(|report| { + report + .processes + .into_iter() + .map(|process| LeftoverProcess { + pid: process.pid, + command: process.command.unwrap_or_default(), + }) + .collect() + }) + .unwrap_or_default(), + error: summary.error.map(|error| error.to_string()), + } + } +} + +/// A range of output the result leaves out. Read it with `job_read` +/// `after: offset`; the `lost_bytes` of it that the environment dropped to +/// bound retained output read back as a gap. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct ModelOutputOmission { + /// Byte offset of the first omitted byte. + pub offset: u64, + pub bytes: u64, + /// Omitted bytes that are gone for good; reading them again does not + /// return them. + #[serde(default, skip_serializing_if = "is_zero")] + pub lost_bytes: u64, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -#[serde(rename_all = "camelCase")] pub struct ModelJobResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub handle: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary: Option, + pub summary: Option, + /// Output from the requested position, in order. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub output: Vec, - pub output_next_seq: u64, - pub truncated: bool, + /// Output between `output` and `tail` that is not shown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub omitted: Option, + /// The last output bytes, after the omitted range. #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub artifacts: Vec, + pub tail: Vec, + /// Byte offset after the last output shown; pass it as `after` to read + /// on. + pub output_next: u64, + /// Output bytes the job produced so far. + pub output_end: u64, + /// More output exists after `output_next`. + pub truncated: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -#[serde(rename_all = "camelCase")] pub struct ModelJobResultSet { pub jobs: Vec, } +/// Which stream produced an output segment. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ModelOutputStream { + Stdout, + Stderr, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -#[serde(rename_all = "camelCase")] pub struct ModelJobOutputSegment { - pub stream: environment_protocol::data::jobs::JobOutputStream, + pub stream: ModelOutputStream, #[serde(default, skip_serializing_if = "Option::is_none")] pub text: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -313,21 +565,6 @@ pub struct ModelJobOutputSegment { pub byte_len: Option, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct JobCancelResultSet { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct JobCancelResultEntry { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub handle: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, -} - pub fn visible_job_read_output(jobs: &[ModelJobResult]) -> String { serde_json::to_string_pretty(&ModelJobResultSet { jobs: jobs.to_vec(), @@ -335,15 +572,107 @@ pub fn visible_job_read_output(jobs: &[ModelJobResult]) -> String { .unwrap_or_else(|error| format!("failed to encode job results: {error}")) } +/// The `job/read` entries that show one job: a page forward from `after`, +/// or, without it, the head and the tail of the output, each half of +/// `output_bytes`. [`NormalizeJobResultInput::from_entries`] reassembles the +/// answer. +pub fn job_output_requests( + job_id: &JobId, + after: Option, + output_bytes: u64, +) -> Vec { + match after { + Some(after) => vec![JobReadRequest { + job_id: job_id.clone(), + after: Some(after), + tail_bytes: None, + max_bytes: Some(output_bytes), + }], + None => { + let head = output_bytes / 2; + vec![ + JobReadRequest { + job_id: job_id.clone(), + after: Some(0), + tail_bytes: None, + max_bytes: Some(head), + }, + JobReadRequest { + job_id: job_id.clone(), + after: None, + tail_bytes: Some(output_bytes - head), + max_bytes: Some(output_bytes - head), + }, + ] + } + } +} + +/// A complete `job/read` for `jobs`, each shown as [`job_output_requests`] +/// describes, waiting for nothing. +pub fn job_output_read(jobs: &[JobId], after: Option, output_bytes: u64) -> ReadJobsParams { + let entries = jobs + .iter() + .flat_map(|job_id| job_output_requests(job_id, after, output_bytes)) + .collect::>(); + ReadJobsParams { + max_bytes: output_bytes.saturating_mul(jobs.len() as u64), + jobs: entries, + wait_ms: 0, + return_on: environment_protocol::data::exec::ReturnOn::AnyOutput, + } +} + #[derive(Clone, Debug, Default)] pub struct NormalizeJobResultInput { pub handle: Option, pub summary: Option, - pub output_chunks: Vec, - pub output_next_seq: u64, - pub artifacts: Vec, + /// Output shown from the requested position: `output`, then `omitted` + /// and `tail`. + pub output: ShownOutput, pub error: Option, - pub output_bytes: Option, +} + +impl NormalizeJobResultInput { + /// Reassemble the entries [`job_output_requests`] asked for one job: a + /// summary and its pages, or the entry's error (`notFound` for a job the + /// environment does not know). + pub fn from_entries( + handle: Option, + entries: impl IntoIterator, + ) -> Self { + let mut summary = None; + let mut pages = Vec::new(); + let mut error = None; + for entry in entries { + match entry { + JobReadEntry::Job { + summary: entry_summary, + output, + } => { + summary = Some(*entry_summary); + pages.push(output); + } + JobReadEntry::Error { + job_id, + error: cause, + } => { + error.get_or_insert_with(|| format!("job {job_id}: {cause}")); + } + } + } + let mut pages = pages.into_iter(); + let output = pages + .next() + .map(|head| ShownOutput::from_pages(head, pages.next())) + .unwrap_or_default(); + Self { + handle, + summary, + output, + error, + } + } } pub async fn normalize_job_result( @@ -353,46 +682,76 @@ pub async fn normalize_job_result( let NormalizeJobResultInput { handle, summary, - output_chunks, - output_next_seq, - artifacts, + output, error, - output_bytes, } = input; - let observed_bytes = output_chunks - .iter() - .map(|chunk| chunk.chunk.as_slice().len()) - .sum::(); - let truncated = output_bytes.is_some_and(|limit| observed_bytes >= limit); + let truncated = output.next < output.end; + let omitted = output.omitted.map(|omission| ModelOutputOmission { + offset: omission.offset, + bytes: omission.bytes, + lost_bytes: omission.lost_bytes, + }); + let head = segments(blobs, output.head).await?; + let tail = segments(blobs, output.tail).await?; + Ok(ModelJobResult { + handle: handle.map(|mut handle| { + handle.environment_id = super::handles::environment_handle(&handle.environment_id); + handle + }), + summary: summary.map(Into::into), + output: head, + omitted, + tail, + output_next: output.next, + output_end: output.end, + truncated, + error, + }) +} + +/// A result that carries only an error, such as an unknown job. +pub fn model_job_error(handle: Option, error: String) -> ModelJobResult { + ModelJobResult { + handle: handle.map(|mut handle| { + handle.environment_id = super::handles::environment_handle(&handle.environment_id); + handle + }), + summary: None, + output: Vec::new(), + omitted: None, + tail: Vec::new(), + output_next: 0, + output_end: 0, + truncated: false, + error: Some(error), + } +} + +/// Merge consecutive chunks of one stream; text stays inline, bytes that +/// are not UTF-8 go to CAS and are referenced. +async fn segments( + blobs: &dyn BlobStore, + chunks: Vec, +) -> Result, BlobStoreError> { let mut output = Vec::new(); let mut index = 0; - while index < output_chunks.len() { - let stream = output_chunks[index].stream; + while index < chunks.len() { + let stream = model_stream(chunks[index].stream); let mut bytes = Vec::new(); - while index < output_chunks.len() && output_chunks[index].stream == stream { - bytes.extend_from_slice(output_chunks[index].chunk.as_slice()); + while index < chunks.len() && model_stream(chunks[index].stream) == stream { + bytes.extend_from_slice(chunks[index].data.as_slice()); index += 1; } match String::from_utf8(bytes) { Ok(text) => { if !text.is_empty() { - if let Some(ModelJobOutputSegment { - stream: previous_stream, - text: Some(previous), - .. - }) = output.last_mut() - && *previous_stream == stream - { - previous.push_str(&text); - } else { - output.push(ModelJobOutputSegment { - stream, - text: Some(text), - blob_ref: None, - media_type: None, - byte_len: None, - }); - } + output.push(ModelJobOutputSegment { + stream, + text: Some(text), + blob_ref: None, + media_type: None, + byte_len: None, + }); } } Err(error) => { @@ -409,18 +768,15 @@ pub async fn normalize_job_result( } } } - Ok(ModelJobResult { - handle: handle.map(|mut handle| { - handle.environment_id = super::handles::environment_handle(&handle.environment_id); - handle - }), - summary, - output, - output_next_seq, - truncated, - artifacts, - error, - }) + Ok(output) +} + +fn model_stream(stream: environment_protocol::data::exec::OutputStream) -> ModelOutputStream { + if is_stderr(stream) { + ModelOutputStream::Stderr + } else { + ModelOutputStream::Stdout + } } pub async fn store_model_job_result( @@ -438,6 +794,7 @@ pub async fn store_model_job_result( let edges = result .output .iter() + .chain(&result.tail) .filter_map(|segment| { segment .blob_ref @@ -453,18 +810,44 @@ pub async fn store_model_job_result( Ok(root) } -fn environment_path(path: &FsPath) -> JobExecResult { - EnvironmentPath::new(path.as_str()).map_err(|error| JobError::InvalidRequest { - message: error.to_string(), +fn job_name(name: String) -> JobExecResult { + JobName::new(name).map_err(|error| JobError::InvalidRequest { + message: format!("job name is invalid: {error}"), }) } +fn job_cwd( + cwd: Option<&FsPath>, + working_directory: Option<&str>, +) -> JobExecResult> { + let invalid = |message: String| JobError::InvalidRequest { message }; + let resolved = match (cwd, working_directory) { + (Some(cwd), _) if cwd.is_absolute() => cwd.as_str().to_owned(), + (cwd, Some(base)) => crate::environment::sources::absolute( + std::path::Path::new(base), + cwd.map_or(".", FsPath::as_str), + ) + .map_err(invalid)? + .to_string_lossy() + .into_owned(), + (Some(cwd), None) => { + return Err(invalid(format!( + "job cwd {cwd} is relative and the environment has no working directory" + ))); + } + (None, None) => return Ok(None), + }; + EnvironmentPath::new(resolved) + .map(Some) + .map_err(|error| invalid(error.to_string())) +} + #[cfg(test)] mod tests { use std::sync::{Arc, Mutex}; use async_trait::async_trait; - use environment_protocol::data::jobs::{JobOutputStream, JobStatus}; + use environment_protocol::data::exec::{OutputChunk, OutputPage, OutputStream, SignalName}; use harness::storage::{ BlobEdge, BlobGraphStore, BlobStore, BlobStoreError, InMemoryBlobStore, }; @@ -472,6 +855,10 @@ mod tests { use super::*; + fn job_id(value: &str) -> JobId { + JobId::new(value).unwrap() + } + #[test] fn job_submit_arguments_do_not_accept_an_environment_override() { let error = serde_json::from_value::(json!({ @@ -491,15 +878,65 @@ mod tests { })) .expect("decode job_run"); let spec = args - .into_protocol_spec(JobId::new("job-derived")) + .into_job_spec(job_id("job-derived"), None) .expect("materialize job_run"); - assert_eq!(spec.job_id, JobId::new("job-derived")); - assert_eq!(spec.argv, vec!["cargo", "test"]); - assert_eq!(spec.timeout_ms, Some(JOB_RUN_DEFAULT_TIMEOUT_MS)); + assert_eq!(spec.job_id, job_id("job-derived")); + assert_eq!(spec.command.argv, vec!["cargo", "test"]); + assert_eq!(spec.command.timeout_ms, Some(JOB_RUN_DEFAULT_TIMEOUT_MS)); assert!(spec.depends_on.is_empty()); } + #[test] + fn job_cwd_resolves_against_the_working_directory() { + let spec = |cwd: Option<&str>, base: Option<&str>| { + JobSubmitSpecArgs { + name: None, + job_id: job_id("build"), + argv: vec!["true".to_owned()], + cwd: cwd.map(|cwd| FsPath::new(cwd).unwrap()), + env: BTreeMap::new(), + stdin: Some("input".to_owned()), + timeout_ms: None, + depends_on: Vec::new(), + dependency_policy: JobDependencyPolicyArg::AllSucceeded, + queue_key: None, + } + .into_job_spec(job_id("build"), base) + }; + let resolved = spec(Some("sub"), Some("/srv")).unwrap(); + assert_eq!(resolved.command.cwd.unwrap().as_str(), "/srv/sub"); + assert_eq!( + resolved.command.input, + ExecInput::Bytes { + data: b"input".to_vec().into() + } + ); + assert_eq!( + spec(None, Some("/srv")) + .unwrap() + .command + .cwd + .unwrap() + .as_str(), + "/srv" + ); + assert_eq!( + spec(Some("/abs"), Some("/srv")) + .unwrap() + .command + .cwd + .unwrap() + .as_str(), + "/abs" + ); + assert!(spec(None, None).unwrap().command.cwd.is_none()); + assert!(matches!( + spec(Some("sub"), None), + Err(JobError::InvalidRequest { .. }) + )); + } + #[test] fn job_submit_accepts_snake_case_job_id_dependencies() { let args = serde_json::from_value::(json!({ @@ -509,7 +946,8 @@ mod tests { }, { "job_id": "test", "argv": ["true"], - "depends_on": [{"job_id": "build"}] + "depends_on": [{"job_id": "build"}, {"name": "lint"}], + "dependency_policy": "allTerminal" }] })) .expect("decode job_submit with snake_case dependency id"); @@ -519,41 +957,23 @@ mod tests { .into_iter() .map(|spec| { let job_id = spec.job_id.clone(); - spec.into_protocol_spec(job_id) + spec.into_job_spec(job_id, None) }) .collect::>>() .expect("materialize protocol specs"); assert_eq!( specs[1].depends_on, - vec![ProtocolJobDependency::job_id("build")] - ); - } - - #[test] - fn job_submit_accepts_camel_case_dependency_alias_for_compatibility() { - let args = serde_json::from_value::(json!({ - "jobs": [{ - "job_id": "build", - "argv": ["true"] - }, { - "job_id": "test", - "argv": ["true"], - "depends_on": [{"jobId": "build"}] - }] - })) - .expect("decode job_submit with camelCase dependency id alias"); - - let spec = args.jobs.into_iter().nth(1).expect("dependent job"); - let job_id = spec.job_id.clone(); - let spec = spec - .into_protocol_spec(job_id) - .expect("materialize protocol spec"); - - assert_eq!( - spec.depends_on, - vec![ProtocolJobDependency::job_id("build")] + vec![ + JobDependency::JobId { + job_id: job_id("build") + }, + JobDependency::Name { + name: JobName::new("lint").unwrap() + } + ] ); + assert_eq!(specs[1].dependency_policy, JobDependencyPolicy::AllTerminal); } #[test] @@ -570,7 +990,7 @@ mod tests { let spec = args.jobs.into_iter().next().expect("job"); let job_id = spec.job_id.clone(); let error = spec - .into_protocol_spec(job_id) + .into_job_spec(job_id, None) .expect_err("empty dependency must be rejected locally"); assert!(matches!(error, JobError::InvalidRequest { .. })); @@ -593,43 +1013,34 @@ mod tests { })) .expect("timeout range is a domain validation"); let error = args - .into_protocol_spec(JobId::new("job-derived")) + .into_job_spec(job_id("job-derived"), None) .expect_err("timeout above maximum must fail"); assert!(matches!(error, JobError::InvalidRequest { .. })); } #[test] fn job_ids_must_fit_the_completion_key_shape() { - for job_id in ["build:1", "", "-lead", &"x".repeat(65)] { + for value in ["build:1", "-lead", &"x".repeat(65)] { + let Ok(id) = JobId::new(value) else { + continue; + }; let error = JobSubmitSpecArgs { name: None, - job_id: JobId::new(if job_id.is_empty() { - "placeholder" - } else { - job_id - }), + job_id: id.clone(), argv: vec!["true".to_owned()], cwd: None, env: BTreeMap::new(), stdin: None, timeout_ms: None, depends_on: Vec::new(), - dependency_policy: JobDependencyPolicy::AllSucceeded, + dependency_policy: JobDependencyPolicyArg::AllSucceeded, queue_key: None, } - .into_protocol_spec(JobId::new(if job_id.is_empty() { - "placeholder" - } else { - job_id - })); - if job_id.is_empty() { - error.expect("placeholder id is valid"); - } else { - assert!( - matches!(error, Err(JobError::InvalidRequest { .. })), - "{job_id:?} must be rejected" - ); - } + .into_job_spec(id, None); + assert!( + matches!(error, Err(JobError::InvalidRequest { .. })), + "{value:?} must be rejected" + ); } } @@ -643,6 +1054,88 @@ mod tests { assert_eq!(context.working_directory.as_deref(), Some("/srv")); } + #[test] + fn head_and_tail_requests_split_the_budget() { + let requests = job_output_requests(&job_id("build"), None, 16); + assert_eq!(requests.len(), 2); + assert_eq!( + (requests[0].after, requests[0].max_bytes), + (Some(0), Some(8)) + ); + assert_eq!( + (requests[1].tail_bytes, requests[1].max_bytes), + (Some(8), Some(8)) + ); + let paged = job_output_requests(&job_id("build"), Some(40), 16); + assert_eq!((paged[0].after, paged[0].max_bytes), (Some(40), Some(16))); + let read = job_output_read(&[job_id("a"), job_id("b")], None, 16); + assert_eq!(read.jobs.len(), 4); + assert_eq!(read.max_bytes, 32); + } + + #[tokio::test(flavor = "current_thread")] + async fn job_results_show_head_and_tail_with_the_skipped_count() { + let blobs = InMemoryBlobStore::new(); + let entries = vec![ + JobReadEntry::Job { + summary: Box::new(test_summary()), + output: page( + vec![chunk(0, OutputStream::Stdout, b"compiling\n")], + 10, + 1000, + ), + }, + JobReadEntry::Job { + summary: Box::new(test_summary()), + output: page( + vec![chunk(990, OutputStream::Stderr, b"FAILED: 3\n")], + 1000, + 1000, + ), + }, + ]; + let result = + normalize_job_result(&blobs, NormalizeJobResultInput::from_entries(None, entries)) + .await + .expect("normalize"); + + assert_eq!(result.output[0].text.as_deref(), Some("compiling\n")); + assert_eq!(result.tail[0].text.as_deref(), Some("FAILED: 3\n")); + assert_eq!(result.tail[0].stream, ModelOutputStream::Stderr); + assert_eq!( + result.omitted, + Some(ModelOutputOmission { + offset: 10, + bytes: 980, + lost_bytes: 0, + }) + ); + assert!(!result.truncated); + let value = serde_json::to_value(&result).expect("semantic JSON"); + assert_eq!(value["output_end"], 1000); + assert_eq!(value["summary"]["status"], "failed"); + assert_eq!(value["summary"]["signal"], "SIGSEGV"); + } + + #[tokio::test(flavor = "current_thread")] + async fn unknown_jobs_read_as_errors() { + let blobs = InMemoryBlobStore::new(); + let result = normalize_job_result( + &blobs, + NormalizeJobResultInput::from_entries( + None, + vec![JobReadEntry::Error { + job_id: job_id("gone"), + error: environment_protocol::error::ProtocolError::not_found("unknown job"), + }], + ), + ) + .await + .expect("normalize"); + assert!(result.summary.is_none()); + assert!(result.error.as_deref().is_some_and(|e| e.contains("gone"))); + } + #[tokio::test(flavor = "current_thread")] async fn normalizer_reassembles_utf8_and_preserves_stream_order() { let blobs = InMemoryBlobStore::new(); @@ -650,14 +1143,19 @@ mod tests { &blobs, NormalizeJobResultInput { summary: Some(test_summary()), - output_chunks: vec![ - output_chunk(0, JobOutputStream::Stdout, vec![b'a', 0xe2]), - output_chunk(1, JobOutputStream::Stdout, vec![0x82, 0xac, b'\n']), - output_chunk(2, JobOutputStream::Stderr, b"warning\n".to_vec()), - output_chunk(3, JobOutputStream::Stdout, b"done\n".to_vec()), - ], - output_next_seq: 4, - output_bytes: Some(1024), + output: ShownOutput::from_pages( + page( + vec![ + chunk(0, OutputStream::Stdout, &[b'a', 0xe2]), + chunk(2, OutputStream::Stdout, &[0x82, 0xac, b'\n']), + chunk(5, OutputStream::Stderr, b"warning\n"), + chunk(13, OutputStream::Stdout, b"done\n"), + ], + 18, + 18, + ), + None, + ), ..Default::default() }, ) @@ -666,12 +1164,10 @@ mod tests { assert_eq!(result.output.len(), 3); assert_eq!(result.output[0].text.as_deref(), Some("a€\n")); - assert_eq!(result.output[1].stream, JobOutputStream::Stderr); + assert_eq!(result.output[1].stream, ModelOutputStream::Stderr); assert_eq!(result.output[2].text.as_deref(), Some("done\n")); assert!(!result.truncated); - let value = serde_json::to_value(&result).expect("semantic JSON"); - assert!(value.get("outputChunks").is_none()); - assert_eq!(value["outputNextSeq"], 4); + assert_eq!(result.output_next, 18); } #[tokio::test(flavor = "current_thread")] @@ -681,16 +1177,17 @@ mod tests { &blobs, NormalizeJobResultInput { summary: Some(test_summary()), - output_chunks: vec![output_chunk(0, JobOutputStream::Stdout, vec![0xff, 0x00])], - output_next_seq: 1, - output_bytes: Some(2), + output: ShownOutput::from_pages( + page(vec![chunk(0, OutputStream::Stdout, &[0xff, 0x00])], 2, 4), + None, + ), ..Default::default() }, ) .await .expect("normalize binary"); - assert!(result.truncated); + assert!(result.truncated, "output continues after output_next"); let segment = &result.output[0]; assert!(segment.text.is_none()); assert_eq!( @@ -715,8 +1212,10 @@ mod tests { &blobs, NormalizeJobResultInput { summary: Some(test_summary()), - output_chunks: vec![output_chunk(0, JobOutputStream::Stdout, vec![0xff])], - output_next_seq: 1, + output: ShownOutput::from_pages( + page(vec![chunk(0, OutputStream::Stdout, &[0xff])], 1, 1), + None, + ), ..Default::default() }, ) @@ -746,29 +1245,44 @@ mod tests { } } - fn output_chunk(seq: u64, stream: JobOutputStream, bytes: Vec) -> JobOutputChunk { - JobOutputChunk { - seq, + fn chunk(offset: u64, stream: OutputStream, bytes: &[u8]) -> OutputChunk { + OutputChunk { + offset, stream, - chunk: ByteChunk::from(bytes), + data: bytes.to_vec().into(), + } + } + + fn page(chunks: Vec, next: u64, end: u64) -> OutputPage { + OutputPage { + chunks, + next, + end, + gaps: Vec::new(), } } fn test_summary() -> JobSummary { JobSummary { - namespace: "environment-a".to_owned(), - job_id: JobId::new("job-a"), + job_id: job_id("job-a"), + request_id: environment_protocol::ids::JobRequestId::new("request-a").unwrap(), name: None, - status: JobStatus::Succeeded, - dependencies: Vec::new(), + status: JobStatus::Failed, + depends_on: Vec::new(), + queue_key: None, created_at_ms: 1, queued_at_ms: Some(1), started_at_ms: Some(2), finished_at_ms: Some(3), - exit_code: Some(0), - orphaned_descendants: false, - failure: None, - queue_key: None, + cancel_requested_at_ms: None, + duration_ms: Some(1), + exit: Some(ExitStatus::Signal { + signal: SignalName::new("SIGSEGV").unwrap(), + core_dumped: false, + }), + stopped_by: None, + leftovers: None, + error: None, } } } diff --git a/crates/tools/src/environment/output.rs b/crates/tools/src/environment/output.rs new file mode 100644 index 000000000..e970f4aeb --- /dev/null +++ b/crates/tools/src/environment/output.rs @@ -0,0 +1,411 @@ +//! Shaping command output pages into what the model sees: a head, the +//! bytes omitted after it, and a tail. +//! +//! Environment reads return byte-offset pages with typed gaps. A finished +//! command whose output exceeds the tool budget is shown as its first bytes +//! and its last bytes, with the omitted count in between, because failures +//! are reported at the end. + +use environment_protocol::data::exec::{OutputChunk, OutputPage, OutputStream}; + +/// Output selected for the model, in offset order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ShownOutput { + /// Chunks before the first omitted range. + pub head: Vec, + /// The omitted ranges, merged: where the first starts, how many bytes + /// all of them hold together, and how many of those the environment no + /// longer holds. + pub omitted: Option, + /// Chunks after the first omitted range. + pub tail: Vec, + /// Offset after the last byte or gap shown. + pub next: u64, + /// Bytes the command produced so far. + pub end: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Omission { + pub offset: u64, + pub bytes: u64, + /// Of `bytes`, those the environment's typed gaps report as gone + /// (`overflow`, `expired`): reading them again returns the same gap. + /// The rest was skipped here and can still be read. + pub lost_bytes: u64, +} + +impl ShownOutput { + /// The first page of a read, with an optional second page that starts + /// later (a tail read). Bytes of `tail` that `head` already returned + /// are dropped; the range between the two pages and every gap inside + /// them count as omitted. + pub fn from_pages(head: OutputPage, tail: Option) -> Self { + let mut shown = Self::default(); + let mut items = Vec::new(); + let mut next = head.next; + let mut end = head.end; + push_page(&mut items, head, 0); + if let Some(tail) = tail { + let tail_start = page_start(&tail); + if tail_start > next { + items.push(Item::Gap { + offset: next, + bytes: tail_start - next, + lost: false, + }); + } + let skip_before = next; + next = next.max(tail.next); + end = end.max(tail.end); + push_page(&mut items, tail, skip_before); + } + shown.next = next; + shown.end = end; + for item in items { + match item { + Item::Chunk(chunk) if shown.omitted.is_none() => shown.head.push(chunk), + Item::Chunk(chunk) => shown.tail.push(chunk), + Item::Gap { + offset, + bytes, + lost, + } => { + let lost_bytes = if lost { bytes } else { 0 }; + match &mut shown.omitted { + Some(omission) => { + omission.bytes += bytes; + omission.lost_bytes += lost_bytes; + } + None => { + shown.omitted = Some(Omission { + offset, + bytes, + lost_bytes, + }) + } + } + } + } + } + shown + } + + /// Keep at most `budget` bytes: the first half of the budget from the + /// head and the rest from the end of what was read, omitting the middle. + /// Used when a single page returned more than the model may see. + pub fn trim_to(mut self, budget: u64) -> Self { + let total = self.shown_bytes(); + if total <= budget { + return self; + } + let head_budget = budget / 2; + let tail_budget = budget - head_budget; + let mut chunks: Vec = self.head.drain(..).chain(self.tail.drain(..)).collect(); + let first = chunks + .first() + .map(|chunk| chunk.offset) + .unwrap_or(self.next); + let mut head = Vec::new(); + let mut taken = 0u64; + let mut cut_at = first; + let mut index = 0; + while index < chunks.len() && taken < head_budget { + let chunk = &mut chunks[index]; + let room = head_budget - taken; + let length = chunk.data.len() as u64; + if length <= room { + taken += length; + cut_at = chunk.offset + length; + head.push(chunk.clone()); + index += 1; + } else { + head.push(slice_chunk(chunk, 0, room)); + cut_at = chunk.offset + room; + taken = head_budget; + } + } + let mut tail = Vec::new(); + let mut kept = 0u64; + for chunk in chunks.iter().rev() { + if kept >= tail_budget { + break; + } + let length = chunk.data.len() as u64; + let chunk_end = chunk.offset + length; + let start = chunk.offset.max(cut_at); + if chunk_end <= start { + break; + } + let available = chunk_end - start; + let keep = available.min(tail_budget - kept); + let from = chunk_end - keep - chunk.offset; + tail.push(slice_chunk(chunk, from, keep)); + kept += keep; + } + tail.reverse(); + let dropped = total - taken - kept; + let omitted = self.omitted_bytes() + dropped; + let offset = self + .omitted + .map_or(cut_at, |omission| omission.offset.min(cut_at)); + self.head = head; + self.tail = tail; + let lost_bytes = self.lost_bytes(); + self.omitted = (omitted > 0).then_some(Omission { + offset, + bytes: omitted, + lost_bytes, + }); + self + } + + pub fn shown_bytes(&self) -> u64 { + self.head + .iter() + .chain(&self.tail) + .map(|chunk| chunk.data.len() as u64) + .sum() + } + + pub fn omitted_bytes(&self) -> u64 { + self.omitted.map_or(0, |omission| omission.bytes) + } + + /// Omitted bytes the environment no longer holds. + pub fn lost_bytes(&self) -> u64 { + self.omitted.map_or(0, |omission| omission.lost_bytes) + } +} + +enum Item { + Chunk(OutputChunk), + /// `lost`: a gap the environment reported; otherwise a readable range + /// between two pages. + Gap { + offset: u64, + bytes: u64, + lost: bool, + }, +} + +/// Where a page begins: its first chunk or gap, or `next` when empty. +fn page_start(page: &OutputPage) -> u64 { + let first_chunk = page.chunks.first().map(|chunk| chunk.offset); + let first_gap = page.gaps.first().map(|gap| gap.offset); + match (first_chunk, first_gap) { + (Some(chunk), Some(gap)) => chunk.min(gap), + (Some(offset), None) | (None, Some(offset)) => offset, + (None, None) => page.next, + } +} + +/// Append a page's chunks and gaps in offset order, dropping bytes before +/// `skip_before`. +fn push_page(items: &mut Vec, page: OutputPage, skip_before: u64) { + let mut entries: Vec = page + .chunks + .into_iter() + .map(Item::Chunk) + .chain(page.gaps.into_iter().map(|gap| Item::Gap { + offset: gap.offset, + bytes: gap.omitted_bytes, + lost: true, + })) + .collect(); + entries.sort_by_key(|item| match item { + Item::Chunk(chunk) => chunk.offset, + Item::Gap { offset, .. } => *offset, + }); + for item in entries { + match item { + Item::Chunk(chunk) => { + let length = chunk.data.len() as u64; + if chunk.offset + length <= skip_before { + continue; + } + if chunk.offset < skip_before { + let from = skip_before - chunk.offset; + items.push(Item::Chunk(slice_chunk(&chunk, from, length - from))); + } else { + items.push(Item::Chunk(chunk)); + } + } + Item::Gap { + offset, + bytes, + lost, + } => { + let gap_end = offset + bytes; + if gap_end <= skip_before { + continue; + } + let start = offset.max(skip_before); + items.push(Item::Gap { + offset: start, + bytes: gap_end - start, + lost, + }); + } + } + } +} + +fn slice_chunk(chunk: &OutputChunk, from: u64, length: u64) -> OutputChunk { + let from = from as usize; + let to = from + length as usize; + OutputChunk { + offset: chunk.offset + from as u64, + stream: chunk.stream, + data: chunk.data.as_slice()[from..to].to_vec().into(), + } +} + +/// Whether a chunk belongs to standard error rather than standard output +/// (a PTY merges both into standard output). +pub fn is_stderr(stream: OutputStream) -> bool { + matches!(stream, OutputStream::Stderr) +} + +#[cfg(test)] +mod tests { + use environment_protocol::data::exec::{GapReason, OutputGap}; + + use super::*; + + fn chunk(offset: u64, text: &str) -> OutputChunk { + OutputChunk { + offset, + stream: OutputStream::Stdout, + data: text.as_bytes().to_vec().into(), + } + } + + fn text(chunks: &[OutputChunk]) -> String { + chunks + .iter() + .map(|chunk| String::from_utf8_lossy(chunk.data.as_slice()).into_owned()) + .collect() + } + + #[test] + fn head_and_tail_pages_show_the_skipped_range() { + let head = OutputPage { + chunks: vec![chunk(0, "head")], + next: 4, + end: 100, + gaps: vec![], + }; + let tail = OutputPage { + chunks: vec![chunk(96, "tail")], + next: 100, + end: 100, + gaps: vec![], + }; + let shown = ShownOutput::from_pages(head, Some(tail)); + assert_eq!(text(&shown.head), "head"); + assert_eq!(text(&shown.tail), "tail"); + assert_eq!( + shown.omitted, + Some(Omission { + offset: 4, + bytes: 92, + lost_bytes: 0, + }) + ); + assert_eq!((shown.next, shown.end), (100, 100)); + } + + #[test] + fn overlapping_pages_show_every_byte_once() { + let head = OutputPage { + chunks: vec![chunk(0, "abcdef")], + next: 6, + end: 8, + gaps: vec![], + }; + let tail = OutputPage { + chunks: vec![chunk(4, "efgh")], + next: 8, + end: 8, + gaps: vec![], + }; + let shown = ShownOutput::from_pages(head, Some(tail)); + assert_eq!(text(&shown.head), "abcdefgh"); + assert!(shown.omitted.is_none()); + } + + #[test] + fn daemon_gaps_count_as_omitted() { + let page = OutputPage { + chunks: vec![chunk(0, "ab"), chunk(10, "yz")], + next: 12, + end: 12, + gaps: vec![OutputGap { + offset: 2, + omitted_bytes: 8, + reason: GapReason::Overflow, + }], + }; + let shown = ShownOutput::from_pages(page, None); + assert_eq!(text(&shown.head), "ab"); + assert_eq!(text(&shown.tail), "yz"); + assert_eq!(shown.omitted_bytes(), 8); + assert_eq!( + shown.lost_bytes(), + 8, + "an overflow gap cannot be read again" + ); + } + + #[test] + fn retention_gaps_and_skipped_ranges_are_told_apart() { + let head = OutputPage { + chunks: vec![chunk(0, "head")], + next: 14, + end: 100, + gaps: vec![OutputGap { + offset: 4, + omitted_bytes: 10, + reason: GapReason::Overflow, + }], + }; + let tail = OutputPage { + chunks: vec![chunk(96, "tail")], + next: 100, + end: 100, + gaps: vec![], + }; + let shown = ShownOutput::from_pages(head, Some(tail)); + assert_eq!( + shown.omitted, + Some(Omission { + offset: 4, + bytes: 92, + lost_bytes: 10, + }) + ); + } + + #[test] + fn trimming_keeps_the_first_and_last_bytes() { + let page = OutputPage { + chunks: vec![chunk(0, "0123456789"), chunk(10, "abcdefghij")], + next: 20, + end: 20, + gaps: vec![], + }; + let shown = ShownOutput::from_pages(page, None).trim_to(8); + assert_eq!(text(&shown.head), "0123"); + assert_eq!(text(&shown.tail), "ghij"); + assert_eq!( + shown.omitted, + Some(Omission { + offset: 4, + bytes: 12, + lost_bytes: 0, + }) + ); + assert_eq!(shown.shown_bytes(), 8); + } +} diff --git a/crates/tools/src/environment/process/mod.rs b/crates/tools/src/environment/process/mod.rs index 60a6ca374..7008d6be7 100644 --- a/crates/tools/src/environment/process/mod.rs +++ b/crates/tools/src/environment/process/mod.rs @@ -11,7 +11,7 @@ use std::collections::BTreeMap; use async_trait::async_trait; -use environment_protocol::shared::SecretString; +use environment_protocol::bytes::SecretString; use serde::{Deserialize, Serialize}; use thiserror::Error; diff --git a/crates/tools/src/environment/sources.rs b/crates/tools/src/environment/sources.rs index 22a508cf3..e4cb070dc 100644 --- a/crates/tools/src/environment/sources.rs +++ b/crates/tools/src/environment/sources.rs @@ -1,7 +1,10 @@ //! Shared machine source scope, resolved from the session's effective directory. use environment_protocol::{ - data::inventory::{InventoryLimits, ScanParams}, - shared::EnvironmentPath, + data::{ + fs::TreeFilter, + scan::{ScanLimits, ScanParams, ScanRoot}, + }, + paths::EnvironmentPath, }; use std::{ collections::BTreeSet, @@ -61,30 +64,77 @@ pub fn scan_query( } } } - if roots.len() > 32 { - return Err("discovery exceeds 32 roots".into()); + // One root slot of the protocol's 32 stays free for the working + // directory probe. + if roots.len() > 31 { + return Err("discovery exceeds 31 roots".into()); } + // Discovery roots are often hidden (`.agents`) or ignored by a project's + // ignore files, so both defaults are opted out of below each root. Ok(ScanParams { roots: roots .into_iter() - .map(|p| EnvironmentPath::new(p.to_string_lossy()).map_err(|e| e.to_string())) - .collect::>()?, - include_patterns: if source == "prompts" { - vec!["*.md".into(), "*.txt".into()] - } else { - vec!["SKILL.md".into(), "**/SKILL.md".into()] + .map(|path| { + Ok(ScanRoot { + path: EnvironmentPath::new(path.to_string_lossy()) + .map_err(|e| e.to_string())?, + if_none_match: None, + max_depth: None, + }) + }) + .collect::>()?, + filter: TreeFilter { + include: if source == "prompts" { + vec!["*.md".into(), "*.txt".into()] + } else { + vec!["SKILL.md".into()] + }, + include_hidden: true, + include_ignored: true, + max_depth: Some(if source == "prompts" { 1 } else { 8 }), + ..TreeFilter::default() }, read_content: true, + include_digests: false, follow_symlinks: true, - digest_algorithm: None, - if_none_match: None, - limits: InventoryLimits { + limits: ScanLimits { max_entries: 4096, - max_depth: if source == "prompts" { 1 } else { 8 }, max_file_bytes: 64 * 1024, max_total_bytes: 2 * 1024 * 1024, - max_manifest_bytes: 4 * 1024 * 1024, max_duration_ms: 2000, }, }) } + +/// Add the working-directory probe to a discovery scan: a root examined at +/// depth 0, so the same observation proves `cwd` is an existing directory +/// (`complete` without entries) instead of a separate `fs/getMetadata`. +/// The probe is the last root of the response. +pub fn with_working_directory_probe(params: &mut ScanParams, cwd: &EnvironmentPath) { + params.roots.push(ScanRoot { + path: cwd.clone(), + if_none_match: None, + max_depth: Some(0), + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovery_scans_select_by_name_and_probe_the_working_directory() { + let mut params = scan_query(None, "/project", Some("/home/dev"), "skills").unwrap(); + assert_eq!(params.filter.include, vec!["SKILL.md"]); + assert!(params.filter.include_hidden && params.filter.include_ignored); + let cwd = EnvironmentPath::new("/project").unwrap(); + with_working_directory_probe(&mut params, &cwd); + let probe = params.roots.last().unwrap(); + assert_eq!( + (probe.path.as_str(), probe.max_depth), + ("/project", Some(0)) + ); + let prompts = scan_query(None, "/project", Some("/home/dev"), "prompts").unwrap(); + assert_eq!(prompts.filter.max_depth, Some(1)); + } +} diff --git a/crates/tools/src/environment/tools/jobs.rs b/crates/tools/src/environment/tools/jobs.rs index bb919b948..bb9d31cc0 100644 --- a/crates/tools/src/environment/tools/jobs.rs +++ b/crates/tools/src/environment/tools/jobs.rs @@ -1,14 +1,17 @@ //! Canonical durable-job operations. -use environment_protocol::data::jobs::{ReadJobsParams, StartJobsParams, StartJobsResponse}; +use environment_protocol::{ + data::job::{JobReadEntry, StartJobsParams, StartJobsResponse}, + ids::JobRequestId, +}; use crate::{ environment::{ EnvironmentToolContext, jobs::{ - JobError, JobHandle, JobReadArgs, JobSubmitArgs, JobSubmitResult, JobSubmitted, - ModelJobResultSet, NormalizeJobResultInput, normalize_job_result, - visible_job_read_output, + DEFAULT_JOB_OUTPUT_BYTES, JobHandle, JobReadArgs, JobSubmitArgs, JobSubmitResult, + JobSubmitted, ModelJobResultSet, NormalizeJobResultInput, job_output_read, + normalize_job_result, visible_job_read_output, }, }, error::ToolResult, @@ -37,32 +40,55 @@ pub async fn invoke_job_read( return Err(invalid_request("job_read requires at least one job")); } let jobs = ctx.jobs.as_ref().ok_or_else(unsupported_job_capability)?; - let response = jobs - .read_jobs(ReadJobsParams { - namespace: job_namespace(ctx, None)?, - jobs: args.jobs.into_iter().map(|handle| handle.job_id).collect(), - after_seq: args.after_seq, - max_bytes: args.output_bytes, - include_artifacts: args.include_artifacts, - wait_ms: None, - }) - .await?; - let mut normalized = Vec::with_capacity(response.jobs.len()); - for job in response.jobs { - normalized.push( - normalize_job_result( - ctx.blobs.as_ref(), - NormalizeJobResultInput { - summary: Some(job.summary), - output_chunks: job.output_chunks, - output_next_seq: job.output_next_seq, - artifacts: job.artifacts, - output_bytes: args.output_bytes, - ..Default::default() - }, - ) - .await?, - ); + let job_ids = args + .jobs + .iter() + .map(|handle| handle.job_id.clone()) + .collect::>(); + let mut output_bytes = args + .output_bytes + .unwrap_or(DEFAULT_JOB_OUTPUT_BYTES) + .min(ctx.limits.max_model_visible_output_bytes); + // Without `after` every job takes two `job/read` entries (head and + // tail); the environment's `maxRequestJobs` bounds entries, not jobs, + // so the jobs go out in as many requests as that bound needs. + let mut after = args.after; + let mut jobs_per_request = job_ids.len(); + if let Some(limits) = jobs.limits() { + let max_entries = limits.max_request_jobs as usize; + if job_ids.len() > max_entries { + return Err(invalid_request(format!( + "job_read names {} jobs; the environment reads at most {} at once", + job_ids.len(), + limits.max_request_jobs + ))); + } + if after.is_none() && max_entries < 2 { + // No room for a tail entry: show the head only. + after = Some(0); + } + let per_job = if after.is_some() { 1 } else { 2 }; + jobs_per_request = (max_entries / per_job).max(1); + let in_one_request = jobs_per_request.min(job_ids.len()) as u64; + output_bytes = output_bytes.min(limits.max_read_bytes / in_one_request); + } + let per_job = if after.is_some() { 1 } else { 2 }; + let mut normalized = Vec::with_capacity(job_ids.len()); + for chunk in job_ids.chunks(jobs_per_request.max(1)) { + let response = jobs + .read_jobs(job_output_read(chunk, after, output_bytes)) + .await?; + let mut entries = response.jobs.into_iter(); + for _ in chunk { + let job_entries: Vec = entries.by_ref().take(per_job).collect(); + normalized.push( + normalize_job_result( + ctx.blobs.as_ref(), + NormalizeJobResultInput::from_entries(None, job_entries), + ) + .await?, + ); + } } Ok(ModelJobResultSet { jobs: normalized }) } @@ -75,29 +101,29 @@ fn submit_params_from_args( ctx: &EnvironmentToolContext, args: JobSubmitArgs, ) -> ToolResult { + let working_directory = ctx.process_cwd.as_ref().map(|cwd| cwd.as_str()); let mut specs = Vec::with_capacity(args.jobs.len()); for spec in args.jobs { let job_id = spec.job_id.clone(); - specs.push(spec.into_protocol_spec(job_id)?); + specs.push(spec.into_job_spec(job_id, working_directory)?); } Ok(StartJobsParams { - namespace: job_namespace(ctx, None)?, - request_id: "default".to_owned(), + request_id: request_id(ctx), jobs: specs, }) } -fn job_namespace( - ctx: &EnvironmentToolContext, - explicit_session_id: Option<&str>, -) -> ToolResult { - explicit_session_id - .map(ToOwned::to_owned) - .or_else(|| ctx.session_id.clone()) - .ok_or_else(|| JobError::InvalidRequest { - message: "environment job namespace requires a session_id".to_owned(), +/// One request id per call: the invocation's stable operation identity, so +/// a retried call replays its jobs while a new call that reuses a job id is +/// refused instead of answered with an earlier run. +fn request_id(ctx: &EnvironmentToolContext) -> JobRequestId { + ctx.operation_id + .as_deref() + .and_then(|operation_id| JobRequestId::new(operation_id).ok()) + .unwrap_or_else(|| { + JobRequestId::new(format!("jobreq-{}", uuid::Uuid::new_v4().simple())) + .expect("uuid request id fits the id grammar") }) - .map_err(Into::into) } fn submit_result_from_response( @@ -107,21 +133,162 @@ fn submit_result_from_response( JobSubmitResult { jobs: response .jobs - .into_iter() - .map(|summary| JobSubmitted { - name: summary.name, - handle: ctx.environment_id.clone().map(|environment_id| JobHandle { + .iter() + .map(|summary| { + let handle = ctx.environment_id.clone().map(|environment_id| JobHandle { environment_id: crate::environment::handles::environment_handle( &environment_id, ), job_id: summary.job_id.clone(), - }), - job_id: summary.job_id, - status: summary.status, - dependencies: summary.dependencies, - queue_key: summary.queue_key, - promise: None, + }); + JobSubmitted::from_summary(summary, handle) }) .collect(), } } + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use environment_protocol::{ + data::job::{ + CancelJobsParams, CancelJobsResponse, JobStatus, JobSummary, ReadJobsParams, + ReadJobsResponse, + }, + ids::JobId, + }; + use harness::storage::InMemoryBlobStore; + use serde_json::json; + + use super::*; + use crate::environment::jobs::{JobExecResult, JobExecutor}; + + #[derive(Default)] + struct RecordingJobs { + starts: Mutex>, + reads: Mutex>, + limits: Option, + } + + #[async_trait] + impl JobExecutor for RecordingJobs { + fn limits(&self) -> Option { + self.limits.clone() + } + + async fn start_jobs(&self, request: StartJobsParams) -> JobExecResult { + let jobs = request + .jobs + .iter() + .map(|job| JobSummary { + job_id: job.job_id.clone(), + request_id: request.request_id.clone(), + name: None, + status: JobStatus::Queued, + depends_on: Vec::new(), + queue_key: None, + created_at_ms: 1, + queued_at_ms: None, + started_at_ms: None, + finished_at_ms: None, + cancel_requested_at_ms: None, + duration_ms: None, + exit: None, + stopped_by: None, + leftovers: None, + error: None, + }) + .collect(); + self.starts.lock().unwrap().push(request); + Ok(StartJobsResponse { jobs }) + } + + async fn read_jobs(&self, request: ReadJobsParams) -> JobExecResult { + let jobs = request + .jobs + .iter() + .map(|entry| JobReadEntry::Error { + job_id: entry.job_id.clone(), + error: environment_protocol::error::ProtocolError::not_found("unknown job"), + }) + .collect(); + self.reads.lock().unwrap().push(request); + Ok(ReadJobsResponse { jobs }) + } + + async fn cancel_jobs( + &self, + _request: CancelJobsParams, + ) -> JobExecResult { + unreachable!("not cancelled") + } + } + + #[tokio::test(flavor = "current_thread")] + async fn inline_job_submits_send_a_request_id_per_call() { + let jobs = Arc::new(RecordingJobs::default()); + let base = EnvironmentToolContext::new(None, Arc::new(InMemoryBlobStore::new())) + .with_jobs(jobs.clone()); + let args = || { + serde_json::from_value::(json!({ + "jobs": [{"job_id": "build", "argv": ["make"]}] + })) + .unwrap() + }; + let first = base.clone().with_operation_id("tool-first"); + invoke_job_submit(&first, args()).await.unwrap(); + invoke_job_submit(&first, args()).await.unwrap(); + let second = base.with_operation_id("tool-second"); + let submitted = invoke_job_submit(&second, args()).await.unwrap(); + + let starts = jobs.starts.lock().unwrap(); + assert_eq!(starts[0].request_id.as_str(), "tool-first"); + assert_eq!( + starts[0].request_id, starts[1].request_id, + "a retried call replays" + ); + assert_ne!( + starts[0].request_id, starts[2].request_id, + "a new call is a new run" + ); + assert_eq!(starts[0].jobs[0].job_id, JobId::new("build").unwrap()); + assert_eq!( + serde_json::to_value(&submitted).unwrap()["jobs"][0]["status"], + "queued" + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn job_reads_split_head_and_tail_entries_across_requests() { + let jobs = Arc::new(RecordingJobs { + limits: Some(environment_protocol::data::handshake::JobLimits { + max_read_bytes: 4096, + max_request_jobs: 4, + max_retained_output_bytes: 1 << 20, + record_retention_ms: 1, + cancel_grace_ms: 1, + }), + ..RecordingJobs::default() + }); + let ctx = EnvironmentToolContext::new(None, Arc::new(InMemoryBlobStore::new())) + .with_jobs(jobs.clone()); + let args = serde_json::from_value::(json!({ + "jobs": [{"job_id": "a"}, {"job_id": "b"}, {"job_id": "c"}] + })) + .unwrap(); + let result = invoke_job_read(&ctx, args).await.unwrap(); + + assert_eq!(result.jobs.len(), 3); + let reads = jobs.reads.lock().unwrap(); + assert_eq!(reads.len(), 2, "two jobs of head and tail fill one request"); + assert!(reads.iter().all(|read| read.jobs.len() <= 4)); + assert!(reads.iter().all(|read| read.max_bytes <= 4096)); + let named = reads + .iter() + .flat_map(|read| read.jobs.iter().map(|entry| entry.job_id.as_str())) + .collect::>(); + assert_eq!(named, ["a", "a", "b", "b", "c", "c"]); + } +} diff --git a/crates/tools/src/environment_protocol/conformance.rs b/crates/tools/src/environment_protocol/conformance.rs index 94f17e17e..9a22c42a3 100644 --- a/crates/tools/src/environment_protocol/conformance.rs +++ b/crates/tools/src/environment_protocol/conformance.rs @@ -1,263 +1,1312 @@ -//! Reusable conformance checks for environment data-plane implementations. +//! Data-plane conformance: drives any environment data server through +//! [`DataClient`] and checks the contract envd implements. +//! +//! The suite checks, against the wire and not through tool adapters: +//! +//! - the `initialize` response: plane, version, ceilings and capability +//! families; +//! - `fs/*`: staged writes with `create` and `ifMatch` preconditions and a +//! returned version, ranged reads with `fileSize` and `truncated`, +//! ceilings refused as `invalidParams`, `fs/rename` without overwrite, +//! paged `fs/readDirectory`, `fs/remove` preconditions, typed `notFound` +//! and confinement errors, the shared pattern grammar of +//! `fs/globFiles` and `fs/searchText` (and that the tools' generic glob +//! fallback selects the same files), and per-root `fs/scan` results; +//! - confinement of symlinks (when the `process` family can create them): +//! an in-root link is followed, a link that leaves the root is refused +//! with `permissionDenied {outsideRoot}`; +//! - `process/*` (when the family is present): settled exits with codes and +//! named signals, start-time input larger than a pipe buffer, tail reads, +//! duplicate ids refused before anything runs, unknown handles as +//! `notFound`, and graceful `terminate`; +//! - `job/*` (when the family is present): `requestId` replay versus +//! `jobIdInUse`, per-entry `notFound`, and byte-exact paging. +//! +//! The suite works in a fresh directory below `work_directory` and removes +//! it at the end. Commands run through `/bin/sh`. +//! +//! ```no_run +//! # async fn example(client: environment_client::DataClient) { +//! use tools::environment_protocol::conformance::{DataConformance, run}; +//! +//! let work = environment_protocol::paths::EnvironmentPath::new("/workspace").unwrap(); +//! run(&DataConformance::new(client, work)).await.into_result().expect("data plane conforms"); +//! # } +//! ``` -use std::sync::Arc; +use std::{ + collections::BTreeMap, + fmt, + future::Future, + time::{Duration, Instant}, +}; -use environment_client::{EnvironmentClientError, EnvironmentDataClient, JsonRpcTransport}; +use environment_client::{ClientError, DataClient}; use environment_protocol::{ - data::handshake::{InitializeParams, InitializedParams}, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities}, + ENVD_PROTOCOL_VERSION, + bytes::ByteChunk, + data::{ + exec::{CommandSpec, ExecInput, ExecState, ExitStatus, ReturnOn, StoppedBy}, + fs::{ + CreateDirectoryParams, FileKind, GetMetadataParams, GlobFilesLimits, GlobFilesParams, + ReadDirectoryParams, ReadFileParams, RemoveParams, RenameParams, SearchTextLimits, + SearchTextParams, TreeFilter, WriteCondition, WriteFileParams, WriteFileResponse, + }, + handshake::InitializeResponse, + idle::GetIdleParams, + job::{JobReadEntry, JobReadRequest, JobSpec, ReadJobsParams, StartJobsParams}, + process::{ + ProcessSignal, ReadProcessParams, ReadProcessResponse, StartProcessParams, + TerminateProcessParams, WriteProcessParams, + }, + scan::{ScanLimits, ScanParams, ScanRoot, ScanRootResult}, + }, + error::{ErrorData, PermissionReason, PreconditionReason, ProtocolErrorKind}, + handshake::Plane, + ids::{JobId, JobRequestId, ProcessId}, + limits::MIN_CONCURRENT_REQUESTS, + paths::{EnvironmentPath, TreePath}, }; -use harness::storage::InMemoryBlobStore; use thiserror::Error; -use crate::{ - environment_protocol::RemoteEnvironmentConnection, - error::ToolError, - fs::{ - CopyOptions, CreateDirectoryOptions, FileAccessPolicy, FsError, FsPath, FsPathError, - RemoveOptions, - tools::{ReadFileArgs, WriteFileArgs, invoke_read_file, invoke_write_file}, - }, -}; +/// What to run the suite against. +#[derive(Clone)] +pub struct DataConformance { + /// An initialized connection to the server under test. + pub client: DataClient, + /// An existing, writable directory inside the filesystem root. + pub work_directory: EnvironmentPath, + /// A path outside the server's filesystem root, when the root is + /// narrowed; checks that it is refused with `permissionDenied`. + pub outside_root: Option, + /// Bound on waiting for a command to exit. Default 30 s. + pub command_timeout: Duration, +} +impl DataConformance { + pub fn new(client: DataClient, work_directory: EnvironmentPath) -> Self { + Self { + client, + work_directory, + outside_root: None, + command_timeout: Duration::from_secs(30), + } + } +} + +impl fmt::Debug for DataConformance { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DataConformance") + .field("work_directory", &self.work_directory) + .field("outside_root", &self.outside_root) + .finish_non_exhaustive() + } +} + +/// The outcome of one check. #[derive(Clone, Debug)] -pub struct EnvironmentDataConformanceOptions { - pub initialize: InitializeParams, - pub expected_capabilities: EnvironmentCapabilities, - pub expected_default_cwd: FsPath, - /// A relative directory that the suite may create and remove. - pub test_directory: FsPath, - /// An absolute path outside the provider's exposed filesystem route. - pub forbidden_path: FsPath, +pub struct CheckResult { + /// Stable check name, such as `fs.writeFile.preconditions`. + pub name: &'static str, + /// Why the check failed. + pub failure: Option, } -#[derive(Debug, Error)] -pub enum EnvironmentDataConformanceError { - #[error(transparent)] - Client(#[from] EnvironmentClientError), +/// Every check that ran, in order. +#[derive(Clone, Debug, Default)] +pub struct DataConformanceReport { + pub checks: Vec, +} - #[error(transparent)] - Filesystem(#[from] FsError), +impl DataConformanceReport { + pub fn passed(&self) -> bool { + self.checks.iter().all(|check| check.failure.is_none()) + } - #[error(transparent)] - Path(#[from] FsPathError), + pub fn failures(&self) -> impl Iterator { + self.checks.iter().filter(|check| check.failure.is_some()) + } - #[error(transparent)] - Tool(#[from] ToolError), + /// `Ok` when every check passed. + pub fn into_result(self) -> Result { + if self.passed() { + Ok(self) + } else { + Err(DataConformanceFailure(self)) + } + } +} + +/// A report with at least one failed check. +#[derive(Clone, Debug, Error)] +pub struct DataConformanceFailure(pub DataConformanceReport); + +impl fmt::Display for DataConformanceFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "data conformance failed:")?; + for check in self.0.failures() { + write!( + formatter, + "\n- {}: {}", + check.name, + check.failure.as_deref().unwrap_or_default() + )?; + } + Ok(()) + } +} + +type Check = Result<(), String>; + +/// Run every check that the server's capability families allow. +pub async fn run(suite: &DataConformance) -> DataConformanceReport { + let mut report = DataConformanceReport::default(); + let initialized = suite.client.initialize_response().clone(); + let handshake = check_initialize(&initialized); + let handshake_failed = handshake.is_err(); + record(&mut report, "initialize", handshake); + if handshake_failed { + return report; + } + record( + &mut report, + "env.getIdle", + call(suite.client.get_idle(&GetIdleParams {})) + .await + .map(drop), + ); - #[error("environment data conformance check failed: {0}")] - Check(String), + let work = match prepare(suite).await { + Ok(work) => work, + Err(failure) => { + record(&mut report, "fs.prepare", Err(failure)); + return report; + } + }; + let context = Context { + client: &suite.client, + initialized: &initialized, + work: &work, + timeout: suite.command_timeout, + }; + record( + &mut report, + "fs.writeFile.preconditions", + context.write_preconditions().await, + ); + record( + &mut report, + "fs.readFile.range", + context.ranged_reads().await, + ); + record( + &mut report, + "fs.readFile.notRegular", + context.reads_refuse_directories().await, + ); + record(&mut report, "fs.rename", context.rename().await); + record( + &mut report, + "fs.readDirectory.paging", + context.directory_paging().await, + ); + record(&mut report, "fs.remove", context.remove().await); + record( + &mut report, + "fs.errors", + context.errors(suite.outside_root.as_ref()).await, + ); + record( + &mut report, + "fs.globFiles.grammar", + context.glob_grammar().await, + ); + record( + &mut report, + "fs.glob.genericParity", + context.glob_parity().await, + ); + record(&mut report, "fs.searchText", context.search().await); + record(&mut report, "fs.scan.perRoot", context.scan().await); + if initialized.capabilities.process.is_some() { + record( + &mut report, + "fs.confinement.symlinks", + context + .symlink_confinement(suite.outside_root.as_ref()) + .await, + ); + record(&mut report, "process.exit", context.process_exit().await); + record( + &mut report, + "process.signal", + context.process_signal().await, + ); + record( + &mut report, + "process.largeInput", + context.process_large_input().await, + ); + record( + &mut report, + "process.duplicateId", + context.process_duplicate_id().await, + ); + record( + &mut report, + "process.unknown", + context.process_unknown().await, + ); + record( + &mut report, + "process.terminate", + context.process_terminate().await, + ); + } + if initialized.capabilities.job.is_some() { + record(&mut report, "job.requestId", context.job_request_id().await); + record(&mut report, "job.read.unknown", context.job_unknown().await); + record(&mut report, "job.read.paging", context.job_paging().await); + } + record( + &mut report, + "fs.cleanup", + call(suite.client.remove(&RemoveParams { + path: work, + recursive: true, + force: true, + })) + .await + .map(drop), + ); + report +} + +fn record(report: &mut DataConformanceReport, name: &'static str, outcome: Check) { + report.checks.push(CheckResult { + name, + failure: outcome.err(), + }); } -/// Exercises the transport handshake and the generic filesystem adapter. -/// -/// Provider implementations can invoke this from their own integration tests. -/// The supplied test directory is removed after a successful check. -pub async fn assert_environment_data_conformance( - mut client: EnvironmentDataClient, - options: EnvironmentDataConformanceOptions, -) -> Result<(), EnvironmentDataConformanceError> -where - T: JsonRpcTransport + Send + 'static, -{ - if options.test_directory.is_absolute() - || options.test_directory.is_root() - || options.test_directory.has_unresolved_parent() - || options.test_directory.segments().count() != 1 - { - return Err(check( - "test_directory must be a single relative path segment", - )); - } - if !options.forbidden_path.is_absolute() { - return Err(check("forbidden_path must be absolute")); - } - if !options.expected_capabilities.filesystem_read - || !options.expected_capabilities.filesystem_write - { - return Err(check( - "filesystem conformance requires read and write capabilities", - )); - } - - let initialized = client.initialize(&options.initialize).await?; - if initialized.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(check(format!( - "protocol version was {}, expected {CURRENT_PROTOCOL_VERSION}", +fn check_initialize(initialized: &InitializeResponse) -> Check { + ensure( + initialized.plane == Plane::Data, + "initialize did not name the data plane", + )?; + ensure( + initialized.protocol_version == ENVD_PROTOCOL_VERSION, + format!( + "protocolVersion {} differs from {ENVD_PROTOCOL_VERSION}", initialized.protocol_version - ))); - } - if initialized.capabilities != options.expected_capabilities { - return Err(check(format!( - "initialize capabilities were {:?}, expected {:?}", - initialized.capabilities, options.expected_capabilities - ))); - } - if initialized.default_cwd.as_deref() != Some(options.expected_default_cwd.as_str()) { - return Err(check(format!( - "default cwd was {:?}, expected {}", - initialized.default_cwd, options.expected_default_cwd - ))); - } - client.initialized(&InitializedParams {}).await?; - - let connection = RemoteEnvironmentConnection::new(client, initialized.capabilities) - .with_cwd(options.expected_default_cwd.clone()); - if connection.process_executor().is_some() != options.expected_capabilities.process_start { - return Err(check( - "process executor did not follow process_start capability", - )); - } - let has_any_job_capability = options.expected_capabilities.job_start - || options.expected_capabilities.job_list - || options.expected_capabilities.job_read - || options.expected_capabilities.job_cancel; - if connection.job_executor().is_some() != has_any_job_capability { - return Err(check("job executor did not follow advertised capabilities")); - } - - let (fs_context, _) = connection.into_contexts(Arc::new(InMemoryBlobStore::new())); - if fs_context.fs_cwd.as_ref() != Some(&options.expected_default_cwd) { - return Err(check("filesystem tool context did not retain default cwd")); - } - if fs_context.fs.access_policy() != FileAccessPolicy::FullReadWrite { - return Err(check( - "writable environment filesystem was not exposed as read-write", - )); - } - - let nested = options.test_directory.join("nested")?; - let source = nested.join("source.txt")?; - let copy = nested.join("copy.txt")?; - let copied_tree = FsPath::new(format!("{}-copy", options.test_directory))?; - let contents = "environment data conformance\n"; - - let write = invoke_write_file( - &fs_context, - WriteFileArgs { - path: source.clone(), - content: Some(contents.to_owned()), - content_ref: None, - }, - ) - .await?; - let expected_source = options.expected_default_cwd.join_path(&source)?; - if write.resolved_path != expected_source { - return Err(check(format!( - "relative tool path resolved to {}, expected {expected_source}", - write.resolved_path - ))); - } - - let read = invoke_read_file( - &fs_context, - ReadFileArgs { - path: source.clone(), - offset: None, - limit: None, - }, - ) + ), + )?; + ensure( + initialized.limits.max_concurrent_requests >= MIN_CONCURRENT_REQUESTS, + "maxConcurrentRequests is below the protocol minimum", + )?; + let filesystem = initialized + .capabilities + .filesystem + .as_ref() + .ok_or("the suite needs the filesystem family")?; + ensure(filesystem.write, "the suite needs a writable filesystem") +} + +async fn prepare(suite: &DataConformance) -> Result { + let work = join( + &suite.work_directory, + &format!("conformance-{}", uuid::Uuid::new_v4().simple()), + )?; + call(suite.client.create_directory(&CreateDirectoryParams { + path: work.clone(), + recursive: false, + })) .await?; - if read.text != contents.trim_end() { - return Err(check( - "read_file did not return bytes written by write_file", - )); - } - - let source_metadata = fs_context.fs.get_metadata(&expected_source).await?; - if !source_metadata.is_file || source_metadata.is_directory { - return Err(check("file metadata did not identify a regular file")); - } - let expected_nested = options.expected_default_cwd.join_path(&nested)?; - let directory_metadata = fs_context.fs.get_metadata(&expected_nested).await?; - if !directory_metadata.is_directory || directory_metadata.is_file { - return Err(check("directory metadata did not identify a directory")); - } - if !matches!( - fs_context - .fs - .create_directory(&expected_nested, CreateDirectoryOptions::single()) - .await, - Err(FsError::AlreadyExists { .. }) - ) { - return Err(check( - "provider conflict did not map to FsError::AlreadyExists", - )); - } - let entries = fs_context.fs.read_directory(&expected_nested).await?; - if !entries - .iter() - .any(|entry| entry.file_name == "source.txt" && entry.is_file) - { - return Err(check("read_directory omitted the written file")); - } - - let expected_copy = options.expected_default_cwd.join_path(©)?; - fs_context - .fs - .copy(&expected_source, &expected_copy, CopyOptions::file()) + Ok(work) +} + +struct Context<'a> { + client: &'a DataClient, + initialized: &'a InitializeResponse, + work: &'a EnvironmentPath, + timeout: Duration, +} + +impl Context<'_> { + fn path(&self, relative: &str) -> Result { + let tree = TreePath::new(relative).map_err(|error| error.to_string())?; + self.work.join(&tree).map_err(|error| error.to_string()) + } + + async fn write(&self, relative: &str, data: &[u8]) -> Result { + call(self.client.write_file(&WriteFileParams { + path: self.path(relative)?, + data: ByteChunk::from(data), + create: WriteCondition::Always, + if_match: None, + executable: false, + })) + .await + } + + async fn mkdir(&self, relative: &str) -> Check { + call(self.client.create_directory(&CreateDirectoryParams { + path: self.path(relative)?, + recursive: true, + })) + .await + .map(drop) + } + + async fn write_preconditions(&self) -> Check { + let path = self.path("precondition.txt")?; + let written = self.write("precondition.txt", b"one").await?; + let read = call(self.client.read_file(&ReadFileParams { + path: path.clone(), + offset: 0, + max_bytes: 16, + })) + .await?; + ensure( + read.version == written.version, + "a read after a write reports a different version", + )?; + let absent = self + .client + .write_file(&WriteFileParams { + path: path.clone(), + data: ByteChunk::from(&b"two"[..]), + create: WriteCondition::IfAbsent, + if_match: None, + executable: false, + }) + .await; + expect_precondition(absent, PreconditionReason::CreateCondition)?; + let rewritten = call(self.client.write_file(&WriteFileParams { + path: path.clone(), + data: ByteChunk::from(&b"two"[..]), + create: WriteCondition::IfPresent, + if_match: Some(written.version.clone()), + executable: false, + })) + .await?; + ensure( + rewritten.version != written.version, + "a changed file kept its version", + )?; + let stale = self + .client + .write_file(&WriteFileParams { + path, + data: ByteChunk::from(&b"three"[..]), + create: WriteCondition::Always, + if_match: Some(written.version), + executable: false, + }) + .await; + expect_precondition(stale, PreconditionReason::IfMatchFailed) + } + + async fn ranged_reads(&self) -> Check { + self.write("range.txt", b"0123456789").await?; + let path = self.path("range.txt")?; + let middle = call(self.client.read_file(&ReadFileParams { + path: path.clone(), + offset: 2, + max_bytes: 3, + })) .await?; - if fs_context.fs.read_file(&expected_copy).await? != contents.as_bytes() { - return Err(check("file copy did not preserve contents")); - } - - let expected_test_directory = options - .expected_default_cwd - .join_path(&options.test_directory)?; - let expected_copied_tree = options.expected_default_cwd.join_path(&copied_tree)?; - fs_context - .fs - .copy( - &expected_test_directory, - &expected_copied_tree, - CopyOptions::recursive(), + ensure( + middle.data.as_slice() == b"234", + "offset and maxBytes select the range", + )?; + ensure(middle.file_size == 10, "fileSize is the whole file's size")?; + ensure( + middle.truncated, + "truncated is set when bytes follow the range", + )?; + let end = call(self.client.read_file(&ReadFileParams { + path: path.clone(), + offset: 7, + max_bytes: 10, + })) + .await?; + ensure( + end.data.as_slice() == b"789" && !end.truncated, + "a range reaching the end is not truncated", + )?; + let ceiling = self.initialized.limits.filesystem.max_read_file_bytes; + let above = self + .client + .read_file(&ReadFileParams { + path, + offset: 0, + max_bytes: ceiling + 1, + }) + .await; + match above { + Err(error) => match error.protocol_error().map(|error| &error.data) { + Some(ErrorData::InvalidParams { + ceiling: Some(reported), + .. + }) if *reported == ceiling => Ok(()), + _ => Err(format!("maxBytes above the ceiling answered {error}")), + }, + Ok(_) => Err("maxBytes above the ceiling was accepted, not refused".into()), + } + } + + async fn reads_refuse_directories(&self) -> Check { + self.mkdir("directory").await?; + let read = self + .client + .read_file(&ReadFileParams { + path: self.path("directory")?, + offset: 0, + max_bytes: 16, + }) + .await; + match read { + Err(error) + if matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::FailedPrecondition { + reason: PreconditionReason::IsDirectory + | PreconditionReason::NotRegularFile + }) + ) => + { + Ok(()) + } + other => Err(format!("reading a directory answered {other:?}")), + } + } + + async fn rename(&self) -> Check { + self.write("rename-a.txt", b"a").await?; + self.write("rename-b.txt", b"b").await?; + let rename = |source: &str, destination: &str, overwrite| { + Ok::<_, String>(RenameParams { + source: self.path(source)?, + destination: self.path(destination)?, + overwrite, + if_match: None, + }) + }; + expect_kind( + self.client + .rename(&rename("rename-a.txt", "rename-b.txt", false)?) + .await, + ProtocolErrorKind::AlreadyExists, + )?; + call( + self.client + .rename(&rename("rename-a.txt", "rename-c.txt", false)?), ) .await?; - let copied_source = expected_copied_tree.join("nested/source.txt")?; - if fs_context.fs.read_file(&copied_source).await? != contents.as_bytes() { - return Err(check("recursive directory copy did not preserve contents")); - } - - let missing = options - .expected_default_cwd - .join("missing-conformance-file")?; - if !matches!( - fs_context.fs.read_file(&missing).await, - Err(FsError::NotFound { .. }) - ) { - return Err(check("provider notFound did not map to FsError::NotFound")); - } - if !matches!( - fs_context.fs.read_file(&options.forbidden_path).await, - Err(FsError::PermissionDenied { .. }) - ) { - return Err(check( - "route escape did not map to FsError::PermissionDenied", - )); - } - - fs_context - .fs - .remove(&expected_copy, RemoveOptions::file()) + expect_kind( + self.client + .rename(&rename("rename-a.txt", "rename-d.txt", false)?) + .await, + ProtocolErrorKind::NotFound, + )?; + let moved = call(self.client.read_file(&ReadFileParams { + path: self.path("rename-c.txt")?, + offset: 0, + max_bytes: 16, + })) .await?; - fs_context - .fs - .remove(&expected_copied_tree, RemoveOptions::recursive()) + ensure( + moved.data.as_slice() == b"a", + "rename did not move the content", + ) + } + + async fn directory_paging(&self) -> Check { + self.mkdir("listing").await?; + for name in ["c", "a", "b"] { + self.write(&format!("listing/{name}"), b"").await?; + } + let mut names = Vec::new(); + let mut cursor = None; + for _ in 0..5 { + let page = call(self.client.read_directory(&ReadDirectoryParams { + path: self.path("listing")?, + limit: 1, + cursor, + })) + .await?; + ensure(page.entries.len() <= 1, "a page exceeded its limit")?; + names.extend( + page.entries + .into_iter() + .map(|entry| entry.name.as_str().to_owned()), + ); + cursor = page.next_cursor; + if cursor.is_none() { + break; + } + } + ensure( + names == ["a", "b", "c"], + format!("paged listing returned {names:?}, not the names in order"), + ) + } + + async fn remove(&self) -> Check { + self.mkdir("removal").await?; + self.write("removal/file", b"x").await?; + let remove = |recursive, force, relative: &str| { + Ok::<_, String>(RemoveParams { + path: self.path(relative)?, + recursive, + force, + }) + }; + expect_precondition( + self.client.remove(&remove(false, false, "removal")?).await, + PreconditionReason::NotEmpty, + )?; + call(self.client.remove(&remove(true, false, "removal")?)).await?; + call(self.client.remove(&remove(false, true, "removal")?)).await?; + expect_kind( + self.client.remove(&remove(false, false, "removal")?).await, + ProtocolErrorKind::NotFound, + ) + } + + async fn errors(&self, outside_root: Option<&EnvironmentPath>) -> Check { + expect_kind( + self.client + .read_file(&ReadFileParams { + path: self.path("missing")?, + offset: 0, + max_bytes: 1, + }) + .await, + ProtocolErrorKind::NotFound, + )?; + if let Some(outside) = outside_root { + let read = self + .client + .read_file(&ReadFileParams { + path: outside.clone(), + offset: 0, + max_bytes: 1, + }) + .await; + match read { + Err(error) + if matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::PermissionDenied { + reason: Some(PermissionReason::OutsideRoot) + }) + ) => {} + other => return Err(format!("a path outside the root answered {other:?}")), + } + } + Ok(()) + } + + async fn glob_grammar(&self) -> Check { + self.mkdir("glob/sub").await?; + self.write("glob/a.rs", b"").await?; + self.write("glob/sub/b.rs", b"").await?; + let ceilings = self.initialized.limits.filesystem.glob; + let glob = |include: &str, max_depth| GlobFilesParams { + root: self.path("glob").expect("glob root"), + filter: TreeFilter { + include: vec![include.to_owned()], + max_depth, + ..TreeFilter::default() + }, + limits: GlobFilesLimits { + max_matches: ceilings.max_matches.min(100), + max_files: ceilings.max_files.min(1000), + max_duration_ms: ceilings.max_duration_ms.min(10_000), + }, + }; + let names = |matches: Vec| { + matches + .iter() + .filter_map(|path| path.file_name().map(str::to_owned)) + .collect::>() + }; + let segment = call(self.client.glob_files(&glob("*.rs", None))).await?; + ensure( + names(segment.matches) == ["a.rs", "b.rs"], + "a pattern without '/' matches names at any depth", + )?; + let anchored = call(self.client.glob_files(&glob("/*.rs", None))).await?; + ensure( + names(anchored.matches) == ["a.rs"], + "'*' stays within one segment and a leading '/' anchors", + )?; + let shallow = call(self.client.glob_files(&glob("**/*.rs", Some(1)))).await?; + ensure( + names(shallow.matches) == ["a.rs"], + "maxDepth 1 selects the root's direct children", + ) + } + + /// The tools' `glob` over this server and over an in-memory filesystem + /// holding the same tree select the same files: the generic fallback + /// follows the grammar and the walk defaults of `fs/globFiles`. + async fn glob_parity(&self) -> Check { + use crate::fs::{ + CreateDirectoryOptions, FileSystem as _, FsPath, FsToolContext, InMemoryFileSystem, + tools::{GlobArgs, invoke_glob}, + }; + + const FILES: [&str; 10] = [ + "parity/a.rs", + "parity/notes.md", + "parity/sub/b.rs", + "parity/sub/deep/c.rs", + "parity/.hidden/d.rs", + "parity/.e.rs", + "parity/build/out.rs", + "parity/skipped.rs", + "parity/sub/.gitignore", + "parity/.ignore", + ]; + const PATTERNS: [&str; 8] = [ + "*.rs", + "/*.rs", + "sub/*.rs", + "**/*.rs", + "sub/**", + "build/", + "*.{rs,md}", + "deep/*.rs", + ]; + let memory = InMemoryFileSystem::full_access(); + let local_root = FsPath::new("/parity").map_err(|error| error.to_string())?; + for file in FILES { + let contents: &[u8] = match file { + "parity/sub/.gitignore" => b"deep/\n", + "parity/.ignore" => b"skipped.rs\n", + _ => b"", + }; + if let Some((directory, _)) = file.rsplit_once('/') { + self.mkdir(directory).await?; + memory + .create_directory( + &FsPath::new(format!("/{directory}")).map_err(|error| error.to_string())?, + CreateDirectoryOptions::recursive(), + ) + .await + .map_err(|error| error.to_string())?; + } + self.write(file, contents).await?; + memory + .write_file( + &FsPath::new(format!("/{file}")).map_err(|error| error.to_string())?, + contents.to_vec(), + ) + .await + .map_err(|error| error.to_string())?; + } + let blobs: std::sync::Arc = + std::sync::Arc::new(harness::storage::InMemoryBlobStore::new()); + let remote_root = + FsPath::new(self.path("parity")?.as_str()).map_err(|error| error.to_string())?; + let remote = FsToolContext::new( + std::sync::Arc::new( + super::RemoteEnvironmentConnection::new(self.client.clone()).file_system(), + ), + blobs.clone(), + ); + let local = FsToolContext::new(std::sync::Arc::new(memory), blobs); + let relative = |root: &FsPath, matches: Vec| { + matches + .iter() + .map(|path| { + path.as_str() + .strip_prefix(root.as_str()) + .unwrap_or(path.as_str()) + .to_owned() + }) + .collect::>() + }; + for pattern in PATTERNS { + let args = |root: &FsPath| GlobArgs { + pattern: pattern.to_owned(), + path: Some(root.clone()), + max_depth: None, + limit: None, + }; + let native = invoke_glob(&remote, args(&remote_root)) + .await + .map_err(|error| format!("glob {pattern:?} on the server: {error}"))?; + let generic = invoke_glob(&local, args(&local_root)) + .await + .map_err(|error| format!("generic glob {pattern:?}: {error}"))?; + let native = relative(&remote_root, native.matches); + let generic = relative(&local_root, generic.matches); + ensure( + native == generic, + format!( + "glob {pattern:?} selected {native:?} on the server but {generic:?} in the generic fallback" + ), + )?; + } + Ok(()) + } + + async fn search(&self) -> Check { + self.mkdir("search").await?; + self.write("search/notes.txt", b"alpha\nneedle here\n") + .await?; + let ceilings = self.initialized.limits.filesystem.search; + let response = call(self.client.search_text(&SearchTextParams { + root: self.path("search")?, + pattern: "NEEDLE".into(), + pattern_ignore_case: true, + filter: TreeFilter { + include: vec!["*.txt".into()], + ..TreeFilter::default() + }, + limits: SearchTextLimits { + max_matches: ceilings.max_matches.min(10), + max_files: ceilings.max_files.min(100), + max_bytes: ceilings.max_bytes.min(1 << 20), + max_duration_ms: ceilings.max_duration_ms.min(10_000), + max_line_bytes: ceilings.max_line_bytes, + }, + })) .await?; - fs_context - .fs - .remove(&expected_test_directory, RemoveOptions::recursive()) + ensure( + matches!(response.matches.as_slice(), [found] if found.line_number == 2 && found.line == "needle here"), + format!("search answered {:?}", response.matches), + ) + } + + async fn scan(&self) -> Check { + self.mkdir("scan").await?; + self.write("scan/one.md", b"# one").await?; + let ceilings = self.initialized.limits.filesystem.scan; + let response = call(self.client.scan(&ScanParams { + roots: vec![ + ScanRoot { + path: self.path("scan")?, + if_none_match: None, + max_depth: None, + }, + ScanRoot { + path: self.path("scan-missing")?, + if_none_match: None, + max_depth: None, + }, + ScanRoot { + path: self.work.clone(), + if_none_match: None, + max_depth: Some(0), + }, + ], + filter: TreeFilter { + include: vec!["*.md".into()], + ..TreeFilter::default() + }, + read_content: true, + include_digests: false, + follow_symlinks: false, + limits: ScanLimits { + max_entries: ceilings.max_entries.min(100), + max_file_bytes: ceilings.max_file_bytes.min(4096), + max_total_bytes: ceilings.max_total_bytes.min(65536), + max_duration_ms: ceilings.max_duration_ms.min(10_000), + }, + })) .await?; - if !matches!( - fs_context.fs.get_metadata(&expected_test_directory).await, - Err(FsError::NotFound { .. }) - ) { - return Err(check("recursive remove left the test directory behind")); + match response.roots.as_slice() { + [ + ScanRootResult::Complete { entries, .. }, + ScanRootResult::Missing { .. }, + ScanRootResult::Complete { entries: probe, .. }, + ] if entries.len() == 1 + && entries[0].data.as_ref().map(ByteChunk::as_slice) == Some(&b"# one"[..]) + && probe.is_empty() => + { + Ok(()) + } + other => Err(format!("scan answered {other:?}")), + } + } + + /// An in-root symlink is followed; one whose target leaves the root is + /// refused with `permissionDenied {outsideRoot}` by `fs/readFile` and a + /// following `fs/getMetadata`, while the link itself stays describable. + async fn symlink_confinement(&self, outside_root: Option<&EnvironmentPath>) -> Check { + self.write("link-target.txt", b"inside").await?; + let mut script = "ln -s link-target.txt link-inside".to_owned(); + if let Some(outside) = outside_root { + script.push_str(&format!(" && ln -s '{outside}' link-outside")); + } + let process_id = self.start(&script, ExecInput::None {}).await?; + let (_, exited) = self.read_to_exit(&process_id).await?; + ensure( + exited.exit == Some(ExitStatus::Code { code: 0 }), + format!("creating the links exited {:?}", exited.exit), + )?; + let inside = call(self.client.read_file(&ReadFileParams { + path: self.path("link-inside")?, + offset: 0, + max_bytes: 16, + })) + .await?; + ensure( + inside.data.as_slice() == b"inside", + "reading an in-root link did not read its target", + )?; + let followed = call(self.client.get_metadata(&GetMetadataParams { + path: self.path("link-inside")?, + follow_symlinks: true, + })) + .await?; + ensure( + followed.metadata.kind == FileKind::File, + format!( + "a followed in-root link described a {:?}", + followed.metadata.kind + ), + )?; + if outside_root.is_none() { + return Ok(()); + } + let outside = self.path("link-outside")?; + let link = call(self.client.get_metadata(&GetMetadataParams { + path: outside.clone(), + follow_symlinks: false, + })) + .await?; + ensure( + link.metadata.kind == FileKind::Symlink, + "an escaping link is still describable as a symlink", + )?; + let outside_root = |result: Result<_, ClientError>, what: &str| match result { + Err(error) + if matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::PermissionDenied { + reason: Some(PermissionReason::OutsideRoot) + }) + ) => + { + Ok(()) + } + Err(error) => Err(format!("{what} through an escaping link answered {error}")), + Ok(_) => Err(format!("{what} followed a link that leaves the root")), + }; + outside_root( + self.client + .read_file(&ReadFileParams { + path: outside.clone(), + offset: 0, + max_bytes: 16, + }) + .await + .map(drop), + "fs/readFile", + )?; + outside_root( + self.client + .get_metadata(&GetMetadataParams { + path: outside, + follow_symlinks: true, + }) + .await + .map(drop), + "fs/getMetadata {followSymlinks}", + ) + } + + fn process_id(&self) -> ProcessId { + ProcessId::new(format!("conformance-{}", uuid::Uuid::new_v4().simple())) + .expect("generated ids fit the grammar") + } + + async fn start(&self, script: &str, input: ExecInput) -> Result { + let process_id = self.process_id(); + call(self.client.start_process(&StartProcessParams { + process_id: process_id.clone(), + command: shell(script, input, self.work), + pty: None, + })) + .await?; + Ok(process_id) + } + + /// Read from the cursor until the exit has settled and all output is in. + async fn read_to_exit( + &self, + process_id: &ProcessId, + ) -> Result<(Vec, ReadProcessResponse), String> { + let deadline = Instant::now() + self.timeout; + let mut output = Vec::new(); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err("the command did not exit in time".into()); + } + let response = call(self.client.read_process(&ReadProcessParams { + process_id: process_id.clone(), + after: None, + tail_bytes: None, + ack: None, + max_bytes: self.initialized.limits.process.max_read_bytes, + wait_ms: (remaining.as_millis() as u64).min(self.initialized.limits.max_wait_ms), + return_on: ReturnOn::Exit, + })) + .await?; + ensure( + response.output.gaps.is_empty(), + "output gaps in a short command", + )?; + for chunk in &response.output.chunks { + output.extend_from_slice(chunk.data.as_slice()); + } + if response.state == ExecState::Exited && response.output.next == response.output.end { + return Ok((output, response)); + } + } + } + + async fn process_exit(&self) -> Check { + let process_id = self + .start("printf out; printf err >&2; exit 3", ExecInput::None {}) + .await?; + let (output, response) = self.read_to_exit(&process_id).await?; + ensure( + response.exit == Some(ExitStatus::Code { code: 3 }), + format!("exit was {:?}, not code 3", response.exit), + )?; + let text = String::from_utf8_lossy(&output); + ensure( + text.contains("out") && text.contains("err"), + "stdout and stderr both reach the output", + ) + } + + async fn process_signal(&self) -> Check { + let process_id = self.start("kill -TERM $$", ExecInput::None {}).await?; + let (_, response) = self.read_to_exit(&process_id).await?; + match &response.exit { + Some(ExitStatus::Signal { signal, .. }) if signal.as_str() == "SIGTERM" => Ok(()), + other => Err(format!("a signal death was reported as {other:?}")), + } + } + + async fn process_large_input(&self) -> Check { + let input = vec![b'x'; 1 << 20]; + let process_id = self + .start( + "cat", + ExecInput::Bytes { + data: ByteChunk::from(input.clone()), + }, + ) + .await?; + let (output, response) = self.read_to_exit(&process_id).await?; + ensure( + response.output.end == input.len() as u64 && output == input, + format!( + "cat echoed {} of {} input bytes", + response.output.end, + input.len() + ), + )?; + let tail = self + .client + .read_process(&ReadProcessParams { + process_id, + after: None, + tail_bytes: Some(10), + ack: None, + max_bytes: 10, + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + }) + .await; + match tail { + // A fully read process may already be released. + Err(error) if error.protocol_kind() == Some(ProtocolErrorKind::NotFound) => Ok(()), + Err(error) => Err(error.to_string()), + Ok(tail) => ensure( + tail.output.next == tail.output.end + && tail + .output + .chunks + .first() + .is_none_or(|chunk| chunk.offset >= tail.output.end - 10), + "a tail read starts at end - tailBytes", + ), + } + } + + async fn process_duplicate_id(&self) -> Check { + let marker = self.path("duplicate-marker")?; + let script = format!("echo ran >> '{marker}'"); + let process_id = self.start("sleep 30", ExecInput::None {}).await?; + let duplicate = self + .client + .start_process(&StartProcessParams { + process_id: process_id.clone(), + command: shell(&script, ExecInput::None {}, self.work), + pty: None, + }) + .await; + let _ = self + .client + .terminate_process(&TerminateProcessParams { + process_id, + signal: ProcessSignal::Kill, + }) + .await; + expect_kind(duplicate, ProtocolErrorKind::AlreadyExists)?; + expect_kind( + self.client + .read_file(&ReadFileParams { + path: marker, + offset: 0, + max_bytes: 16, + }) + .await, + ProtocolErrorKind::NotFound, + ) + .map_err(|_| "a refused duplicate start ran its command".to_owned()) + } + + async fn process_unknown(&self) -> Check { + let unknown = self.process_id(); + expect_kind( + self.client + .read_process(&ReadProcessParams { + process_id: unknown.clone(), + after: None, + tail_bytes: None, + ack: None, + max_bytes: 1, + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + }) + .await, + ProtocolErrorKind::NotFound, + )?; + expect_kind( + self.client + .write_process(&WriteProcessParams { + process_id: unknown.clone(), + data: ByteChunk::from(&b"x"[..]), + close_input: false, + }) + .await, + ProtocolErrorKind::NotFound, + )?; + expect_kind( + self.client + .terminate_process(&TerminateProcessParams { + process_id: unknown, + signal: ProcessSignal::Kill, + }) + .await, + ProtocolErrorKind::NotFound, + ) } - Ok(()) + async fn process_terminate(&self) -> Check { + let process_id = self.start("sleep 60", ExecInput::None {}).await?; + let response = call(self.client.terminate_process(&TerminateProcessParams { + process_id: process_id.clone(), + signal: ProcessSignal::Terminate, + })) + .await?; + ensure( + response.running, + "terminate of a running process reported it not running", + )?; + let (_, exited) = self.read_to_exit(&process_id).await?; + ensure( + exited.stopped_by == Some(StoppedBy::Terminate), + format!("stoppedBy was {:?}", exited.stopped_by), + ) + } + + fn job(&self, job_id: &JobId, script: &str) -> JobSpec { + JobSpec { + job_id: job_id.clone(), + name: None, + command: shell(script, ExecInput::None {}, self.work), + depends_on: Vec::new(), + dependency_policy: Default::default(), + queue_key: None, + } + } + + fn job_ids(&self) -> (JobId, JobRequestId, JobRequestId) { + let suffix = uuid::Uuid::new_v4().simple(); + ( + JobId::new(format!("conformance-{suffix}")).expect("job id"), + JobRequestId::new(format!("request-a-{suffix}")).expect("request id"), + JobRequestId::new(format!("request-b-{suffix}")).expect("request id"), + ) + } + + async fn job_request_id(&self) -> Check { + let (job_id, first, second) = self.job_ids(); + let start = |request_id: &JobRequestId| StartJobsParams { + request_id: request_id.clone(), + jobs: vec![self.job(&job_id, "true")], + }; + let started = call(self.client.start_jobs(&start(&first))).await?; + let replayed = call(self.client.start_jobs(&start(&first))).await?; + ensure( + started.jobs[0].created_at_ms == replayed.jobs[0].created_at_ms, + "a retried start with the same requestId created another run", + )?; + expect_precondition( + self.client.start_jobs(&start(&second)).await, + PreconditionReason::JobIdInUse, + ) + } + + async fn job_unknown(&self) -> Check { + let (job_id, _, _) = self.job_ids(); + let response = call(self.client.read_jobs(&ReadJobsParams { + jobs: vec![JobReadRequest { + job_id, + after: None, + tail_bytes: None, + max_bytes: None, + }], + max_bytes: 0, + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + })) + .await?; + match response.jobs.as_slice() { + [JobReadEntry::Error { error, .. }] if error.kind() == ProtocolErrorKind::NotFound => { + Ok(()) + } + other => Err(format!("an unknown job read answered {other:?}")), + } + } + + async fn job_paging(&self) -> Check { + let (job_id, request_id, _) = self.job_ids(); + // 20 000 bytes: 2 000 numbered lines of 10 bytes each. + let script = "i=0; while [ $i -lt 2000 ]; do printf '%09d\\n' $i; i=$((i+1)); done"; + call(self.client.start_jobs(&StartJobsParams { + request_id, + jobs: vec![self.job(&job_id, script)], + })) + .await?; + let deadline = Instant::now() + self.timeout; + let mut output = Vec::new(); + let mut after = 0; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err("the job did not finish in time".into()); + } + let response = call(self.client.read_jobs(&ReadJobsParams { + jobs: vec![JobReadRequest { + job_id: job_id.clone(), + after: Some(after), + tail_bytes: None, + max_bytes: None, + }], + max_bytes: 4096, + wait_ms: (remaining.as_millis() as u64).min(1000), + return_on: ReturnOn::Window, + })) + .await?; + let [ + JobReadEntry::Job { + summary, + output: page, + }, + ] = response.jobs.as_slice() + else { + return Err(format!("job read answered {:?}", response.jobs)); + }; + ensure(page.gaps.is_empty(), "a 20 000-byte job reported gaps")?; + for chunk in &page.chunks { + ensure( + chunk.offset == output.len() as u64, + "pages skipped or repeated bytes", + )?; + output.extend_from_slice(chunk.data.as_slice()); + } + after = page.next; + if summary.status.is_terminal() && page.next == page.end { + break; + } + } + let expected = (0..2000).map(|i| format!("{i:09}\n")).collect::(); + ensure( + output == expected.as_bytes(), + format!( + "paged output was {} bytes, not 20 000 exact bytes", + output.len() + ), + ) + } +} + +fn shell(script: &str, input: ExecInput, cwd: &EnvironmentPath) -> CommandSpec { + CommandSpec { + argv: vec!["/bin/sh".into(), "-c".into(), script.into()], + cwd: Some(cwd.clone()), + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input, + timeout_ms: Some(120_000), + } +} + +fn join(base: &EnvironmentPath, name: &str) -> Result { + let tree = TreePath::new(name).map_err(|error| error.to_string())?; + base.join(&tree).map_err(|error| error.to_string()) +} + +fn ensure(condition: bool, message: impl Into) -> Check { + if condition { + Ok(()) + } else { + Err(message.into()) + } +} + +async fn call(future: impl Future>) -> Result { + future.await.map_err(|error| error.to_string()) } -fn check(message: impl Into) -> EnvironmentDataConformanceError { - EnvironmentDataConformanceError::Check(message.into()) +fn expect_kind(result: Result, kind: ProtocolErrorKind) -> Check { + match result { + Err(error) if error.protocol_kind() == Some(kind) => Ok(()), + other => Err(format!("expected {kind}, got {other:?}")), + } +} + +fn expect_precondition( + result: Result, + reason: PreconditionReason, +) -> Check { + match &result { + Err(error) + if matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::FailedPrecondition { reason: actual }) if *actual == reason + ) => + { + Ok(()) + } + other => Err(format!( + "expected failedPrecondition ({reason:?}), got {other:?}" + )), + } } diff --git a/crates/tools/src/environment_protocol/mod.rs b/crates/tools/src/environment_protocol/mod.rs index 72eda9b9a..127ef5a0b 100644 --- a/crates/tools/src/environment_protocol/mod.rs +++ b/crates/tools/src/environment_protocol/mod.rs @@ -3,8 +3,8 @@ pub mod conformance; pub mod remote; -pub use conformance::{ - EnvironmentDataConformanceError, EnvironmentDataConformanceOptions, - assert_environment_data_conformance, +pub use conformance::{DataConformance, DataConformanceFailure, DataConformanceReport}; +pub use remote::{ + RemoteEnvironmentConnection, RemoteEnvironmentFileSystem, RemoteJobExecutor, + RemoteProcessExecutor, }; -pub use remote::{RemoteEnvironmentConnection, RemoteEnvironmentFileSystem, RemoteProcessExecutor}; diff --git a/crates/tools/src/environment_protocol/remote.rs b/crates/tools/src/environment_protocol/remote.rs index 9ac7a60f9..85872ffb7 100644 --- a/crates/tools/src/environment_protocol/remote.rs +++ b/crates/tools/src/environment_protocol/remote.rs @@ -1,33 +1,59 @@ //! Remote environment adapters backed by `environment-client`. +//! +//! One [`DataClient`] serves every adapter of a connection: calls are +//! multiplexed, so a long process read never blocks a file read. The +//! daemon's `initialize` response decides which adapters exist (capability +//! families) and bounds every request (advertised ceilings). use std::{ sync::{ Arc, LazyLock, atomic::{AtomicU64, Ordering}, }, - time::{SystemTime, UNIX_EPOCH}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use async_trait::async_trait; -use environment_client::{EnvironmentClientError, EnvironmentDataClient, JsonRpcTransport}; +use environment_client::{ClientError, DataClient}; use environment_protocol::{ + bytes::ByteChunk, data::{ - fs as remote_fs, jobs as remote_jobs, + exec::{ + CommandSpec, ExecInput, ExecState, ExitStatus, OutputPage, PtySize, ReturnOn, StoppedBy, + }, + fs::{ + CreateDirectoryParams, FileKind, GetMetadataParams, GlobFilesLimits, GlobFilesParams, + GlobFilesStop, ReadDirectoryParams, ReadFileParams, RemoveParams, RenameParams, + SearchTextLimits, SearchTextParams, SearchTextStop, TreeFilter, WriteCondition, + WriteFileParams, + }, + handshake::{FilesystemLimits, InitializeResponse, JobLimits, TransferCeilings}, + job::{ + CancelJobsParams, CancelJobsResponse, ReadJobsParams, ReadJobsResponse, + StartJobsParams, StartJobsResponse, + }, process::{ - self as remote_process, ProcessOutputStream, ProcessTermination, ReadProcessResponse, - WriteProcessStatus, + ReadProcessParams, ReadProcessResponse, StartProcessParams, TerminateProcessParams, + WriteProcessParams, + }, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, ListTransferMissingParams, + ListTransferMissingResponse, ReadTransferContentParams, ReadTransferContentResponse, + ReadTransferInventoryParams, ReadTransferInventoryResponse, TransferOperationParams, + TransferStatus, WriteTransferContentParams, WriteTransferContentResponse, }, }, - error::EnvironmentProtocolErrorCode, - shared::{ByteChunk, EnvironmentCapabilities, EnvironmentPath, ProcessId}, + error::{Capability, ErrorData, PreconditionReason, ProtocolErrorKind}, + ids::ProcessId, + paths::EnvironmentPath, }; use harness::storage::BlobStore; -use tokio::sync::Mutex as AsyncMutex; use crate::{ environment::{ EnvironmentToolContext, jobs::{JobError, JobExecResult, JobExecutor}, + output::{ShownOutput, is_stderr}, process::{ ContinueProcessRequest, LeftoverProcess, ProcessError, ProcessExecResult, ProcessExecutor, ProcessHandle, ProcessOutput, ProcessRequest, ProcessSignal, @@ -35,13 +61,22 @@ use crate::{ }, }, fs::{ - CopyOptions, CreateDirectoryOptions, FileAccessPolicy, FileMetadata, FileSystem, FsError, - FsGlobRequest, FsGlobResponse, FsPath, FsRangedRead, FsResult, FsSearchStop, - FsTextSearchMatch, FsTextSearchRequest, FsTextSearchResponse, FsToolContext, - ReadDirectoryEntry, RemoveOptions, ranged_read_from_full, + CreateDirectoryOptions, FileAccessPolicy, FileMetadata, FileSystem, FsError, FsGlobRequest, + FsGlobResponse, FsPath, FsRangedRead, FsResult, FsSearchStop, FsTextSearchMatch, + FsTextSearchRequest, FsTextSearchResponse, FsToolContext, FsVersion, ReadDirectoryEntry, + RemoveOptions, RenameOptions, VersionedRead, WriteOptions, }, + transfer::EnvironmentTransfer, }; +/// Output a process read shows when the caller names no budget. +const DEFAULT_PROCESS_OUTPUT_BYTES: u64 = 512 * 1024; +/// Terminal size of a PTY the model asks for with `tty: true`. +const DEFAULT_PTY_SIZE: PtySize = PtySize { rows: 24, cols: 80 }; +/// How long input that the process does not read yet is retried before the +/// write gives up. +const INPUT_RETRY_WINDOW: Duration = Duration::from_secs(10); + static NEXT_PROCESS_EXECUTOR_ID: AtomicU64 = AtomicU64::new(1); static PROCESS_ID_NONCE: LazyLock = LazyLock::new(|| { let started_at_nanos = SystemTime::now() @@ -51,103 +86,73 @@ static PROCESS_ID_NONCE: LazyLock = LazyLock::new(|| { format!("{}-{started_at_nanos}", std::process::id()) }); -pub struct RemoteEnvironmentConnection { - client: Arc>>, - capabilities: EnvironmentCapabilities, +/// A data-plane connection and the tool adapters it supports. +#[derive(Clone)] +pub struct RemoteEnvironmentConnection { + client: DataClient, cwd: Option, } -impl Clone for RemoteEnvironmentConnection { - fn clone(&self) -> Self { - Self { - client: self.client.clone(), - capabilities: self.capabilities.clone(), - cwd: self.cwd.clone(), - } +impl RemoteEnvironmentConnection { + /// Adapters over an initialized client; capabilities and ceilings come + /// from its `initialize` response. + pub fn new(client: DataClient) -> Self { + Self { client, cwd: None } } -} -#[async_trait] -impl crate::transfer::EnvironmentTransfer - for RemoteEnvironmentConnection -{ - async fn request( - &self, - request: environment_protocol::data::transfer_session::TransferRequest, - ) -> crate::error::ToolResult - { - if !self.capabilities.filesystem_transfer { - return Err(crate::error::ToolError::UnsupportedCapability { - message: "environment transfer unavailable".into(), - }); - } - self.client - .lock() - .await - .transfer(&request) - .await - .map_err(|e| crate::error::ToolError::InvalidRequest { - message: e.to_string(), - }) + pub fn with_cwd(mut self, cwd: FsPath) -> Self { + self.cwd = Some(cwd); + self } -} -impl RemoteEnvironmentConnection -where - T: JsonRpcTransport + Send + 'static, -{ - pub fn new(client: EnvironmentDataClient, capabilities: EnvironmentCapabilities) -> Self { - Self { - client: Arc::new(AsyncMutex::new(client)), - capabilities, - cwd: None, - } + pub fn client(&self) -> &DataClient { + &self.client } - pub fn with_cwd(mut self, cwd: FsPath) -> Self { - self.cwd = Some(cwd); - self + pub fn initialize_response(&self) -> &InitializeResponse { + self.client.initialize_response() } - /// Gracefully shut down the shared data-plane transport after all tool - /// adapters derived from this connection have finished using it. - pub async fn close(&self) -> Result<(), EnvironmentClientError> { - self.client.lock().await.close().await + /// Close the shared connection once every adapter derived from it has + /// finished. + pub async fn close(&self) -> Result<(), ClientError> { + self.client.close().await } - pub fn file_system(&self) -> RemoteEnvironmentFileSystem { + pub fn file_system(&self) -> RemoteEnvironmentFileSystem { + let initialized = self.initialize_response(); RemoteEnvironmentFileSystem { client: self.client.clone(), - access_policy: access_policy_for_capabilities(&self.capabilities), - filesystem_search: self.capabilities.filesystem_search, - filesystem_glob: self.capabilities.filesystem_glob, - filesystem_ranged_read: self.capabilities.filesystem_ranged_read, + writable: initialized + .capabilities + .filesystem + .as_ref() + .is_some_and(|filesystem| filesystem.write), + limits: initialized.limits.filesystem.clone(), } } - pub fn process_executor(&self) -> Option> { - if !self.capabilities.process_start { - return None; - } - let process_id_prefix = NEXT_PROCESS_EXECUTOR_ID.fetch_add(1, Ordering::Relaxed); + /// Present when the daemon serves the `process` family. + pub fn process_executor(&self) -> Option { + let initialized = self.initialize_response(); + let process = initialized.capabilities.process.as_ref()?; Some(RemoteProcessExecutor { client: self.client.clone(), - process_id_prefix, + pty: process.pty, + max_read_bytes: initialized.limits.process.max_read_bytes, + max_wait_ms: initialized.limits.max_wait_ms, + process_id_prefix: NEXT_PROCESS_EXECUTOR_ID.fetch_add(1, Ordering::Relaxed), next_process_id: Arc::new(AtomicU64::new(1)), }) } - pub fn job_executor(&self) -> Option> { - if !self.capabilities.job_start - && !self.capabilities.job_list - && !self.capabilities.job_read - && !self.capabilities.job_cancel - { - return None; - } + /// Present when the daemon serves the `job` family. + pub fn job_executor(&self) -> Option { + let initialized = self.initialize_response(); + initialized.capabilities.job.as_ref()?; Some(RemoteJobExecutor { client: self.client.clone(), - capabilities: self.capabilities.clone(), + limits: initialized.limits.job.clone(), }) } @@ -155,6 +160,7 @@ where self, blobs: Arc, ) -> (FsToolContext, EnvironmentToolContext) { + let has_filesystem = self.initialize_response().capabilities.filesystem.is_some(); let fs = Arc::new(self.file_system()); let process = self .process_executor() @@ -164,8 +170,8 @@ where .map(|executor| Arc::new(executor) as Arc); let mut fs_ctx = FsToolContext::new(fs, blobs.clone()); let mut env_ctx = EnvironmentToolContext::new(process, blobs); - if self.capabilities.filesystem_transfer { - env_ctx.transfer = Some(Arc::new(self.clone())); + if has_filesystem { + env_ctx.transfer = Some(Arc::new(self.client.clone())); } if let Some(jobs) = jobs { env_ctx = env_ctx.with_jobs(jobs); @@ -174,46 +180,166 @@ where fs_ctx = fs_ctx.with_cwd(cwd.clone()); env_ctx = env_ctx.with_process_cwd(cwd); } - // Filesystem tools are exposed only when the daemon negotiated read - // access; write access is enforced by the file system's access policy. - if self.capabilities.filesystem_read { + // Filesystem tools exist only when the daemon serves the filesystem + // family; write access follows its `write` fact through the file + // system's access policy. + if has_filesystem { env_ctx = env_ctx.with_filesystem(fs_ctx.clone()); } (fs_ctx, env_ctx) } } -#[derive(Clone)] -pub struct RemoteEnvironmentFileSystem { - client: Arc>>, - access_policy: FileAccessPolicy, - filesystem_search: bool, - filesystem_glob: bool, - filesystem_ranged_read: bool, -} - #[async_trait] -impl FileSystem for RemoteEnvironmentFileSystem -where - T: JsonRpcTransport + Send + 'static, -{ - fn access_policy(&self) -> FileAccessPolicy { - self.access_policy.clone() +impl EnvironmentTransfer for DataClient { + fn transfer_ceilings(&self) -> TransferCeilings { + self.initialize_response().limits.filesystem.transfer } - async fn read_file(&self, path: &FsPath) -> FsResult> { + async fn begin(&self, params: &BeginTransferParams) -> Result { + self.transfer_begin(params).await + } + + async fn advance( + &self, + params: &TransferOperationParams, + ) -> Result { + self.transfer_advance(params).await + } + + async fn get_status( + &self, + params: &TransferOperationParams, + ) -> Result { + self.transfer_get_status(params).await + } + + async fn read_inventory( + &self, + params: &ReadTransferInventoryParams, + ) -> Result { + self.transfer_read_inventory(params).await + } + + async fn append( + &self, + params: &AppendTransferInventoryParams, + ) -> Result { + self.transfer_append(params).await + } + + async fn list_missing( + &self, + params: &ListTransferMissingParams, + ) -> Result { + self.transfer_list_missing(params).await + } + + async fn read( + &self, + params: &ReadTransferContentParams, + ) -> Result { + self.transfer_read(params).await + } + + async fn write( + &self, + params: &WriteTransferContentParams, + ) -> Result { + self.transfer_write(params).await + } + + async fn commit( + &self, + params: &TransferOperationParams, + ) -> Result { + self.transfer_commit(params).await + } + + async fn abort(&self, params: &TransferOperationParams) -> Result { + self.transfer_abort(params).await + } +} + +/// The environment filesystem over `fs/*`. Every bound comes from the +/// daemon's advertised ceilings: a read longer than one `fs/readFile` is +/// refused rather than returned short, and writes are staged and atomic. +#[derive(Clone)] +pub struct RemoteEnvironmentFileSystem { + client: DataClient, + writable: bool, + limits: FilesystemLimits, +} + +impl RemoteEnvironmentFileSystem { + async fn read_whole(&self, path: &FsPath) -> FsResult { let response = self .client - .lock() - .await - .read_file(&remote_fs::ReadFileParams { + .read_file(&ReadFileParams { path: environment_path(path)?, - offset: None, - max_bytes: None, + offset: 0, + max_bytes: self.limits.max_read_file_bytes, }) .await .map_err(|error| map_fs_error(error, path))?; - Ok(response.data.into_inner()) + if response.truncated { + return Err(FsError::InvalidInput { + message: format!( + "{path} is {} bytes; the environment returns at most {} bytes per read", + response.file_size, self.limits.max_read_file_bytes + ), + }); + } + Ok(VersionedRead { + bytes: response.data.into_inner(), + version: Some(FsVersion::new(response.version.as_str())), + }) + } + + async fn metadata(&self, path: &FsPath, follow_symlinks: bool) -> FsResult { + let metadata = self + .client + .get_metadata(&GetMetadataParams { + path: environment_path(path)?, + follow_symlinks, + }) + .await + .map_err(|error| map_fs_error(error, path))? + .metadata; + Ok(FileKindFacts { + kind: metadata.kind, + created_at_ms: metadata.created_at_ms.unwrap_or(0), + modified_at_ms: metadata.modified_at_ms, + }) + } +} + +struct FileKindFacts { + kind: FileKind, + created_at_ms: i64, + modified_at_ms: i64, +} + +#[async_trait] +impl FileSystem for RemoteEnvironmentFileSystem { + fn access_policy(&self) -> FileAccessPolicy { + if self.writable { + FileAccessPolicy::FullReadWrite + } else { + FileAccessPolicy::FullReadOnly + } + } + + async fn read_file(&self, path: &FsPath) -> FsResult> { + Ok(self.read_whole(path).await?.bytes) + } + + async fn read_file_versioned(&self, path: &FsPath) -> FsResult { + self.read_whole(path).await + } + + fn max_read_bytes(&self) -> Option { + Some(self.limits.max_read_file_bytes) } async fn read_file_range( @@ -222,39 +348,60 @@ where offset: u64, max_bytes: Option, ) -> FsResult { - if !self.filesystem_ranged_read { - let bytes = self.read_file(path).await?; - return Ok(ranged_read_from_full(bytes, offset, max_bytes)); - } + let ceiling = self.limits.max_read_file_bytes; let response = self .client - .lock() - .await - .read_file(&remote_fs::ReadFileParams { + .read_file(&ReadFileParams { path: environment_path(path)?, - offset: Some(offset), - max_bytes, + offset, + max_bytes: max_bytes.map_or(ceiling, |max| max.min(ceiling)), }) .await .map_err(|error| map_fs_error(error, path))?; - let bytes = response.data.into_inner(); - let file_size = response - .file_size - .unwrap_or_else(|| offset.saturating_add(bytes.len() as u64)); Ok(FsRangedRead { - file_size, + file_size: response.file_size, truncated: response.truncated, - bytes, + bytes: response.data.into_inner(), }) } async fn write_file(&self, path: &FsPath, contents: Vec) -> FsResult<()> { - self.client - .lock() + self.write_file_with(path, contents, WriteOptions::default()) .await - .write_file(&remote_fs::WriteFileParams { + } + + async fn write_file_with( + &self, + path: &FsPath, + contents: Vec, + options: WriteOptions, + ) -> FsResult<()> { + if contents.len() as u64 > self.limits.max_write_file_bytes { + return Err(FsError::InvalidInput { + message: format!( + "{} bytes exceed the environment's {}-byte inline write limit; write larger files from a content_ref", + contents.len(), + self.limits.max_write_file_bytes + ), + }); + } + let if_match = options + .if_match + .map(|version| { + environment_protocol::data::fs::FileVersion::new(version.as_str()).map_err( + |error| FsError::InvalidInput { + message: error.to_string(), + }, + ) + }) + .transpose()?; + self.client + .write_file(&WriteFileParams { path: environment_path(path)?, data: ByteChunk::from(contents), + create: WriteCondition::Always, + if_match, + executable: false, }) .await .map_err(|error| map_fs_error(error, path))?; @@ -267,11 +414,9 @@ where options: CreateDirectoryOptions, ) -> FsResult<()> { self.client - .lock() - .await - .create_directory(&remote_fs::CreateDirectoryParams { + .create_directory(&CreateDirectoryParams { path: environment_path(path)?, - recursive: Some(options.recursive), + recursive: options.recursive, }) .await .map_err(|error| map_fs_error(error, path))?; @@ -279,75 +424,81 @@ where } async fn get_metadata(&self, path: &FsPath) -> FsResult { - let response = self - .client - .lock() - .await - .get_metadata(&remote_fs::GetMetadataParams { - path: environment_path(path)?, - }) - .await - .map_err(|error| map_fs_error(error, path))?; + let own = self.metadata(path, false).await?; + let is_symlink = own.kind == FileKind::Symlink; + let target = if is_symlink { + self.metadata(path, true).await? + } else { + own + }; Ok(FileMetadata { - is_directory: response.is_directory, - is_file: response.is_file, - is_symlink: response.is_symlink, - created_at_ms: response.created_at_ms, - modified_at_ms: response.modified_at_ms, + is_directory: target.kind == FileKind::Directory, + is_file: target.kind == FileKind::File, + is_symlink, + created_at_ms: target.created_at_ms, + modified_at_ms: target.modified_at_ms, }) } async fn read_directory(&self, path: &FsPath) -> FsResult> { - let response = self - .client - .lock() - .await - .read_directory(&remote_fs::ReadDirectoryParams { - path: environment_path(path)?, - }) - .await - .map_err(|error| map_fs_error(error, path))?; - Ok(response - .entries - .into_iter() - .map(|entry| ReadDirectoryEntry { - file_name: entry.file_name, - is_directory: entry.is_directory, - is_file: entry.is_file, - }) - .collect()) + let directory = environment_path(path)?; + let mut entries = Vec::new(); + let mut cursor = None; + loop { + let page = self + .client + .read_directory(&ReadDirectoryParams { + path: directory.clone(), + limit: self.limits.max_directory_entries, + cursor, + }) + .await + .map_err(|error| map_fs_error(error, path))?; + entries.extend(page.entries.into_iter().map(|entry| ReadDirectoryEntry { + file_name: entry.name.as_str().to_owned(), + is_directory: entry.metadata.kind == FileKind::Directory, + is_file: entry.metadata.kind == FileKind::File, + })); + match page.next_cursor { + Some(next) => cursor = Some(next), + None => break, + } + } + Ok(entries) } async fn remove(&self, path: &FsPath, options: RemoveOptions) -> FsResult<()> { self.client - .lock() - .await - .remove(&remote_fs::RemoveParams { + .remove(&RemoveParams { path: environment_path(path)?, - recursive: Some(options.recursive), - force: Some(options.force), + recursive: options.recursive, + force: options.force, }) .await .map_err(|error| map_fs_error(error, path))?; Ok(()) } - async fn copy( + async fn rename( &self, source_path: &FsPath, destination_path: &FsPath, - options: CopyOptions, + options: RenameOptions, ) -> FsResult<()> { self.client - .lock() - .await - .copy(&remote_fs::CopyParams { - source_path: environment_path(source_path)?, - destination_path: environment_path(destination_path)?, - recursive: options.recursive, + .rename(&RenameParams { + source: environment_path(source_path)?, + destination: environment_path(destination_path)?, + overwrite: options.overwrite, + if_match: None, }) .await - .map_err(|error| map_fs_error(error, destination_path))?; + .map_err(|error| match error.protocol_kind() { + Some(ProtocolErrorKind::AlreadyExists) => FsError::AlreadyExists { + path: destination_path.clone(), + }, + _ => map_fs_error(error, source_path), + })?; Ok(()) } @@ -355,75 +506,92 @@ where &self, request: &FsTextSearchRequest, ) -> FsResult> { - if !self.filesystem_search { - return Ok(None); - } - let params = remote_fs::SearchTextParams { + let ceilings = self.limits.search; + let mut params = SearchTextParams { root: environment_path(&request.root)?, pattern: request.pattern.clone(), - include: request.include.clone(), - case_sensitive: request.case_sensitive, - max_depth: request - .max_depth - .map(|depth| u32::try_from(depth).unwrap_or(u32::MAX)), - limits: remote_fs::SearchTextLimits { - max_matches: request.limits.max_matches, - max_files: request.limits.max_files, - max_bytes: request.limits.max_bytes, - max_duration_ms: request.limits.max_duration_ms, + pattern_ignore_case: !request.case_sensitive, + filter: TreeFilter { + include: request.include.iter().cloned().collect(), + max_depth: request.max_depth.map(depth), + ..TreeFilter::default() + }, + limits: SearchTextLimits { + max_matches: count(request.limits.max_matches, ceilings.max_matches), + max_files: count(request.limits.max_files, ceilings.max_files), + max_bytes: request.limits.max_bytes.min(ceilings.max_bytes), + max_duration_ms: request.limits.max_duration_ms.min(ceilings.max_duration_ms), + max_line_bytes: ceilings.max_line_bytes, }, }; - let response = self - .client - .lock() - .await - .search_text(¶ms) - .await - .map_err(|error| map_fs_error(error, &request.root))?; + let response = match self.client.search_text(¶ms).await { + Ok(response) => response, + // `fs/searchText` walks directories; a file root searches that + // one file, as the generic fallback does: its parent, limited to + // the file's own name. + Err(error) if is_not_directory(&error) => { + let Some(file_root) = file_root_search(request)? else { + return Ok(Some(FsTextSearchResponse { + matches: Vec::new(), + files_searched: 0, + bytes_searched: 0, + skipped_files: 0, + stopped: None, + })); + }; + params.root = file_root.parent; + params.filter = file_root.filter; + self.client + .search_text(¶ms) + .await + .map_err(|error| map_fs_error(error, &request.root))? + } + Err(error) => return Err(map_fs_error(error, &request.root)), + }; let matches = response .matches .into_iter() .map(|item| { Ok(FsTextSearchMatch { path: FsPath::new(item.path.as_str()).map_err(FsError::from)?, - line_number: item.line_number, + line_number: u64::from(item.line_number), line: item.line, + line_truncated: item.line_truncated, }) }) .collect::>>()?; Ok(Some(FsTextSearchResponse { matches, - files_searched: response.files_searched, - bytes_searched: response.bytes_searched, + files_searched: u64::from(response.searched_file_count), + bytes_searched: response.searched_bytes, + skipped_files: u64::from(response.skipped_file_count), stopped: response.stopped.map(|stop| match stop { - remote_fs::SearchTextStop::MatchLimit => FsSearchStop::MatchLimit, - remote_fs::SearchTextStop::FileLimit => FsSearchStop::FileLimit, - remote_fs::SearchTextStop::ByteLimit => FsSearchStop::ByteLimit, - remote_fs::SearchTextStop::TimeLimit => FsSearchStop::TimeLimit, + SearchTextStop::MatchLimit => FsSearchStop::MatchLimit, + SearchTextStop::FileLimit => FsSearchStop::FileLimit, + SearchTextStop::ByteLimit => FsSearchStop::ByteLimit, + SearchTextStop::TimeLimit => FsSearchStop::TimeLimit, + SearchTextStop::ResponseLimit => FsSearchStop::ResponseLimit, }), })) } async fn glob_files(&self, request: &FsGlobRequest) -> FsResult> { - if !self.filesystem_glob { - return Ok(None); - } - let params = remote_fs::GlobFilesParams { + let ceilings = self.limits.glob; + let params = GlobFilesParams { root: environment_path(&request.root)?, - pattern: request.pattern.clone(), - max_depth: request - .max_depth - .map(|depth| u32::try_from(depth).unwrap_or(u32::MAX)), - limits: remote_fs::GlobFilesLimits { - max_matches: request.limits.max_matches, - max_entries: request.limits.max_entries, - max_duration_ms: request.limits.max_duration_ms, + filter: TreeFilter { + include: vec![request.pattern.clone()], + max_depth: request.max_depth.map(depth), + ..TreeFilter::default() + }, + limits: GlobFilesLimits { + max_matches: count(request.limits.max_matches, ceilings.max_matches), + max_files: count(request.limits.max_entries, ceilings.max_files), + max_duration_ms: request.limits.max_duration_ms.min(ceilings.max_duration_ms), }, }; let response = self .client - .lock() - .await .glob_files(¶ms) .await .map_err(|error| map_fs_error(error, &request.root))?; @@ -434,38 +602,96 @@ where .collect::>>()?; Ok(Some(FsGlobResponse { matches, - entries_visited: response.entries_visited, + entries_visited: u64::from(response.visited_file_count), stopped: response.stopped.map(|stop| match stop { - remote_fs::GlobFilesStop::MatchLimit => FsSearchStop::MatchLimit, - remote_fs::GlobFilesStop::EntryLimit => FsSearchStop::FileLimit, - remote_fs::GlobFilesStop::TimeLimit => FsSearchStop::TimeLimit, + GlobFilesStop::MatchLimit => FsSearchStop::MatchLimit, + GlobFilesStop::FileLimit => FsSearchStop::FileLimit, + GlobFilesStop::TimeLimit => FsSearchStop::TimeLimit, + GlobFilesStop::ResponseLimit => FsSearchStop::ResponseLimit, }), })) } } +fn is_not_directory(error: &ClientError) -> bool { + matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::FailedPrecondition { + reason: PreconditionReason::NotDirectory + }) + ) +} + +/// A text search rooted at a file, as a search of its parent. +struct FileRootSearch { + parent: EnvironmentPath, + filter: TreeFilter, +} + +/// The parent walk that searches only `request.root`, or `None` when the +/// request's include does not select the file (nothing to search). +fn file_root_search(request: &FsTextSearchRequest) -> FsResult> { + let invalid = || FsError::InvalidInput { + message: format!("{} is neither a directory nor a file", request.root), + }; + let name = request.root.file_name().ok_or_else(invalid)?; + let parent = request.root.parent().ok_or_else(invalid)?; + if let Some(include) = request.include.as_deref() { + let pattern = crate::fs::pattern::PathPattern::new(include, false).map_err(|error| { + FsError::InvalidInput { + message: error.to_string(), + } + })?; + if !pattern.matches(name, false) { + return Ok(None); + } + } + Ok(Some(FileRootSearch { + parent: environment_path(&parent)?, + filter: TreeFilter { + include: vec![format!("/{}", globset::escape(name))], + max_depth: Some(1), + include_hidden: true, + include_ignored: true, + ..TreeFilter::default() + }, + })) +} + +fn count(requested: u64, ceiling: u32) -> u32 { + u32::try_from(requested).unwrap_or(u32::MAX).min(ceiling) +} + +fn depth(depth: usize) -> u32 { + u32::try_from(depth).unwrap_or(u32::MAX) +} + /// Process executor over the data plane. It keeps no per-handle state: the /// daemon owns the read cursor, so a handle continued from a later batch or /// connection reads only what was not delivered yet. #[derive(Clone)] -pub struct RemoteProcessExecutor { - client: Arc>>, +pub struct RemoteProcessExecutor { + client: DataClient, + pty: bool, + max_read_bytes: u64, + max_wait_ms: u64, process_id_prefix: u64, next_process_id: Arc, } #[async_trait] -impl ProcessExecutor for RemoteProcessExecutor -where - T: JsonRpcTransport + Send + 'static, -{ +impl ProcessExecutor for RemoteProcessExecutor { async fn run_process(&self, request: ProcessRequest) -> ProcessExecResult { + if request.tty && !self.pty { + return Err(ProcessError::Unsupported { + message: "this environment cannot run commands under a PTY (tty: true)".to_owned(), + }); + } let process_id = self.next_process_id(); - let response = { - let mut client = self.client.lock().await; - client - .start_process(&remote_process::StartProcessParams { - process_id: process_id.clone(), + self.client + .start_process(&StartProcessParams { + process_id: process_id.clone(), + command: CommandSpec { argv: request.argv, cwd: request .cwd @@ -474,23 +700,26 @@ where .transpose()?, env: request.env, secret_env: request.secret_env, - stdin: request.stdin.map(ByteChunk::from), + input: match request.stdin { + Some(stdin) => ExecInput::Bytes { + data: ByteChunk::from(stdin), + }, + None => ExecInput::None {}, + }, timeout_ms: request.timeout_ms, - tty: request.tty, - }) - .await - .map_err(map_process_error)?; - client - .read_process(&remote_process::ReadProcessParams { - process_id: process_id.clone(), - after_seq: None, - max_bytes: request.max_output_bytes.map(|value| value as usize), - wait_ms: request.yield_ms, - }) - .await - .map_err(map_process_error)? - }; - Ok(output_from_read(process_id, response)) + }, + pty: request.tty.then_some(DEFAULT_PTY_SIZE), + }) + .await + .map_err(map_process_error)?; + let handle = ProcessHandle::new(process_id.as_str()); + self.read_output( + &handle, + &process_id, + request.yield_ms, + request.max_output_bytes, + ) + .await } async fn continue_process( @@ -498,64 +727,43 @@ where request: ContinueProcessRequest, ) -> ProcessExecResult { let handle = request.handle; - let process_id = ProcessId::new(handle.as_str().to_owned()); - let input = request.input.filter(|input| !input.is_empty()); - let response = { - let mut client = self.client.lock().await; - if let Some(signal) = request.signal { - client - .terminate_process(&remote_process::TerminateProcessParams { - process_id: process_id.clone(), - signal: match signal { - ProcessSignal::Interrupt => remote_process::ProcessSignal::Interrupt, - ProcessSignal::Kill => remote_process::ProcessSignal::Kill, - }, - }) - .await - .map_err(map_process_error)?; - } else if input.is_some() || request.close_stdin { - let write = client - .write_process(&remote_process::WriteProcessParams { - process_id: process_id.clone(), - chunk: input.map(ByteChunk::from), - close_stdin: request.close_stdin, - }) - .await - .map_err(map_process_error)?; - match write.status { - WriteProcessStatus::Accepted | WriteProcessStatus::Starting => {} - WriteProcessStatus::UnknownProcess => { - return Err(ProcessError::UnknownHandle { handle }); - } - WriteProcessStatus::StdinClosed => { - return Err(ProcessError::StdinClosed { handle }); - } - } - } - client - .read_process(&remote_process::ReadProcessParams { + let process_id = + ProcessId::new(handle.as_str()).map_err(|_| ProcessError::UnknownHandle { + handle: handle.clone(), + })?; + if let Some(signal) = request.signal { + self.client + .terminate_process(&TerminateProcessParams { process_id: process_id.clone(), - after_seq: None, - max_bytes: request.max_output_bytes.map(|value| value as usize), - wait_ms: request.wait_ms, + signal: match signal { + ProcessSignal::Interrupt => { + environment_protocol::data::process::ProcessSignal::Interrupt + } + ProcessSignal::Kill => { + environment_protocol::data::process::ProcessSignal::Kill + } + }, }) .await - .map_err(|error| match error { - EnvironmentClientError::Protocol(error) - if error.code == EnvironmentProtocolErrorCode::NotFound => - { - ProcessError::UnknownHandle { - handle: handle.clone(), - } - } - other => map_process_error(other), - })? - }; - Ok(output_from_read(process_id, response)) + .map_err(|error| map_handle_error(error, &handle))?; + } else { + let input = request.input.unwrap_or_default(); + if !input.is_empty() || request.close_stdin { + self.write_input(&handle, &process_id, input, request.close_stdin) + .await?; + } + } + self.read_output( + &handle, + &process_id, + request.wait_ms, + request.max_output_bytes, + ) + .await } } -impl RemoteProcessExecutor { +impl RemoteProcessExecutor { fn next_process_id(&self) -> ProcessId { let id = self.next_process_id.fetch_add(1, Ordering::Relaxed); ProcessId::new(format!( @@ -563,117 +771,311 @@ impl RemoteProcessExecutor { PROCESS_ID_NONCE.as_str(), self.process_id_prefix )) + .expect("generated process ids fit the id grammar") + } + + /// Queue `input` for the process, resending what a full input queue did + /// not accept, then close the input when asked. + async fn write_input( + &self, + handle: &ProcessHandle, + process_id: &ProcessId, + input: Vec, + close_input: bool, + ) -> ProcessExecResult<()> { + let started = Instant::now(); + let mut offset = 0usize; + loop { + let rest = &input[offset..]; + let accepted = self + .client + .write_process(&WriteProcessParams { + process_id: process_id.clone(), + data: ByteChunk::from(rest), + close_input, + }) + .await + .map_err(|error| map_handle_error(error, handle))? + .accepted_bytes; + offset += usize::try_from(accepted) + .unwrap_or(usize::MAX) + .min(rest.len()); + if offset >= input.len() { + return Ok(()); + } + if started.elapsed() >= INPUT_RETRY_WINDOW { + return Err(ProcessError::Failed { + message: format!( + "process {handle} accepted {offset} of {} input bytes; it is not reading its input", + input.len() + ), + }); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// Read until the process exits and its output settles, or until + /// `wait_ms` elapses. A finished command whose output exceeds the budget + /// shows its head and tail with the omitted count in between; a running + /// one keeps the rest behind the daemon cursor for the next call. + async fn read_output( + &self, + handle: &ProcessHandle, + process_id: &ProcessId, + wait_ms: Option, + max_output_bytes: Option, + ) -> ProcessExecResult { + let budget = max_output_bytes + .unwrap_or(DEFAULT_PROCESS_OUTPUT_BYTES) + .max(1); + let deadline = wait_ms.map(|wait| Instant::now() + Duration::from_millis(wait)); + let mut pages = Vec::new(); + let mut collected = 0u64; + let mut settled = false; + let last = loop { + // Once the exit has settled, further pages only drain output. + let wait = match deadline { + _ if settled => 0, + Some(deadline) => deadline + .saturating_duration_since(Instant::now()) + .as_millis() as u64, + None => self.max_wait_ms, + } + .min(self.max_wait_ms); + let response = self + .read_page( + handle, + ReadProcessParams { + process_id: process_id.clone(), + after: None, + tail_bytes: None, + ack: None, + max_bytes: (budget - collected).min(self.max_read_bytes), + wait_ms: wait, + return_on: ReturnOn::Exit, + }, + ) + .await?; + settled = response.state == ExecState::Exited; + collected += page_bytes(&response.output); + pages.push(response.output.clone()); + let more_output = response.output.next < response.output.end; + let out_of_time = deadline.is_some_and(|deadline| Instant::now() >= deadline); + match response.state { + ExecState::Exited if !more_output => break response, + ExecState::Exited if collected >= budget => { + let tail_bytes = budget / 2; + let tail = self + .read_page( + handle, + ReadProcessParams { + process_id: process_id.clone(), + after: None, + tail_bytes: Some(tail_bytes), + ack: None, + max_bytes: tail_bytes.min(self.max_read_bytes), + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + }, + ) + .await?; + pages.push(tail.output.clone()); + break tail; + } + // The page ended before the budget (its chunk limit): read on. + ExecState::Exited => continue, + ExecState::Running if collected >= budget || out_of_time => break response, + ExecState::Running => continue, + } + }; + Ok(output_from_read(handle, budget, pages, last)) + } + + async fn read_page( + &self, + handle: &ProcessHandle, + params: ReadProcessParams, + ) -> ProcessExecResult { + self.client + .read_process(¶ms) + .await + .map_err(|error| map_handle_error(error, handle)) } } -fn output_from_read(process_id: ProcessId, response: ReadProcessResponse) -> ProcessOutput { - let (stdout, stderr) = split_output(response.chunks, response.omitted_bytes); - let status = match response.termination { - Some(ProcessTermination::TimedOut) => ProcessStatus::TimedOut, - Some(ProcessTermination::Killed) => ProcessStatus::Killed, - None if response.failure.is_some() => ProcessStatus::Failed, - None if response.exited => match response.exit_code { - Some(0) => ProcessStatus::Succeeded, - _ => ProcessStatus::Failed, - }, - None => ProcessStatus::Running, - }; - let handle = if response.closed || response.exited { - None +fn page_bytes(page: &OutputPage) -> u64 { + page.chunks + .iter() + .map(|chunk| chunk.data.len() as u64) + .sum() +} + +/// Shape the pages of one call into the model-facing output. The last page +/// may be a tail read; earlier pages are consecutive cursor reads. +fn output_from_read( + handle: &ProcessHandle, + budget: u64, + mut pages: Vec, + last: ReadProcessResponse, +) -> ProcessOutput { + let exited = last.state == ExecState::Exited; + let tail = (pages.len() > 1 && exited && last.output.next == last.output.end) + .then(|| pages.pop()) + .flatten(); + let head = merge_pages(pages); + let shown = match tail { + Some(tail) if tail.end > head.next => ShownOutput::from_pages(head, Some(tail)), + Some(tail) => ShownOutput::from_pages(merge_pages(vec![head, tail]), None), + None => ShownOutput::from_pages(head, None), + } + .trim_to(budget); + let (stdout, stderr) = split_output(&shown); + let (status, exit_code, failure) = if !exited { + (ProcessStatus::Running, None, None) } else { - Some(ProcessHandle::new(process_id.0)) + match (&last.stopped_by, &last.exit, &last.error) { + (Some(StoppedBy::Timeout), exit, _) => (ProcessStatus::TimedOut, exit_code(exit), None), + (Some(StoppedBy::Kill | StoppedBy::Terminate), exit, _) => { + (ProcessStatus::Killed, exit_code(exit), None) + } + (None, _, Some(error)) => (ProcessStatus::Failed, None, Some(error.to_string())), + (None, Some(ExitStatus::Code { code: 0 }), None) => { + (ProcessStatus::Succeeded, Some(0), None) + } + (None, Some(ExitStatus::Code { code }), None) => { + (ProcessStatus::Failed, Some(*code), None) + } + ( + None, + Some(ExitStatus::Signal { + signal, + core_dumped, + }), + None, + ) => ( + ProcessStatus::Failed, + None, + Some(format!( + "killed by {signal}{}", + if *core_dumped { " (core dumped)" } else { "" } + )), + ), + (None, None, None) => ( + ProcessStatus::Failed, + None, + Some("the environment reported no exit status".to_owned()), + ), + } }; ProcessOutput { status, - handle, - pid: response.pid, - exit_code: response.exit_code, - failure: response.failure, + handle: (!exited).then(|| handle.clone()), + pid: Some(last.pid), + exit_code, + failure, stdout, stderr, - omitted_bytes: response.omitted_bytes, - leftover_processes: response - .leftover_processes - .into_iter() - .map(|member| LeftoverProcess { - pid: member.pid, - command: member.command, + omitted_bytes: shown.omitted_bytes(), + leftover_processes: last + .leftovers + .map(|report| { + report + .processes + .into_iter() + .map(|member| LeftoverProcess { + pid: member.pid, + command: member.command.unwrap_or_default(), + }) + .collect() }) - .collect(), + .unwrap_or_default(), + } +} + +fn exit_code(exit: &Option) -> Option { + match exit { + Some(ExitStatus::Code { code }) => Some(*code), + _ => None, + } +} + +/// Consecutive pages as one: chunks and gaps in order, the last page's +/// position. +fn merge_pages(pages: Vec) -> OutputPage { + let mut merged = OutputPage::default(); + for page in pages { + merged.chunks.extend(page.chunks); + merged.gaps.extend(page.gaps); + merged.next = merged.next.max(page.next); + merged.end = merged.end.max(page.end); + } + merged +} + +/// Sorts chunks into their streams. The omission marker goes where output +/// was left out: before the first chunk after the omitted range, or at the +/// end of standard output when nothing follows it. +fn split_output(shown: &ShownOutput) -> (StreamOutput, StreamOutput) { + let mut stdout = StreamOutput::default(); + let mut stderr = StreamOutput::default(); + for chunk in &shown.head { + let stream = if is_stderr(chunk.stream) { + &mut stderr + } else { + &mut stdout + }; + stream.bytes.extend_from_slice(chunk.data.as_slice()); } + if shown.omitted.is_some() { + let stream = match shown.tail.first() { + Some(chunk) if is_stderr(chunk.stream) => &mut stderr, + _ => &mut stdout, + }; + stream.omitted_at = Some(stream.bytes.len()); + } + for chunk in &shown.tail { + let stream = if is_stderr(chunk.stream) { + &mut stderr + } else { + &mut stdout + }; + stream.bytes.extend_from_slice(chunk.data.as_slice()); + } + (stdout, stderr) } #[derive(Clone)] -pub struct RemoteJobExecutor { - client: Arc>>, - capabilities: EnvironmentCapabilities, +pub struct RemoteJobExecutor { + client: DataClient, + limits: JobLimits, } #[async_trait] -impl JobExecutor for RemoteJobExecutor -where - T: JsonRpcTransport + Send + 'static, -{ - async fn start_jobs( - &self, - request: remote_jobs::StartJobsParams, - ) -> JobExecResult { - if !self.capabilities.job_start { - return Err(JobError::Unsupported { - message: "provider target does not support job/start".to_owned(), - }); - } +impl JobExecutor for RemoteJobExecutor { + fn limits(&self) -> Option { + Some(self.limits.clone()) + } + + async fn start_jobs(&self, request: StartJobsParams) -> JobExecResult { self.client - .lock() - .await .start_jobs(&request) .await .map_err(map_job_error) } - async fn read_jobs( - &self, - request: remote_jobs::ReadJobsParams, - ) -> JobExecResult { - if !self.capabilities.job_read { - return Err(JobError::Unsupported { - message: "provider target does not support job/read".to_owned(), - }); - } - self.client - .lock() - .await - .read_jobs(&request) - .await - .map_err(map_job_error) + async fn read_jobs(&self, request: ReadJobsParams) -> JobExecResult { + self.client.read_jobs(&request).await.map_err(map_job_error) } - async fn cancel_jobs( - &self, - request: remote_jobs::CancelJobsParams, - ) -> JobExecResult { - if !self.capabilities.job_cancel { - return Err(JobError::Unsupported { - message: "provider target does not support job/cancel".to_owned(), - }); - } + async fn cancel_jobs(&self, request: CancelJobsParams) -> JobExecResult { self.client - .lock() - .await .cancel_jobs(&request) .await .map_err(map_job_error) } } -fn access_policy_for_capabilities(capabilities: &EnvironmentCapabilities) -> FileAccessPolicy { - if capabilities.filesystem_write { - FileAccessPolicy::FullReadWrite - } else { - FileAccessPolicy::FullReadOnly - } -} - fn environment_path(path: &FsPath) -> Result { EnvironmentPath::new(path.as_str()).map_err(|error| FsError::InvalidInput { message: error.to_string(), @@ -686,279 +1088,317 @@ fn process_environment_path(path: &FsPath) -> Result FsError { - match error { - EnvironmentClientError::Protocol(error) => match error.code { - EnvironmentProtocolErrorCode::NotFound => FsError::NotFound { path: path.clone() }, - EnvironmentProtocolErrorCode::Conflict => FsError::AlreadyExists { path: path.clone() }, - EnvironmentProtocolErrorCode::Unauthorized - | EnvironmentProtocolErrorCode::Forbidden => { - FsError::PermissionDenied { path: path.clone() } - } - EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable => FsError::Unsupported { - message: error.message, - }, - EnvironmentProtocolErrorCode::InvalidRequest => FsError::InvalidInput { - message: error.message, - }, - _ => FsError::Failed { - message: error.message, - }, - }, - other => FsError::Failed { - message: other.to_string(), +fn map_fs_error(error: ClientError, path: &FsPath) -> FsError { + let Some(protocol) = error.protocol_error() else { + return FsError::Failed { + message: error.to_string(), + }; + }; + let message = protocol.message.clone(); + match &protocol.data { + ErrorData::NotFound { .. } => FsError::NotFound { path: path.clone() }, + ErrorData::AlreadyExists {} => FsError::AlreadyExists { path: path.clone() }, + ErrorData::PermissionDenied { .. } => FsError::PermissionDenied { path: path.clone() }, + ErrorData::FailedPrecondition { + reason: PreconditionReason::IfMatchFailed, + } => FsError::Conflict { path: path.clone() }, + ErrorData::CapabilityUnavailable { + capability: Capability::FilesystemWrite, + } => FsError::PermissionDenied { path: path.clone() }, + ErrorData::CapabilityUnavailable { .. } => FsError::Unsupported { message }, + ErrorData::FailedPrecondition { .. } + | ErrorData::InvalidParams { .. } + | ErrorData::InvalidRequest {} + | ErrorData::PayloadTooLarge { .. } => FsError::InvalidInput { message }, + _ => FsError::Failed { + message: protocol.to_string(), }, } } -fn map_process_error(error: EnvironmentClientError) -> ProcessError { - match error { - EnvironmentClientError::Protocol(error) => match error.code { - EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable => ProcessError::Unsupported { - message: error.message, - }, - EnvironmentProtocolErrorCode::InvalidRequest - | EnvironmentProtocolErrorCode::NotFound => ProcessError::InvalidRequest { - message: error.message, - }, - _ => ProcessError::Failed { - message: error.message, - }, +fn map_process_error(error: ClientError) -> ProcessError { + let Some(protocol) = error.protocol_error() else { + return ProcessError::Failed { + message: error.to_string(), + }; + }; + let message = protocol.message.clone(); + match protocol.kind() { + ProtocolErrorKind::CapabilityUnavailable => ProcessError::Unsupported { message }, + ProtocolErrorKind::InvalidParams + | ProtocolErrorKind::InvalidRequest + | ProtocolErrorKind::NotFound + | ProtocolErrorKind::PermissionDenied + | ProtocolErrorKind::FailedPrecondition => ProcessError::InvalidRequest { + message: protocol.to_string(), }, - other => ProcessError::Failed { - message: other.to_string(), + _ => ProcessError::Failed { + message: protocol.to_string(), }, } } -fn map_job_error(error: EnvironmentClientError) -> JobError { - match error { - EnvironmentClientError::Protocol(error) => match error.code { - EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable => JobError::Unsupported { - message: error.message, - }, - EnvironmentProtocolErrorCode::InvalidRequest - | EnvironmentProtocolErrorCode::NotFound - | EnvironmentProtocolErrorCode::Conflict => JobError::InvalidRequest { - message: error.message, - }, - _ => JobError::Failed { - message: error.message, - }, +/// Errors of calls on an existing handle: an unknown, expired or earlier +/// daemon's handle is `notFound`, closed input `inputClosed`. +fn map_handle_error(error: ClientError, handle: &ProcessHandle) -> ProcessError { + match error.protocol_error().map(|protocol| &protocol.data) { + Some(ErrorData::NotFound { .. }) => ProcessError::UnknownHandle { + handle: handle.clone(), }, - other => JobError::Failed { - message: other.to_string(), + Some(ErrorData::FailedPrecondition { + reason: PreconditionReason::InputClosed, + }) => ProcessError::StdinClosed { + handle: handle.clone(), }, + _ => map_process_error(error), } } -/// Sorts chunks into their streams. The omission marker goes where the -/// daemon dropped the middle: at the first sequence gap, or at the start of -/// the output when the gap precedes everything returned. -fn split_output( - chunks: Vec, - omitted_bytes: u64, -) -> (StreamOutput, StreamOutput) { - let mut stdout = StreamOutput::default(); - let mut stderr = StreamOutput::default(); - let mut omission_placed = omitted_bytes == 0; - let gap_index = chunks - .windows(2) - .position(|pair| pair[1].seq != pair[0].seq + 1) - .map(|index| index + 1) - .unwrap_or(0); - for (index, chunk) in chunks.into_iter().enumerate() { - let stream = match chunk.stream { - ProcessOutputStream::Stdout | ProcessOutputStream::Pty => &mut stdout, - ProcessOutputStream::Stderr => &mut stderr, +fn map_job_error(error: ClientError) -> JobError { + let Some(protocol) = error.protocol_error() else { + return JobError::Failed { + message: error.to_string(), }; - if !omission_placed && index == gap_index { - stream.omitted_at = Some(stream.bytes.len()); - omission_placed = true; - } - stream.bytes.extend(chunk.chunk.into_inner()); - } - if !omission_placed { - stdout.omitted_at = Some(stdout.bytes.len()); + }; + let message = protocol.message.clone(); + match &protocol.data { + ErrorData::FailedPrecondition { + reason: PreconditionReason::JobIdInUse, + } => JobError::JobIdInUse { + message: format!("{message}; choose a new job_id for a new run"), + }, + ErrorData::CapabilityUnavailable { .. } => JobError::Unsupported { message }, + ErrorData::InvalidParams { .. } + | ErrorData::InvalidRequest {} + | ErrorData::FailedPrecondition { .. } + | ErrorData::NotFound { .. } => JobError::InvalidRequest { message }, + _ => JobError::Failed { + message: protocol.to_string(), + }, } - (stdout, stderr) } #[cfg(test)] mod tests { use std::{ collections::{BTreeMap, VecDeque}, - sync::{Arc, Mutex}, + sync::Mutex, }; - use environment_client::EnvironmentClientResult; + use environment_protocol::{ + data::exec::{OutputChunk, OutputStream}, + error::ProtocolError, + jsonrpc::{Request, Response}, + }; use harness::storage::InMemoryBlobStore; use serde_json::{Value, json}; use super::*; use crate::fs::tools::{ReadFileArgs, WriteFileArgs, invoke_read_file, invoke_write_file}; - struct MockTransport { - sent: Arc>>, - recv: Arc>>, + /// Answers each request with the next scripted result and records it. + struct Script { + replies: Mutex>>, + sent: Mutex>, } - impl MockTransport { - fn new(messages: impl IntoIterator) -> (Self, Arc>>) { - let sent = Arc::new(Mutex::new(Vec::new())); - ( - Self { - sent: sent.clone(), - recv: Arc::new(Mutex::new(messages.into_iter().collect())), - }, - sent, - ) - } + fn initialize(capabilities: Value) -> InitializeResponse { + let fixture: Value = serde_json::from_str(include_str!( + "../../../environment-protocol/fixtures/data/initialize.json" + )) + .unwrap(); + let mut response = fixture["response"].clone(); + response["capabilities"] = capabilities; + response["defaultCwd"] = json!("/workspace"); + serde_json::from_value(response).unwrap() } - #[async_trait] - impl JsonRpcTransport for MockTransport { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()> { - self.sent.lock().expect("sent lock").push(message); - Ok(()) - } + fn scripted( + capabilities: Value, + replies: impl IntoIterator>, + ) -> (DataClient, Arc