diff --git a/Cargo.lock b/Cargo.lock index 074ab70de..f73f85622 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1304,7 +1304,7 @@ dependencies = [ name = "environment-client" version = "0.1.0" dependencies = [ - "async-trait", + "environment-client", "environment-protocol", "futures-util", "serde", @@ -1319,8 +1319,6 @@ name = "environment-daemon" version = "0.4.0" dependencies = [ "anyhow", - "api", - "async-trait", "clap", "dotenvy", "ed25519-dalek", @@ -1328,7 +1326,7 @@ dependencies = [ "environment-protocol", "flate2", "futures-util", - "glob", + "globset", "grep-regex", "grep-searcher", "harness", @@ -1351,6 +1349,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tokio-tungstenite 0.30.0", + "tokio-util", "tools", "vfs", "webpki-roots 1.0.9", @@ -1361,9 +1360,12 @@ name = "environment-protocol" version = "0.1.0" dependencies = [ "base64 0.23.1", + "environment-protocol", "schemars", "serde", "serde_json", + "serde_path_to_error", + "sha2 0.11.0", "thiserror 2.0.21", ] @@ -1376,6 +1378,7 @@ dependencies = [ "axum", "clap", "dotenvy", + "environment-client", "environment-protocol", "futures-util", "release-info", @@ -1387,6 +1390,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tokio-tungstenite 0.30.0", + "tools", "uuid", ] @@ -1854,12 +1858,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - [[package]] name = "globset" version = "0.4.20" @@ -5281,7 +5279,6 @@ dependencies = [ "bots", "channels", "chrono", - "environment-protocol", "futures", "futures-util", "harness", @@ -5934,8 +5931,9 @@ dependencies = [ "environment-client", "environment-protocol", "futures-util", - "glob", + "globset", "harness", + "ignore", "jsonschema", "regex", "reqwest", diff --git a/clients/typescript/schema/api.schema.json b/clients/typescript/schema/api.schema.json index d2bce1b0a..2fa7322b8 100644 --- a/clients/typescript/schema/api.schema.json +++ b/clients/typescript/schema/api.schema.json @@ -139,7 +139,7 @@ }, { "const": "environment_not_ready", - "description": "The environment exists but is not reachable yet: it is still\nprovisioning/booting, or it was powered down and a wake (desired\npower `running`) has been initiated by this request. Distinct from\n`Rejected` so clients retry with backoff instead of failing —\npolling and automation callers lean on this for wake-on-use.", + "description": "The environment exists but is not reachable yet: it is still\nprovisioning, its provider is waking or booting it after a data\nroute dial, or its registered daemon is reconnecting. Distinct from\n`Rejected` so clients retry with backoff instead of failing —\npolling and automation callers lean on this for wake-on-use.", "type": "string" }, { @@ -1084,23 +1084,6 @@ ], "type": "object" }, - "AgentApiOutcomeOfDeploymentEnvironmentAdoptResponse": { - "properties": { - "notifications": { - "items": { - "$ref": "#/definitions/AgentNotification" - }, - "type": "array" - }, - "result": { - "$ref": "#/definitions/DeploymentEnvironmentAdoptResponse" - } - }, - "required": [ - "result" - ], - "type": "object" - }, "AgentApiOutcomeOfDeploymentEnvironmentProviderDeleteResponse": { "properties": { "notifications": { @@ -1390,23 +1373,6 @@ ], "type": "object" }, - "AgentApiOutcomeOfEnvironmentIdlePolicyPutResponse": { - "properties": { - "notifications": { - "items": { - "$ref": "#/definitions/AgentNotification" - }, - "type": "array" - }, - "result": { - "$ref": "#/definitions/EnvironmentIdlePolicyPutResponse" - } - }, - "required": [ - "result" - ], - "type": "object" - }, "AgentApiOutcomeOfEnvironmentIngressPutResponse": { "properties": { "notifications": { @@ -1628,6 +1594,40 @@ ], "type": "object" }, + "AgentApiOutcomeOfEnvironmentRegistrationKeyRotateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/definitions/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/definitions/EnvironmentRegistrationKeyRotateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, + "AgentApiOutcomeOfEnvironmentRegistrationKeyUpdateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/definitions/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/definitions/EnvironmentRegistrationKeyUpdateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, "AgentApiOutcomeOfEnvironmentTemplateListResponse": { "properties": { "notifications": { @@ -1662,6 +1662,23 @@ ], "type": "object" }, + "AgentApiOutcomeOfEnvironmentUpdateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/definitions/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/definitions/EnvironmentUpdateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, "AgentApiOutcomeOfInitializeResponse": { "properties": { "notifications": { @@ -8710,74 +8727,6 @@ ], "type": "object" }, - "DeploymentEnvironmentAdoptParams": { - "properties": { - "bindingId": { - "type": "string" - }, - "displayName": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "requestId": { - "description": "Stable caller-generated retry identity inside the destination universe.", - "type": "string" - }, - "sourceTarget": { - "description": "Provider-native source reference. The Incus provider accepts\n`/` or an instance name in the `default` project.", - "type": "string" - }, - "takeOwnership": { - "description": "Required acknowledgement that Lightspeed will move, reconfigure, and\ndelete the target as part of its ordinary managed lifecycle.", - "type": "boolean" - }, - "universeId": { - "type": "string" - } - }, - "required": [ - "universeId", - "requestId", - "bindingId", - "sourceTarget", - "takeOwnership" - ], - "type": "object" - }, - "DeploymentEnvironmentAdoptResponse": { - "properties": { - "environment": { - "$ref": "#/definitions/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, - "DeploymentEnvironmentProviderConnection": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/definitions/DeploymentEnvironmentProviderTransport" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "DeploymentEnvironmentProviderDeleteParams": { "properties": { "providerId": { @@ -8818,21 +8767,35 @@ "type": "object" }, "DeploymentEnvironmentProviderPutParams": { + "description": "Register or replace a provider. `credential` is the optional bearer\ntoken Lightspeed presents to the provider's controller and data routes:\nit is sealed on receipt and never returned by any method. `Debug` output\nredacts it; request logging must never echo these params.", "properties": { - "controllerConnection": { - "$ref": "#/definitions/DeploymentEnvironmentProviderConnection" + "clearCredential": { + "description": "Remove the stored credential. Excludes `credential`.", + "type": "boolean" }, - "displayName": { + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" + }, + "credential": { + "description": "A new bearer credential; omit to keep the stored one. A provider needs\nnone. Excludes `clearCredential`.", "type": [ "string", "null" ] }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "dataRouteBase": { + "description": "`ws://` or `wss://` base of the provider's data routes; omit to derive\nit from the controller endpoint. Required for a `unix://` controller,\nand must not be `ws://` for a `wss://` controller.", + "type": [ + "string", + "null" + ] + }, + "displayName": { + "type": [ + "string", + "null" + ] }, "providerId": { "type": "string" @@ -8840,7 +8803,7 @@ }, "required": [ "providerId", - "controllerConnection" + "controllerEndpoint" ], "type": "object" }, @@ -8877,94 +8840,39 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderTransport": { - "oneOf": [ - { - "properties": { - "type": { - "const": "webSocket", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "http", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "stdio", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "ssh", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "providerType": { - "type": "string" - }, - "type": { - "const": "provider", - "type": "string" - } - }, - "required": [ - "type", - "providerType" - ], - "type": "object" - } - ] - }, "DeploymentEnvironmentProviderView": { "properties": { - "controllerConnection": { - "$ref": "#/definitions/DeploymentEnvironmentProviderConnection" + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" }, "createdAtMs": { "format": "int64", "type": "integer" }, + "dataRouteBase": { + "description": "Operator-configured `ws://` or `wss://` base of the provider's data\nroutes; absent derives it from the controller endpoint.", + "type": [ + "string", + "null" + ] + }, "displayName": { "type": [ "string", "null" ] }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "effectiveDataRouteBase": { + "description": "The data route base Lightspeed dials: `dataRouteBase`, or the\ncontroller endpoint's origin followed by `/routes`. Absent only when it\ncannot be derived.", + "type": [ + "string", + "null" + ] + }, + "hasCredential": { + "description": "A bearer credential is stored and presented to the controller and\ndata routes; without one no `Authorization` header is sent.", + "type": "boolean" }, "providerId": { "type": "string" @@ -8976,7 +8884,8 @@ }, "required": [ "providerId", - "controllerConnection", + "controllerEndpoint", + "hasCredential", "createdAtMs", "updatedAtMs" ], @@ -8984,7 +8893,7 @@ }, "DeploymentProviderBindingDeleteParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" }, "universeId": { @@ -8993,7 +8902,7 @@ }, "required": [ "universeId", - "bindingId" + "providerId" ], "type": "object" }, @@ -9009,39 +8918,23 @@ "type": "object" }, "DeploymentProviderBindingPutParams": { + "description": "Create or replace a universe's binding to a deployment provider.", "properties": { - "bindingId": { - "type": "string" - }, - "expectedRevision": { - "format": "uint64", - "minimum": 0, - "type": [ - "integer", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "enabled": { + "description": "Admits creates, template listing and data routes for the universe.", + "type": "boolean" }, "providerId": { "type": "string" }, - "status": { - "$ref": "#/definitions/EnvironmentProviderBindingStatusView" - }, "universeId": { "type": "string" } }, "required": [ "universeId", - "bindingId", "providerId", - "status" + "enabled" ], "type": "object" }, @@ -9328,77 +9221,61 @@ ], "type": "object" }, - "EnvironmentCloseResponse": { - "properties": { - "environment": { - "$ref": "#/definitions/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, - "EnvironmentConnectionTransportView": { + "EnvironmentCloseReasonView": { + "description": "Why an environment is closing or closed.", "oneOf": [ { - "enum": [ - "webSocket", - "http", - "stdio", - "ssh" - ], + "const": "requested", + "description": "`environments/close`.", "type": "string" }, { - "additionalProperties": false, - "properties": { - "provider": { - "properties": { - "provider_type": { - "type": "string" - } - }, - "required": [ - "provider_type" - ], - "type": "object" - } - }, - "required": [ - "provider" - ], - "type": "object" + "const": "keyRevoked", + "description": "Its registration key was revoked with `closeEnvironments`.", + "type": "string" + }, + { + "const": "ephemeralGrace", + "description": "Its ephemeral daemon stayed away longer than the key's disconnect\ngrace.", + "type": "string" + }, + { + "const": "departed", + "description": "Its ephemeral daemon said goodbye.", + "type": "string" + }, + { + "const": "offlineTimeout", + "description": "Its persistent daemon stayed away longer than the key's\n`offlineCloseAfterMs`.", + "type": "string" + }, + { + "const": "providerEnded", + "description": "The provider reported the target closed or gone.", + "type": "string" } ] }, - "EnvironmentConnectionView": { + "EnvironmentCloseResponse": { "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/definitions/EnvironmentConnectionTransportView" + "environment": { + "$ref": "#/definitions/EnvironmentView" } }, "required": [ - "endpoint", - "transport" + "environment" ], "type": "object" }, "EnvironmentCreateParams": { "properties": { - "bindingId": { - "type": "string" - }, "displayName": { "type": [ "string", "null" ] }, - "idlePolicy": { + "idle": { "anyOf": [ { "$ref": "#/definitions/EnvironmentIdlePolicyView" @@ -9407,7 +9284,7 @@ "type": "null" } ], - "description": "Optional staged idle policy applied by the power reaper." + "description": "Idle policy passed to the provider in place of the template's `idle`.\nThe provider runs the idle clock; it cannot be changed after create." }, "metadata": { "additionalProperties": { @@ -9415,18 +9292,22 @@ }, "type": "object" }, + "providerId": { + "description": "The provider to create the environment with; the universe needs an\nenabled binding to it.", + "type": "string" + }, "requestId": { - "description": "Stable caller-generated retry identity, unique inside the universe.", + "description": "Stable caller-generated retry identity, unique inside the universe.\nRetrying with identical inputs returns the same environment.", "type": "string" }, "templateId": { - "description": "Immutable provider template-version identity.", + "description": "The provider template to create the environment from.", "type": "string" } }, "required": [ "requestId", - "bindingId", + "providerId", "templateId" ], "type": "object" @@ -9609,7 +9490,7 @@ "type": "string" }, "protocolVersion": { - "description": "Environment protocol version the gateway speaks. A daemon registers\nonly when it reports exactly this number.", + "description": "The envd protocol version (data plane and registration channel)\nthe gateway speaks. A daemon registers only when it reports exactly\nthis number.", "format": "uint32", "minimum": 0, "type": "integer" @@ -9633,22 +9514,44 @@ ], "type": "object" }, - "EnvironmentExternalCreateParams": { + "EnvironmentDaemonView": { + "description": "The envd build serving an environment, from its registration or\nLightspeed's open handshake.", "properties": { - "connection": { - "allOf": [ - { - "$ref": "#/definitions/EnvironmentConnectionView" - } - ], - "description": "Connection to an envd instance reachable from Lightspeed." + "gitSha": { + "type": [ + "string", + "null" + ] + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" }, + "version": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "protocolVersion" + ], + "type": "object" + }, + "EnvironmentExternalCreateParams": { + "properties": { "displayName": { "type": [ "string", "null" ] }, + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of an envd instance reachable from\nLightspeed.", + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" @@ -9656,12 +9559,13 @@ "type": "object" }, "requestId": { + "description": "Stable caller-generated retry identity, unique inside the universe.", "type": "string" } }, "required": [ "requestId", - "connection" + "endpoint" ], "type": "object" }, @@ -9677,55 +9581,23 @@ "type": "object" }, "EnvironmentIdentityModeView": { - "description": "What Lightspeed does with a registered environment while its daemon is\ndisconnected. Registration-key policy, copied onto each environment.", + "description": "What happens to a registered environment while its daemon is\ndisconnected. Policy of the registration key that admitted it.", "oneOf": [ { "const": "persistent", - "description": "Stays offline until explicitly closed.", + "description": "Stays open: tools wait up to two minutes for the daemon to reconnect.\nClosed explicitly, or after the key's `offlineCloseAfterMs`.", "type": "string" }, { "const": "ephemeral", - "description": "Closed once the daemon has been away longer than the key's\ndisconnect grace.", + "description": "Unusable while disconnected; closed once the daemon has been away\nlonger than the key's disconnect grace, or at once when it says\ngoodbye.", "type": "string" } ] }, - "EnvironmentIdlePolicyPutParams": { - "properties": { - "environmentId": { - "type": "string" - }, - "idlePolicy": { - "anyOf": [ - { - "$ref": "#/definitions/EnvironmentIdlePolicyView" - }, - { - "type": "null" - } - ], - "description": "The complete new policy; omit to clear it." - } - }, - "required": [ - "environmentId" - ], - "type": "object" - }, - "EnvironmentIdlePolicyPutResponse": { - "properties": { - "environment": { - "$ref": "#/definitions/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, "EnvironmentIdlePolicyView": { - "description": "Staged idle policy. Thresholds are milliseconds of daemon-reported idle\ntime and must be non-decreasing in the order pause, suspend, stop, close.\nStages whose power state the provider does not support are skipped.", + "additionalProperties": false, + "description": "Staged idle policy the provider applies. Thresholds are milliseconds of\nenvd-reported idle time (running processes, jobs and leftover process\ngroups hold it at zero) and must be non-decreasing in the order pause,\nsuspend, stop, close. Stages whose power state the template does not\nsupport are skipped.", "properties": { "closeAfterMs": { "format": "uint64", @@ -9762,55 +9634,10 @@ }, "type": "object" }, - "EnvironmentIncarnationView": { - "properties": { - "createdAtMs": { - "format": "int64", - "type": "integer" - }, - "incarnationId": { - "type": "string" - }, - "powerStates": { - "description": "Power states the provider reported for this target; empty until\nobserved or when the provider offers no power control.", - "items": { - "$ref": "#/definitions/EnvironmentPowerStateView" - }, - "type": "array" - }, - "providerTargetId": { - "type": [ - "string", - "null" - ] - }, - "provisionRequestId": { - "type": [ - "string", - "null" - ] - }, - "templateId": { - "type": [ - "string", - "null" - ] - }, - "updatedAtMs": { - "format": "int64", - "type": "integer" - } - }, - "required": [ - "incarnationId", - "createdAtMs", - "updatedAtMs" - ], - "type": "object" - }, "EnvironmentIngressPutParams": { "properties": { "enabled": { + "description": "Ensure (`true`) or remove (`false`) the provider's public ingress.", "type": "boolean" }, "environmentId": { @@ -9918,7 +9745,8 @@ }, "EnvironmentJobReadParams": { "properties": { - "afterSeq": { + "after": { + "description": "Byte offset to read each job's output from: a previous entry's\n`outputNext`. Omitted with `tailBytes` omitted reads from the start.", "format": "uint64", "minimum": 0, "type": [ @@ -9926,18 +9754,24 @@ "null" ] }, - "includeArtifacts": { - "default": false, - "type": "boolean" - }, "jobs": { "items": { "$ref": "#/definitions/SessionJobHandleInput" }, "type": "array" }, - "outputBytes": { - "format": "uint", + "maxBytes": { + "description": "Most output bytes per job; omit for the server's default page.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "tailBytes": { + "description": "Read each job's last `tailBytes` of output instead; excludes `after`.", + "format": "uint64", "minimum": 0, "type": [ "integer", @@ -9962,45 +9796,29 @@ }, "type": "object" }, - "EnvironmentLifecycleStatusView": { - "oneOf": [ - { - "enum": [ - "provisioning", - "booting", - "ready", - "closing", - "closed", - "failed", - "unknown" - ], - "type": "string" - }, - { - "const": "paused", - "description": "Execution frozen; wakes on next use.", - "type": "string" - }, - { - "const": "suspended", - "description": "Execution state saved to disk; wakes on next use.", - "type": "string" - }, - { - "const": "offline", - "description": "Powered off; provisioned environments wake on next use when the\nprovider supports power control.", - "type": "string" - } - ] - }, "EnvironmentListParams": { "properties": { - "bindingId": { + "after": { + "description": "Exclusive environment id cursor: a previous page's `nextAfter`.", "type": [ "string", "null" ] }, + "includeClosed": { + "description": "Also list closed environments; they are omitted by default.", + "type": "boolean" + }, + "limit": { + "description": "Most environments per page; omit for 100.", + "format": "uint32", + "maximum": 500, + "minimum": 1, + "type": [ + "integer", + "null" + ] + }, "metadata": { "additionalProperties": { "type": "string" @@ -10009,6 +9827,7 @@ "type": "object" }, "providerId": { + "description": "Only provisioned environments of this provider.", "type": [ "string", "null" @@ -10021,15 +9840,16 @@ "null" ] }, - "status": { + "state": { "anyOf": [ { - "$ref": "#/definitions/EnvironmentLifecycleStatusView" + "$ref": "#/definitions/EnvironmentStateView" }, { "type": "null" } - ] + ], + "description": "Only environments in this lifecycle state. `closed` lists closed\nenvironments without `includeClosed`." } }, "type": "object" @@ -10038,12 +9858,54 @@ "properties": { "environments": { "default": [], + "description": "Environments in environment id order.", "items": { "$ref": "#/definitions/EnvironmentView" }, "type": "array" + }, + "nextAfter": { + "description": "Pass as `after` to read the next page; absent on the last page.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "EnvironmentObservedTargetView": { + "description": "What the provider last reported about a provisioned environment's\ntarget. Display only: Lightspeed never converges it, and it may be up to\na few minutes old.", + "properties": { + "message": { + "type": [ + "string", + "null" + ] + }, + "observedAtMs": { + "format": "int64", + "type": "integer" + }, + "phase": { + "$ref": "#/definitions/EnvironmentTargetPhaseView" + }, + "power": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentPowerStateView" + }, + { + "type": "null" + } + ], + "description": "Present only while `phase` is `active`." } }, + "required": [ + "phase", + "observedAtMs" + ], "type": "object" }, "EnvironmentPowerPutParams": { @@ -10057,7 +9919,7 @@ "$ref": "#/definitions/EnvironmentPowerStateView" } ], - "description": "Desired steady power state. Must be one of the provider-reported\n`incarnation.powerStates`." + "description": "Power state to move the target to; one of its template's\n`powerStates`." } }, "required": [ @@ -10078,7 +9940,7 @@ "type": "object" }, "EnvironmentPowerStateView": { - "description": "Steady power state of a provisioned environment.", + "description": "Power state of a provisioned environment's target.", "oneOf": [ { "enum": [ @@ -10140,12 +10002,12 @@ }, "EnvironmentProviderBindingReadParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" } }, "required": [ - "bindingId" + "providerId" ], "type": "object" }, @@ -10160,49 +10022,27 @@ ], "type": "object" }, - "EnvironmentProviderBindingStatusView": { - "enum": [ - "enabled", - "disabled" - ], - "type": "string" - }, "EnvironmentProviderBindingView": { + "description": "A universe's binding to a deployment provider. Creates, templates and\ndata routes of the provider need it enabled.", "properties": { - "bindingId": { - "type": "string" - }, "createdAtMs": { "format": "int64", "type": "integer" }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "enabled": { + "type": "boolean" }, "providerId": { "type": "string" }, - "revision": { - "format": "uint64", - "minimum": 0, - "type": "integer" - }, - "status": { - "$ref": "#/definitions/EnvironmentProviderBindingStatusView" - }, - "updatedAtMs": { - "format": "int64", + "updatedAtMs": { + "format": "int64", "type": "integer" } }, "required": [ - "bindingId", "providerId", - "status", - "revision", + "enabled", "createdAtMs", "updatedAtMs" ], @@ -10253,7 +10093,12 @@ ] }, "identityMode": { - "$ref": "#/definitions/EnvironmentIdentityModeView" + "allOf": [ + { + "$ref": "#/definitions/EnvironmentIdentityModeView" + } + ], + "description": "Immutable once the key exists." }, "maxActiveEnvironments": { "description": "Non-closed environments the key may have at once; omit for unlimited.", @@ -10263,6 +10108,15 @@ "integer", "null" ] + }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: close environments whose daemon has been away\nlonger than this; omit to keep them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] } }, "required": [ @@ -10362,6 +10216,37 @@ ], "type": "object" }, + "EnvironmentRegistrationKeyRotateParams": { + "properties": { + "registrationKeyId": { + "type": "string" + } + }, + "required": [ + "registrationKeyId" + ], + "type": "object" + }, + "EnvironmentRegistrationKeyRotateResponse": { + "properties": { + "registrationKey": { + "$ref": "#/definitions/EnvironmentRegistrationKeyView" + }, + "secret": { + "allOf": [ + { + "$ref": "#/definitions/EnvironmentRegistrationSecretView" + } + ], + "description": "The new plaintext secret, returned only here." + } + }, + "required": [ + "registrationKey", + "secret" + ], + "type": "object" + }, "EnvironmentRegistrationKeyStatusView": { "enum": [ "active", @@ -10370,6 +10255,68 @@ ], "type": "string" }, + "EnvironmentRegistrationKeyUpdateParams": { + "description": "The complete editable policy of a registration key; an absent field is\nunset. The identity mode is immutable.", + "properties": { + "displayName": { + "description": "Group name for the environments this key admits.", + "type": "string" + }, + "ephemeralDisconnectGraceMs": { + "description": "Disconnect grace for ephemeral environments; omit for the default.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "expiresAtMs": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "maxActiveEnvironments": { + "description": "Non-closed environments the key may have at once; omit for unlimited.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: close environments whose daemon has been away\nlonger than this; omit to keep them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "registrationKeyId": { + "type": "string" + } + }, + "required": [ + "registrationKeyId", + "displayName" + ], + "type": "object" + }, + "EnvironmentRegistrationKeyUpdateResponse": { + "properties": { + "registrationKey": { + "$ref": "#/definitions/EnvironmentRegistrationKeyView" + } + }, + "required": [ + "registrationKey" + ], + "type": "object" + }, "EnvironmentRegistrationKeyView": { "properties": { "activeEnvironmentCount": { @@ -10403,7 +10350,7 @@ "$ref": "#/definitions/EnvironmentIdentityModeView" }, "keyPrefix": { - "description": "First characters of the secret, for identification only.", + "description": "First characters of the key's live secret, for identification only.", "type": "string" }, "lastRegisteredAtMs": { @@ -10421,6 +10368,15 @@ "null" ] }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: environments whose daemon has been away longer\nthan this are closed; absent keeps them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "registeredEnvironmentCount": { "description": "Environments ever admitted by this key, closed included.", "format": "uint64", @@ -10439,6 +10395,10 @@ }, "status": { "$ref": "#/definitions/EnvironmentRegistrationKeyStatusView" + }, + "updatedAtMs": { + "format": "int64", + "type": "integer" } }, "required": [ @@ -10450,7 +10410,8 @@ "status", "registeredEnvironmentCount", "activeEnvironmentCount", - "createdAtMs" + "createdAtMs", + "updatedAtMs" ], "type": "object" }, @@ -10479,13 +10440,50 @@ "EnvironmentSourceView": { "oneOf": [ { + "description": "A target a deployment provider created from a template. The provider\nowns its power, idle policy and ingress.", "properties": { - "bindingId": { - "type": "string" + "idle": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentIdlePolicyView" + }, + { + "type": "null" + } + ], + "description": "The create-time idle policy, when one replaced the template's." + }, + "observed": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentObservedTargetView" + }, + { + "type": "null" + } + ], + "description": "Absent until the provider first reported the target." }, "providerId": { "type": "string" }, + "publicEndpoint": { + "description": "The public endpoint the provider last reported.", + "type": [ + "string", + "null" + ] + }, + "targetId": { + "description": "The provider's handle for the target; absent until the provider\naccepted the create.", + "type": [ + "string", + "null" + ] + }, + "templateId": { + "type": "string" + }, "type": { "const": "provisioned", "type": "string" @@ -10494,14 +10492,15 @@ "required": [ "type", "providerId", - "bindingId" + "templateId" ], "type": "object" }, { "properties": { - "connection": { - "$ref": "#/definitions/EnvironmentConnectionView" + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL Lightspeed dials envd at.", + "type": "string" }, "type": { "const": "external", @@ -10510,18 +10509,37 @@ }, "required": [ "type", - "connection" + "endpoint" ], "type": "object" }, { "description": "An envd that dialed the gateway outbound and was admitted by a\nregistration key. The key is the environment's group; the daemon id\nis derived from the daemon's public key and is its identity.", "properties": { + "connected": { + "description": "The daemon's control connection is up.", + "type": "boolean" + }, + "connectionChangedAtMs": { + "description": "When the daemon last connected or disconnected.", + "format": "int64", + "type": "integer" + }, "daemonId": { "type": "string" }, "identityMode": { - "$ref": "#/definitions/EnvironmentIdentityModeView" + "allOf": [ + { + "$ref": "#/definitions/EnvironmentIdentityModeView" + } + ], + "description": "The admitting key's identity mode." + }, + "lastSeenAtMs": { + "description": "When the gateway last recorded the daemon's control connection:\nat connect, at disconnect, and at most every five minutes while\nconnected.", + "format": "int64", + "type": "integer" }, "registrationKeyId": { "type": "string" @@ -10535,15 +10553,98 @@ "type", "registrationKeyId", "daemonId", - "identityMode" + "identityMode", + "connected", + "connectionChangedAtMs", + "lastSeenAtMs" ], "type": "object" } ] }, + "EnvironmentStateView": { + "description": "The lifecycle Lightspeed owns. External and registered environments are\nonly ever `open` or `closed`.", + "oneOf": [ + { + "enum": [ + "open", + "closed" + ], + "type": "string" + }, + { + "const": "provisioning", + "description": "The provider is creating the target; the environment opens after\nLightspeed's first handshake with its envd.", + "type": "string" + }, + { + "const": "closing", + "description": "The provider is removing the target.", + "type": "string" + }, + { + "const": "failed", + "description": "Provisioning or the provider failed; `stateMessage` says why.", + "type": "string" + } + ] + }, + "EnvironmentStatusView": { + "description": "Display status derived from `state` and the latest observation; never\nstored. Tools that use a paused, suspended or stopped environment wake\nit.", + "oneOf": [ + { + "enum": [ + "provisioning", + "ready", + "closing", + "closed", + "failed" + ], + "type": "string" + }, + { + "const": "paused", + "description": "Execution frozen with RAM resident.", + "type": "string" + }, + { + "const": "suspended", + "description": "Execution state saved to disk.", + "type": "string" + }, + { + "const": "stopped", + "description": "Powered off with disk retained.", + "type": "string" + }, + { + "const": "unknown", + "description": "Open, but the provider's latest observation is inconclusive.", + "type": "string" + }, + { + "const": "away", + "description": "Registered, open, and its daemon is not connected.", + "type": "string" + } + ] + }, + "EnvironmentTargetPhaseView": { + "description": "Lifecycle phase of a provider target, as the provider reports it.", + "enum": [ + "provisioning", + "active", + "closing", + "closed", + "failed", + "unknown" + ], + "type": "string" + }, "EnvironmentTemplateListParams": { "properties": { - "bindingId": { + "providerId": { + "description": "Only this provider's templates; omit for every enabled binding.", "type": [ "string", "null" @@ -10568,7 +10669,7 @@ }, "EnvironmentTemplateReadParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" }, "templateId": { @@ -10576,7 +10677,7 @@ } }, "required": [ - "bindingId", + "providerId", "templateId" ], "type": "object" @@ -10592,11 +10693,38 @@ ], "type": "object" }, - "EnvironmentTemplateView": { + "EnvironmentTemplateResourcesView": { + "description": "A template's compute, in provider-neutral units. Each field is absent\nwhen the provider does not fix it.", "properties": { - "bindingId": { - "type": "string" + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "EnvironmentTemplateView": { + "properties": { "deprecated": { "type": "boolean" }, @@ -10609,99 +10737,177 @@ "displayName": { "type": "string" }, + "idle": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentIdlePolicyView" + }, + { + "type": "null" + } + ], + "description": "Idle policy targets get when `environments/create` names none." + }, + "metadata": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "powerStates": { + "description": "Power states `environments/power/put` can move targets of this\ntemplate to; empty without power control.", + "items": { + "$ref": "#/definitions/EnvironmentPowerStateView" + }, + "type": "array" + }, + "providerId": { + "type": "string" + }, + "publicIngress": { + "description": "`environments/ingress/put` can enable public ingress on targets of\nthis template.", + "type": "boolean" + }, + "resources": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentTemplateResourcesView" + }, + { + "type": "null" + } + ], + "description": "Compute targets of this template get; absent when the provider does\nnot fix it." + }, + "templateId": { + "type": "string" + } + }, + "required": [ + "templateId", + "providerId", + "displayName", + "publicIngress", + "deprecated", + "powerStates" + ], + "type": "object" + }, + "EnvironmentUpdateParams": { + "description": "Replace an environment's display name and caller metadata.", + "properties": { + "displayName": { + "description": "The new display name; omit to clear it.", + "type": [ + "string", + "null" + ] + }, + "environmentId": { + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" }, + "description": "The complete new caller metadata; omit or `{}` to clear it.", "type": "object" - }, - "providerId": { - "type": "string" - }, - "publicIngress": { - "type": "boolean" - }, - "templateId": { - "type": "string" } }, "required": [ - "templateId", - "providerId", - "bindingId", - "displayName", - "publicIngress", - "deprecated" + "environmentId" + ], + "type": "object" + }, + "EnvironmentUpdateResponse": { + "properties": { + "environment": { + "$ref": "#/definitions/EnvironmentView" + } + }, + "required": [ + "environment" ], "type": "object" }, "EnvironmentView": { "properties": { - "createdAtMs": { - "format": "int64", - "type": "integer" - }, - "desiredPower": { - "allOf": [ + "closeReason": { + "anyOf": [ { - "$ref": "#/definitions/EnvironmentPowerStateView" + "$ref": "#/definitions/EnvironmentCloseReasonView" + }, + { + "type": "null" } ], - "description": "Lightspeed-owned power intent; `status` is the observed state." + "description": "Present exactly while `state` is `closing` or `closed`." }, - "displayName": { + "closedAtMs": { + "format": "int64", "type": [ - "string", + "integer", "null" ] }, - "environmentId": { - "type": "string" + "createdAtMs": { + "format": "int64", + "type": "integer" }, - "idlePolicy": { + "daemon": { "anyOf": [ { - "$ref": "#/definitions/EnvironmentIdlePolicyView" + "$ref": "#/definitions/EnvironmentDaemonView" }, { "type": "null" } - ] - }, - "incarnation": { - "$ref": "#/definitions/EnvironmentIncarnationView" + ], + "description": "The envd build serving the environment; absent until known." }, - "lastSeenAtMs": { - "description": "Registered environments only: when the gateway last saw the daemon's\ncontrol connection.", - "format": "int64", + "displayName": { "type": [ - "integer", + "string", "null" ] }, + "environmentId": { + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" }, + "description": "Caller metadata.", "type": "object" }, - "publicEndpoint": { + "requestId": { + "description": "The caller's retry identity; absent for registered environments.", "type": [ "string", "null" ] }, - "publicIngressEnabled": { - "type": "boolean" - }, - "requestId": { - "type": "string" - }, "source": { "$ref": "#/definitions/EnvironmentSourceView" }, + "state": { + "$ref": "#/definitions/EnvironmentStateView" + }, + "stateChangedAtMs": { + "description": "When `state` last changed.", + "format": "int64", + "type": "integer" + }, + "stateMessage": { + "description": "Why the environment failed, or what provisioning or closing is\nwaiting on.", + "type": [ + "string", + "null" + ] + }, "status": { - "$ref": "#/definitions/EnvironmentLifecycleStatusView" + "$ref": "#/definitions/EnvironmentStatusView" }, "updatedAtMs": { "format": "int64", @@ -10710,12 +10916,10 @@ }, "required": [ "environmentId", - "requestId", "source", + "state", "status", - "desiredPower", - "incarnation", - "publicIngressEnabled", + "stateChangedAtMs", "createdAtMs", "updatedAtMs" ], @@ -12286,7 +12490,7 @@ }, { "const": "deployment/environment-providers", - "description": "Environment providers, their bindings and adoption.", + "description": "Environment providers and their universe bindings.", "type": "string" }, { @@ -15777,29 +15981,6 @@ ], "type": "object" }, - "SessionJobArtifactView": { - "properties": { - "kind": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "type": "object" - }, "SessionJobCancelEntryView": { "properties": { "error": { @@ -15862,6 +16043,49 @@ ], "type": "string" }, + "SessionJobExitView": { + "description": "How a job's root process ended.", + "oneOf": [ + { + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "type": { + "const": "code", + "type": "string" + } + }, + "required": [ + "type", + "code" + ], + "type": "object" + }, + { + "description": "Killed by this signal, named like `SIGSEGV`.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "signal": { + "type": "string" + }, + "type": { + "const": "signal", + "type": "string" + } + }, + "required": [ + "type", + "signal" + ], + "type": "object" + } + ] + }, "SessionJobHandleInput": { "properties": { "environmentId": { @@ -15897,7 +16121,8 @@ "dataBase64": { "type": "string" }, - "seq": { + "offset": { + "description": "Byte offset of the chunk's first byte in the job's combined output.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15907,12 +16132,50 @@ } }, "required": [ - "seq", + "offset", "stream", "dataBase64" ], "type": "object" }, + "SessionJobOutputGapReasonView": { + "oneOf": [ + { + "const": "overflow", + "description": "Dropped from the middle because the job's retained output exceeded\nits cap; the head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Released after it was delivered.", + "type": "string" + } + ] + }, + "SessionJobOutputGapView": { + "description": "Output bytes in `[offset, offset + omittedBytes)` that cannot be\nreturned.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/definitions/SessionJobOutputGapReasonView" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, "SessionJobOutputStreamView": { "enum": [ "stdout", @@ -15921,13 +16184,8 @@ "type": "string" }, "SessionJobReadEntryView": { + "description": "One job's summary and output page, or why it could not be read: an\nentry with `error` and no summary names a job the environment does not\nknow (never created, or garbage-collected).", "properties": { - "artifacts": { - "items": { - "$ref": "#/definitions/SessionJobArtifactView" - }, - "type": "array" - }, "error": { "type": [ "string", @@ -15945,12 +16203,27 @@ ] }, "outputChunks": { + "description": "Output in offset order.", "items": { "$ref": "#/definitions/SessionJobOutputChunkView" }, "type": "array" }, - "outputNextSeq": { + "outputEnd": { + "description": "Output bytes the job produced so far; final once it is terminal.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "outputGaps": { + "description": "Output ranges between the requested start and `outputNext` that\ncannot be returned.", + "items": { + "$ref": "#/definitions/SessionJobOutputGapView" + }, + "type": "array" + }, + "outputNext": { + "description": "Byte offset to continue reading from: pass it as `after`.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15967,7 +16240,8 @@ } }, "required": [ - "outputNextSeq" + "outputNext", + "outputEnd" ], "type": "object" }, @@ -16087,26 +16361,84 @@ "type": "object" }, "SessionJobStatusView": { - "enum": [ - "accepted", - "queued", - "running", - "succeeded", - "failed", - "cancelRequested", - "cancelled", - "timedOut", - "dependencyFailed", - "interrupted", - "lost" - ], - "type": "string" + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its timeout expired and it was stopped.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of a cancel.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The environment daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "SessionJobStoppedByView": { + "description": "Why Lightspeed stopped a running job.", + "oneOf": [ + { + "const": "timeout", + "description": "Its timeout expired: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "terminate", + "description": "A cancel without `force`: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "kill", + "description": "A cancel with `force`: killed at once.", + "type": "string" + } + ] }, "SessionJobSummaryView": { + "description": "What a job is and how it ended. Instants are the environment's wall\nclock.", "properties": { + "cancelRequestedAtMs": { + "description": "When a cancel reached the job while it ran.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "createdAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": "integer" }, "dependencies": { @@ -16115,13 +16447,26 @@ }, "type": "array" }, - "exitCode": { - "format": "int32", + "durationMs": { + "description": "Run time from start to finish, immune to clock jumps.", + "format": "uint64", + "minimum": 0, "type": [ "integer", "null" ] }, + "exit": { + "anyOf": [ + { + "$ref": "#/definitions/SessionJobExitView" + }, + { + "type": "null" + } + ], + "description": "How the job's root process ended, once it ran and exited." + }, "failure": { "type": [ "string", @@ -16129,8 +16474,7 @@ ] }, "finishedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16145,9 +16489,6 @@ "null" ] }, - "namespace": { - "type": "string" - }, "orphanedDescendants": { "description": "True when the job's root process exited while descendants it spawned\nwere still running; the host terminated them best-effort at job\ncompletion.", "type": "boolean" @@ -16159,16 +16500,18 @@ ] }, "queuedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" ] }, + "requestId": { + "description": "The start request that created the job; part of its identity.", + "type": "string" + }, "startedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16176,11 +16519,22 @@ }, "status": { "$ref": "#/definitions/SessionJobStatusView" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/definitions/SessionJobStoppedByView" + }, + { + "type": "null" + } + ], + "description": "Set when Lightspeed stopped the running job." } }, "required": [ - "namespace", "jobId", + "requestId", "status", "createdAtMs" ], diff --git a/clients/typescript/src/generated/methods.ts b/clients/typescript/src/generated/methods.ts index e9e78e345..daf8840cc 100644 --- a/clients/typescript/src/generated/methods.ts +++ b/clients/typescript/src/generated/methods.ts @@ -38,12 +38,12 @@ export const METHODS = [ "environments/credentials/unbind", "environments/create", "environments/read", + "environments/update", "environments/list", "environments/close", "environments/external/create", "environments/ingress/put", "environments/power/put", - "environments/idle-policy/put", "environments/provider-bindings/list", "environments/provider-bindings/read", "environments/templates/list", @@ -54,6 +54,8 @@ export const METHODS = [ "environments/registration-keys/create", "environments/registration-keys/read", "environments/registration-keys/list", + "environments/registration-keys/update", + "environments/registration-keys/rotate", "environments/registration-keys/revoke", "transcriptions/start", "transcriptions/read", @@ -143,7 +145,6 @@ export const METHODS = [ "deployment/environment-providers/delete", "deployment/environment-providers/bindings/put", "deployment/environment-providers/bindings/delete", - "deployment/environments/adopt", "deployment/channels/accounts/list", ] as const; @@ -338,85 +339,85 @@ export const METHOD_INFO = { scope: "universe", access: {"action":"configure_resource","kind":"universe"}, summary: "Create an environment", - description: "Records an idempotent provisioning intent against an enabled universe binding, attributed to the caller. The provider validates its provider-wide template and provisions through its backend asynchronously.", + description: "Creates a provisioned environment, attributed to the caller, from a template of a provider enabled for the universe. Idempotent per requestId: same inputs, same environment; others conflict. Provisioning until the target's first envd handshake, then open; idle overrides the template default.", }, "environments/read": { scope: "universe", access: {"action":"read","kind":"universe"}, summary: "Read an environment", - description: "Returns the durable universe resource, source binding, logical lifecycle state, and minimal current-incarnation identity.", + description: "Returns one environment: its source, lifecycle state, derived display status, state message, close reason, daemon build, and the latest provider observation.", + }, + "environments/update": { + scope: "universe", + access: {"action":"configure_resource","kind":"universe"}, + summary: "Update an environment", + description: "Replaces the display name and caller metadata of an environment of any kind. Refused for a closed environment.", }, "environments/list": { scope: "universe", access: {"action":"read","kind":"universe"}, summary: "List environments", - description: "Lists the universe environments, optionally filtered by provider, binding, or logical lifecycle state.", + description: "Lists the universe environments in id order, one page at a time, optionally filtered by provider, registration key, state, or metadata. Closed environments are omitted unless includeClosed is set or state is closed. Continue from nextAfter.", }, "environments/close": { scope: "universe", access: {"action":"configure_resource","kind":"universe"}, summary: "Close an environment", - description: "Records an asynchronous idempotent close intent. Provider cleanup is resumed by lifecycle reconciliation; quota is released only after Closed.", + description: "Closes an environment; idempotent. A provisioned environment moves to closing while Lightspeed asks its provider to remove the target, retrying until it is gone, then closed. External and registered environments close at once, and a connected registered daemon is told its environment closed.", }, "environments/external/create": { scope: "universe", access: {"action":"configure_resource","kind":"universe"}, summary: "Register an external environment", - description: "Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, attributed to the caller. Reachability is checked on demand.", + description: "Creates an open environment backed by a Lightspeed-reachable envd endpoint, attributed to the caller. Idempotent per requestId. Reachability is checked on use.", }, "environments/ingress/put": { scope: "universe", access: {"action":"configure_resource","kind":"universe"}, summary: "Configure environment public ingress", - description: "Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health.", + description: "Synchronously ensures or removes public ingress of an open provisioned environment through its provider; source.publicEndpoint carries the reported endpoint. Rejected when the provider offers no ingress; an internal error, safe to retry, when the provider is unreachable or times out. The provider owns hostname, guest port, routing, TLS, and health.", }, "environments/power/put": { scope: "universe", access: {"action":"configure_resource","kind":"universe"}, - summary: "Set environment power intent", - description: "Records the desired power state (running, paused, suspended, or stopped) of a provisioned environment; the lifecycle reconciler converges the provider target asynchronously. Powered-down environments wake transparently on their next use. Rejected when the provider does not support the state.", - }, - "environments/idle-policy/put": { - scope: "universe", - access: {"action":"configure_resource","kind":"universe"}, - summary: "Set environment idle policy", - description: "Replaces or clears the staged idle policy of a provisioned environment. The power reaper measures the daemon's idle duration against the pause/suspend/stop/close thresholds and escalates through the stages the provider supports.", + summary: "Set environment power", + description: "Synchronously moves an open provisioned environment's target to running, paused, suspended, or stopped via its provider and records the observation; the provider may also change power on idle. Rejected without provider power control or when the template lacks the state; an internal error, safe to retry, when the provider is unreachable or times out.", }, "environments/provider-bindings/list": { scope: "universe", access: {"action":"read","kind":"universe"}, summary: "List environment provider bindings", - description: "Lists this universe's revisioned routing and admission bindings to deployment-scoped physical providers.", + description: "Lists this universe's bindings to deployment providers and whether each is enabled.", }, "environments/provider-bindings/read": { scope: "universe", access: {"action":"read","kind":"universe"}, summary: "Read an environment provider binding", - description: "Returns one universe routing and admission binding. Provider-wide templates and physical resource, network, and ingress policy remain provider-owned.", + description: "Returns this universe's binding to one provider. Templates and physical resource, network, and ingress policy remain provider-owned.", }, "environments/templates/list": { scope: "universe", access: {"action":"read","kind":"universe"}, summary: "List environment templates", - description: "Reads immutable templates directly from the selected bound provider controller.", + description: "Reads templates live from the controllers of the universe's enabled provider bindings, or of one provider: compute, power states, public ingress, and default idle policy.", }, "environments/templates/read": { scope: "universe", access: {"action":"read","kind":"universe"}, summary: "Read an environment template", - description: "Returns one immutable template version from the selected bound provider controller.", + description: "Returns one template, read live from a bound provider's controller.", }, "environments/jobs/create": { scope: "universe", access: {"action":"use_resource","kind":"universe"}, summary: "Create environment jobs", - description: "Starts a dependency-aware job group on one environment instance, injecting the environment's configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance rather than a session. A powered-down environment is woken on use: the call fails with environment_not_ready while the wake is in progress; retry with backoff.", + description: "Starts a dependency-aware job group on one environment instance, injecting its configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance, not a session. A powered-down environment wakes on its data route; the call fails with environment_not_ready while it wakes or reconnects; retry with backoff.", }, "environments/jobs/read": { scope: "universe", access: {"action":"read","kind":"universe"}, summary: "Read environment jobs", - description: "Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling.", + description: "Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status.", }, "environments/jobs/cancel": { scope: "universe", @@ -428,7 +429,7 @@ export const METHOD_INFO = { scope: "universe", access: {"action":"configure_resource","kind":"universe"}, summary: "Mint an environment registration key", - description: "Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode, active limit, disconnect grace, and expiry are the key's policy. Treat the secret like a cluster-join credential.", + description: "Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode (immutable), active limit, disconnect grace, offline close-after, and expiry are the key's policy. Treat the secret like a cluster-join credential.", }, "environments/registration-keys/read": { scope: "universe", @@ -442,6 +443,18 @@ export const METHOD_INFO = { summary: "List environment registration keys", description: "Lists this universe's registration keys with policy, status, and derived counts. Each key is the group of the environments it admitted.", }, + "environments/registration-keys/update": { + scope: "universe", + access: {"action":"configure_resource","kind":"universe"}, + summary: "Update an environment registration key", + description: "Replaces a key's editable policy: display name, active limit, disconnect grace, offline close-after, and expiry; an omitted field is unset. The identity mode is immutable. The active limit and expiry apply to later admissions; a changed disconnect grace or offline close-after applies at the next sweep to environments whose daemon is away.", + }, + "environments/registration-keys/rotate": { + scope: "universe", + access: {"action":"configure_resource","kind":"universe"}, + summary: "Rotate an environment registration key", + description: "Replaces the key's secret and display prefix and retires the old secret at once. The key keeps its id, group, policy, and the environments it admitted; daemons already registered are unaffected because they never present the key again. Returns the new secret once. Refused for a revoked key.", + }, "environments/registration-keys/revoke": { scope: "universe", access: {"action":"configure_resource","kind":"universe"}, @@ -944,43 +957,37 @@ export const METHOD_INFO = { scope: "deployment", access: {"kind":"deployment"}, summary: "Put an environment provider", - description: "Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed.", + description: "Registers or replaces a deployment provider: controller endpoint, optional data route base, and optional bearer credential for controller and data routes (write-only: kept when omitted, removed by clearCredential, never returned). The controller is probed: a definitive refusal is invalid_request, unreachable is accepted. The provider never calls Lightspeed.", }, "deployment/environment-providers/list": { scope: "deployment", access: {"kind":"deployment"}, summary: "List environment providers", - description: "Returns every deployment-registered deployment provider and its controller connection.", + description: "Returns every deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored.", }, "deployment/environment-providers/read": { scope: "deployment", access: {"kind":"deployment"}, summary: "Read an environment provider", - description: "Returns one deployment-registered deployment provider and its controller connection.", + description: "Returns one deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored.", }, "deployment/environment-providers/delete": { scope: "deployment", access: {"kind":"deployment"}, summary: "Delete an environment provider", - description: "Deletes a deployment provider only when no universe binding references it.", + description: "Deletes a deployment provider only when no universe binding and no non-closed environment references it.", }, "deployment/environment-providers/bindings/put": { scope: "deployment", access: {"kind":"deployment"}, summary: "Put an environment provider binding", - description: "Creates or replaces one universe's complete revisioned routing and admission binding. A deployment provider may have at most one binding in a universe.", + description: "Creates or replaces one universe's binding to a deployment provider. Only an enabled binding admits creates, template listing, and data routes for the universe.", }, "deployment/environment-providers/bindings/delete": { scope: "deployment", access: {"kind":"deployment"}, summary: "Delete an environment provider binding", - description: "Deletes a universe provider binding only after every referencing environment has reached Closed.", - }, - "deployment/environments/adopt": { - scope: "deployment", - access: {"kind":"deployment"}, - summary: "Adopt a provider environment", - description: "Creates a universe environment by transferring an existing provider target into Lightspeed's managed lifecycle. The caller must explicitly accept ownership transfer.", + description: "Deletes a universe's binding to a provider only when the universe has no non-closed environment of that provider.", }, "deployment/channels/accounts/list": { scope: "deployment", @@ -1285,7 +1292,7 @@ export interface MethodMap { /** * Create an environment * - * Records an idempotent provisioning intent against an enabled universe binding, attributed to the caller. The provider validates its provider-wide template and provisions through its backend asynchronously. + * Creates a provisioned environment, attributed to the caller, from a template of a provider enabled for the universe. Idempotent per requestId: same inputs, same environment; others conflict. Provisioning until the target's first envd handshake, then open; idle overrides the template default. */ "environments/create": { params: Api.EnvironmentCreateParams; @@ -1294,16 +1301,25 @@ export interface MethodMap { /** * Read an environment * - * Returns the durable universe resource, source binding, logical lifecycle state, and minimal current-incarnation identity. + * Returns one environment: its source, lifecycle state, derived display status, state message, close reason, daemon build, and the latest provider observation. */ "environments/read": { params: Api.EnvironmentReadParams; result: Api.AgentApiOutcomeOfEnvironmentReadResponse; }; + /** + * Update an environment + * + * Replaces the display name and caller metadata of an environment of any kind. Refused for a closed environment. + */ + "environments/update": { + params: Api.EnvironmentUpdateParams; + result: Api.AgentApiOutcomeOfEnvironmentUpdateResponse; + }; /** * List environments * - * Lists the universe environments, optionally filtered by provider, binding, or logical lifecycle state. + * Lists the universe environments in id order, one page at a time, optionally filtered by provider, registration key, state, or metadata. Closed environments are omitted unless includeClosed is set or state is closed. Continue from nextAfter. */ "environments/list": { params: Api.EnvironmentListParams; @@ -1312,7 +1328,7 @@ export interface MethodMap { /** * Close an environment * - * Records an asynchronous idempotent close intent. Provider cleanup is resumed by lifecycle reconciliation; quota is released only after Closed. + * Closes an environment; idempotent. A provisioned environment moves to closing while Lightspeed asks its provider to remove the target, retrying until it is gone, then closed. External and registered environments close at once, and a connected registered daemon is told its environment closed. */ "environments/close": { params: Api.EnvironmentCloseParams; @@ -1321,7 +1337,7 @@ export interface MethodMap { /** * Register an external environment * - * Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, attributed to the caller. Reachability is checked on demand. + * Creates an open environment backed by a Lightspeed-reachable envd endpoint, attributed to the caller. Idempotent per requestId. Reachability is checked on use. */ "environments/external/create": { params: Api.EnvironmentExternalCreateParams; @@ -1330,34 +1346,25 @@ export interface MethodMap { /** * Configure environment public ingress * - * Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health. + * Synchronously ensures or removes public ingress of an open provisioned environment through its provider; source.publicEndpoint carries the reported endpoint. Rejected when the provider offers no ingress; an internal error, safe to retry, when the provider is unreachable or times out. The provider owns hostname, guest port, routing, TLS, and health. */ "environments/ingress/put": { params: Api.EnvironmentIngressPutParams; result: Api.AgentApiOutcomeOfEnvironmentIngressPutResponse; }; /** - * Set environment power intent + * Set environment power * - * Records the desired power state (running, paused, suspended, or stopped) of a provisioned environment; the lifecycle reconciler converges the provider target asynchronously. Powered-down environments wake transparently on their next use. Rejected when the provider does not support the state. + * Synchronously moves an open provisioned environment's target to running, paused, suspended, or stopped via its provider and records the observation; the provider may also change power on idle. Rejected without provider power control or when the template lacks the state; an internal error, safe to retry, when the provider is unreachable or times out. */ "environments/power/put": { params: Api.EnvironmentPowerPutParams; result: Api.AgentApiOutcomeOfEnvironmentPowerPutResponse; }; - /** - * Set environment idle policy - * - * Replaces or clears the staged idle policy of a provisioned environment. The power reaper measures the daemon's idle duration against the pause/suspend/stop/close thresholds and escalates through the stages the provider supports. - */ - "environments/idle-policy/put": { - params: Api.EnvironmentIdlePolicyPutParams; - result: Api.AgentApiOutcomeOfEnvironmentIdlePolicyPutResponse; - }; /** * List environment provider bindings * - * Lists this universe's revisioned routing and admission bindings to deployment-scoped physical providers. + * Lists this universe's bindings to deployment providers and whether each is enabled. */ "environments/provider-bindings/list": { params: Api.EnvironmentProviderBindingListParams; @@ -1366,7 +1373,7 @@ export interface MethodMap { /** * Read an environment provider binding * - * Returns one universe routing and admission binding. Provider-wide templates and physical resource, network, and ingress policy remain provider-owned. + * Returns this universe's binding to one provider. Templates and physical resource, network, and ingress policy remain provider-owned. */ "environments/provider-bindings/read": { params: Api.EnvironmentProviderBindingReadParams; @@ -1375,7 +1382,7 @@ export interface MethodMap { /** * List environment templates * - * Reads immutable templates directly from the selected bound provider controller. + * Reads templates live from the controllers of the universe's enabled provider bindings, or of one provider: compute, power states, public ingress, and default idle policy. */ "environments/templates/list": { params: Api.EnvironmentTemplateListParams; @@ -1384,7 +1391,7 @@ export interface MethodMap { /** * Read an environment template * - * Returns one immutable template version from the selected bound provider controller. + * Returns one template, read live from a bound provider's controller. */ "environments/templates/read": { params: Api.EnvironmentTemplateReadParams; @@ -1393,7 +1400,7 @@ export interface MethodMap { /** * Create environment jobs * - * Starts a dependency-aware job group on one environment instance, injecting the environment's configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance rather than a session. A powered-down environment is woken on use: the call fails with environment_not_ready while the wake is in progress; retry with backoff. + * Starts a dependency-aware job group on one environment instance, injecting its configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance, not a session. A powered-down environment wakes on its data route; the call fails with environment_not_ready while it wakes or reconnects; retry with backoff. */ "environments/jobs/create": { params: Api.EnvironmentJobCreateParams; @@ -1402,7 +1409,7 @@ export interface MethodMap { /** * Read environment jobs * - * Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling. + * Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status. */ "environments/jobs/read": { params: Api.EnvironmentJobReadParams; @@ -1420,7 +1427,7 @@ export interface MethodMap { /** * Mint an environment registration key * - * Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode, active limit, disconnect grace, and expiry are the key's policy. Treat the secret like a cluster-join credential. + * Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode (immutable), active limit, disconnect grace, offline close-after, and expiry are the key's policy. Treat the secret like a cluster-join credential. */ "environments/registration-keys/create": { params: Api.EnvironmentRegistrationKeyCreateParams; @@ -1444,6 +1451,24 @@ export interface MethodMap { params: Api.EnvironmentRegistrationKeyListParams; result: Api.AgentApiOutcomeOfEnvironmentRegistrationKeyListResponse; }; + /** + * Update an environment registration key + * + * Replaces a key's editable policy: display name, active limit, disconnect grace, offline close-after, and expiry; an omitted field is unset. The identity mode is immutable. The active limit and expiry apply to later admissions; a changed disconnect grace or offline close-after applies at the next sweep to environments whose daemon is away. + */ + "environments/registration-keys/update": { + params: Api.EnvironmentRegistrationKeyUpdateParams; + result: Api.AgentApiOutcomeOfEnvironmentRegistrationKeyUpdateResponse; + }; + /** + * Rotate an environment registration key + * + * Replaces the key's secret and display prefix and retires the old secret at once. The key keeps its id, group, policy, and the environments it admitted; daemons already registered are unaffected because they never present the key again. Returns the new secret once. Refused for a revoked key. + */ + "environments/registration-keys/rotate": { + params: Api.EnvironmentRegistrationKeyRotateParams; + result: Api.AgentApiOutcomeOfEnvironmentRegistrationKeyRotateResponse; + }; /** * Revoke an environment registration key * @@ -2194,7 +2219,7 @@ export interface MethodMap { /** * Put an environment provider * - * Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed. + * Registers or replaces a deployment provider: controller endpoint, optional data route base, and optional bearer credential for controller and data routes (write-only: kept when omitted, removed by clearCredential, never returned). The controller is probed: a definitive refusal is invalid_request, unreachable is accepted. The provider never calls Lightspeed. */ "deployment/environment-providers/put": { params: Api.DeploymentEnvironmentProviderPutParams; @@ -2203,7 +2228,7 @@ export interface MethodMap { /** * List environment providers * - * Returns every deployment-registered deployment provider and its controller connection. + * Returns every deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored. */ "deployment/environment-providers/list": { params: Api.DeploymentEnvironmentProviderListParams; @@ -2212,7 +2237,7 @@ export interface MethodMap { /** * Read an environment provider * - * Returns one deployment-registered deployment provider and its controller connection. + * Returns one deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored. */ "deployment/environment-providers/read": { params: Api.DeploymentEnvironmentProviderReadParams; @@ -2221,7 +2246,7 @@ export interface MethodMap { /** * Delete an environment provider * - * Deletes a deployment provider only when no universe binding references it. + * Deletes a deployment provider only when no universe binding and no non-closed environment references it. */ "deployment/environment-providers/delete": { params: Api.DeploymentEnvironmentProviderDeleteParams; @@ -2230,7 +2255,7 @@ export interface MethodMap { /** * Put an environment provider binding * - * Creates or replaces one universe's complete revisioned routing and admission binding. A deployment provider may have at most one binding in a universe. + * Creates or replaces one universe's binding to a deployment provider. Only an enabled binding admits creates, template listing, and data routes for the universe. */ "deployment/environment-providers/bindings/put": { params: Api.DeploymentProviderBindingPutParams; @@ -2239,21 +2264,12 @@ export interface MethodMap { /** * Delete an environment provider binding * - * Deletes a universe provider binding only after every referencing environment has reached Closed. + * Deletes a universe's binding to a provider only when the universe has no non-closed environment of that provider. */ "deployment/environment-providers/bindings/delete": { params: Api.DeploymentProviderBindingDeleteParams; result: Api.AgentApiOutcomeOfDeploymentProviderBindingDeleteResponse; }; - /** - * Adopt a provider environment - * - * Creates a universe environment by transferring an existing provider target into Lightspeed's managed lifecycle. The caller must explicitly accept ownership transfer. - */ - "deployment/environments/adopt": { - params: Api.DeploymentEnvironmentAdoptParams; - result: Api.AgentApiOutcomeOfDeploymentEnvironmentAdoptResponse; - }; /** * List channel accounts across universes * @@ -2524,7 +2540,7 @@ export const rpc = { /** * Create an environment * - * Records an idempotent provisioning intent against an enabled universe binding, attributed to the caller. The provider validates its provider-wide template and provisions through its backend asynchronously. + * Creates a provisioned environment, attributed to the caller, from a template of a provider enabled for the universe. Idempotent per requestId: same inputs, same environment; others conflict. Provisioning until the target's first envd handshake, then open; idle overrides the template default. */ environmentsCreate(client: RpcCaller, params: Api.EnvironmentCreateParams): Promise { return client.call("environments/create", params); @@ -2532,15 +2548,23 @@ export const rpc = { /** * Read an environment * - * Returns the durable universe resource, source binding, logical lifecycle state, and minimal current-incarnation identity. + * Returns one environment: its source, lifecycle state, derived display status, state message, close reason, daemon build, and the latest provider observation. */ environmentsRead(client: RpcCaller, params: Api.EnvironmentReadParams): Promise { return client.call("environments/read", params); }, + /** + * Update an environment + * + * Replaces the display name and caller metadata of an environment of any kind. Refused for a closed environment. + */ + environmentsUpdate(client: RpcCaller, params: Api.EnvironmentUpdateParams): Promise { + return client.call("environments/update", params); + }, /** * List environments * - * Lists the universe environments, optionally filtered by provider, binding, or logical lifecycle state. + * Lists the universe environments in id order, one page at a time, optionally filtered by provider, registration key, state, or metadata. Closed environments are omitted unless includeClosed is set or state is closed. Continue from nextAfter. */ environmentsList(client: RpcCaller, params: Api.EnvironmentListParams): Promise { return client.call("environments/list", params); @@ -2548,7 +2572,7 @@ export const rpc = { /** * Close an environment * - * Records an asynchronous idempotent close intent. Provider cleanup is resumed by lifecycle reconciliation; quota is released only after Closed. + * Closes an environment; idempotent. A provisioned environment moves to closing while Lightspeed asks its provider to remove the target, retrying until it is gone, then closed. External and registered environments close at once, and a connected registered daemon is told its environment closed. */ environmentsClose(client: RpcCaller, params: Api.EnvironmentCloseParams): Promise { return client.call("environments/close", params); @@ -2556,7 +2580,7 @@ export const rpc = { /** * Register an external environment * - * Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, attributed to the caller. Reachability is checked on demand. + * Creates an open environment backed by a Lightspeed-reachable envd endpoint, attributed to the caller. Idempotent per requestId. Reachability is checked on use. */ environmentsExternalCreate(client: RpcCaller, params: Api.EnvironmentExternalCreateParams): Promise { return client.call("environments/external/create", params); @@ -2564,31 +2588,23 @@ export const rpc = { /** * Configure environment public ingress * - * Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health. + * Synchronously ensures or removes public ingress of an open provisioned environment through its provider; source.publicEndpoint carries the reported endpoint. Rejected when the provider offers no ingress; an internal error, safe to retry, when the provider is unreachable or times out. The provider owns hostname, guest port, routing, TLS, and health. */ environmentsIngressPut(client: RpcCaller, params: Api.EnvironmentIngressPutParams): Promise { return client.call("environments/ingress/put", params); }, /** - * Set environment power intent + * Set environment power * - * Records the desired power state (running, paused, suspended, or stopped) of a provisioned environment; the lifecycle reconciler converges the provider target asynchronously. Powered-down environments wake transparently on their next use. Rejected when the provider does not support the state. + * Synchronously moves an open provisioned environment's target to running, paused, suspended, or stopped via its provider and records the observation; the provider may also change power on idle. Rejected without provider power control or when the template lacks the state; an internal error, safe to retry, when the provider is unreachable or times out. */ environmentsPowerPut(client: RpcCaller, params: Api.EnvironmentPowerPutParams): Promise { return client.call("environments/power/put", params); }, - /** - * Set environment idle policy - * - * Replaces or clears the staged idle policy of a provisioned environment. The power reaper measures the daemon's idle duration against the pause/suspend/stop/close thresholds and escalates through the stages the provider supports. - */ - environmentsIdlePolicyPut(client: RpcCaller, params: Api.EnvironmentIdlePolicyPutParams): Promise { - return client.call("environments/idle-policy/put", params); - }, /** * List environment provider bindings * - * Lists this universe's revisioned routing and admission bindings to deployment-scoped physical providers. + * Lists this universe's bindings to deployment providers and whether each is enabled. */ environmentsProviderBindingsList(client: RpcCaller, params: Api.EnvironmentProviderBindingListParams): Promise { return client.call("environments/provider-bindings/list", params); @@ -2596,7 +2612,7 @@ export const rpc = { /** * Read an environment provider binding * - * Returns one universe routing and admission binding. Provider-wide templates and physical resource, network, and ingress policy remain provider-owned. + * Returns this universe's binding to one provider. Templates and physical resource, network, and ingress policy remain provider-owned. */ environmentsProviderBindingsRead(client: RpcCaller, params: Api.EnvironmentProviderBindingReadParams): Promise { return client.call("environments/provider-bindings/read", params); @@ -2604,7 +2620,7 @@ export const rpc = { /** * List environment templates * - * Reads immutable templates directly from the selected bound provider controller. + * Reads templates live from the controllers of the universe's enabled provider bindings, or of one provider: compute, power states, public ingress, and default idle policy. */ environmentsTemplatesList(client: RpcCaller, params: Api.EnvironmentTemplateListParams): Promise { return client.call("environments/templates/list", params); @@ -2612,7 +2628,7 @@ export const rpc = { /** * Read an environment template * - * Returns one immutable template version from the selected bound provider controller. + * Returns one template, read live from a bound provider's controller. */ environmentsTemplatesRead(client: RpcCaller, params: Api.EnvironmentTemplateReadParams): Promise { return client.call("environments/templates/read", params); @@ -2620,7 +2636,7 @@ export const rpc = { /** * Create environment jobs * - * Starts a dependency-aware job group on one environment instance, injecting the environment's configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance rather than a session. A powered-down environment is woken on use: the call fails with environment_not_ready while the wake is in progress; retry with backoff. + * Starts a dependency-aware job group on one environment instance, injecting its configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance, not a session. A powered-down environment wakes on its data route; the call fails with environment_not_ready while it wakes or reconnects; retry with backoff. */ environmentsJobsCreate(client: RpcCaller, params: Api.EnvironmentJobCreateParams): Promise { return client.call("environments/jobs/create", params); @@ -2628,7 +2644,7 @@ export const rpc = { /** * Read environment jobs * - * Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling. + * Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status. */ environmentsJobsRead(client: RpcCaller, params: Api.EnvironmentJobReadParams): Promise { return client.call("environments/jobs/read", params); @@ -2644,7 +2660,7 @@ export const rpc = { /** * Mint an environment registration key * - * Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode, active limit, disconnect grace, and expiry are the key's policy. Treat the secret like a cluster-join credential. + * Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode (immutable), active limit, disconnect grace, offline close-after, and expiry are the key's policy. Treat the secret like a cluster-join credential. */ environmentsRegistrationKeysCreate(client: RpcCaller, params: Api.EnvironmentRegistrationKeyCreateParams): Promise { return client.call("environments/registration-keys/create", params); @@ -2665,6 +2681,22 @@ export const rpc = { environmentsRegistrationKeysList(client: RpcCaller, params: Api.EnvironmentRegistrationKeyListParams): Promise { return client.call("environments/registration-keys/list", params); }, + /** + * Update an environment registration key + * + * Replaces a key's editable policy: display name, active limit, disconnect grace, offline close-after, and expiry; an omitted field is unset. The identity mode is immutable. The active limit and expiry apply to later admissions; a changed disconnect grace or offline close-after applies at the next sweep to environments whose daemon is away. + */ + environmentsRegistrationKeysUpdate(client: RpcCaller, params: Api.EnvironmentRegistrationKeyUpdateParams): Promise { + return client.call("environments/registration-keys/update", params); + }, + /** + * Rotate an environment registration key + * + * Replaces the key's secret and display prefix and retires the old secret at once. The key keeps its id, group, policy, and the environments it admitted; daemons already registered are unaffected because they never present the key again. Returns the new secret once. Refused for a revoked key. + */ + environmentsRegistrationKeysRotate(client: RpcCaller, params: Api.EnvironmentRegistrationKeyRotateParams): Promise { + return client.call("environments/registration-keys/rotate", params); + }, /** * Revoke an environment registration key * @@ -3332,7 +3364,7 @@ export const rpc = { /** * Put an environment provider * - * Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed. + * Registers or replaces a deployment provider: controller endpoint, optional data route base, and optional bearer credential for controller and data routes (write-only: kept when omitted, removed by clearCredential, never returned). The controller is probed: a definitive refusal is invalid_request, unreachable is accepted. The provider never calls Lightspeed. */ deploymentEnvironmentProvidersPut(client: RpcCaller, params: Api.DeploymentEnvironmentProviderPutParams): Promise { return client.call("deployment/environment-providers/put", params); @@ -3340,7 +3372,7 @@ export const rpc = { /** * List environment providers * - * Returns every deployment-registered deployment provider and its controller connection. + * Returns every deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored. */ deploymentEnvironmentProvidersList(client: RpcCaller, params: Api.DeploymentEnvironmentProviderListParams): Promise { return client.call("deployment/environment-providers/list", params); @@ -3348,7 +3380,7 @@ export const rpc = { /** * Read an environment provider * - * Returns one deployment-registered deployment provider and its controller connection. + * Returns one deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored. */ deploymentEnvironmentProvidersRead(client: RpcCaller, params: Api.DeploymentEnvironmentProviderReadParams): Promise { return client.call("deployment/environment-providers/read", params); @@ -3356,7 +3388,7 @@ export const rpc = { /** * Delete an environment provider * - * Deletes a deployment provider only when no universe binding references it. + * Deletes a deployment provider only when no universe binding and no non-closed environment references it. */ deploymentEnvironmentProvidersDelete(client: RpcCaller, params: Api.DeploymentEnvironmentProviderDeleteParams): Promise { return client.call("deployment/environment-providers/delete", params); @@ -3364,7 +3396,7 @@ export const rpc = { /** * Put an environment provider binding * - * Creates or replaces one universe's complete revisioned routing and admission binding. A deployment provider may have at most one binding in a universe. + * Creates or replaces one universe's binding to a deployment provider. Only an enabled binding admits creates, template listing, and data routes for the universe. */ deploymentEnvironmentProvidersBindingsPut(client: RpcCaller, params: Api.DeploymentProviderBindingPutParams): Promise { return client.call("deployment/environment-providers/bindings/put", params); @@ -3372,19 +3404,11 @@ export const rpc = { /** * Delete an environment provider binding * - * Deletes a universe provider binding only after every referencing environment has reached Closed. + * Deletes a universe's binding to a provider only when the universe has no non-closed environment of that provider. */ deploymentEnvironmentProvidersBindingsDelete(client: RpcCaller, params: Api.DeploymentProviderBindingDeleteParams): Promise { return client.call("deployment/environment-providers/bindings/delete", params); }, - /** - * Adopt a provider environment - * - * Creates a universe environment by transferring an existing provider target into Lightspeed's managed lifecycle. The caller must explicitly accept ownership transfer. - */ - deploymentEnvironmentsAdopt(client: RpcCaller, params: Api.DeploymentEnvironmentAdoptParams): Promise { - return client.call("deployment/environments/adopt", params); - }, /** * List channel accounts across universes * diff --git a/clients/typescript/src/generated/types.ts b/clients/typescript/src/generated/types.ts index 1092bbcb8..40d2b9cb5 100644 --- a/clients/typescript/src/generated/types.ts +++ b/clients/typescript/src/generated/types.ts @@ -1368,86 +1368,116 @@ export type MethodGroup = | "deployment/environment-providers" | "deployment/channels"; /** - * Steady power state of a provisioned environment. + * Why an environment is closing or closed. * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentPowerStateView". + * via the `definition` "EnvironmentCloseReasonView". */ -export type EnvironmentPowerStateView = "running" | "paused" | "suspended" | "stopped"; +export type EnvironmentCloseReasonView = + "requested" | "keyRevoked" | "ephemeralGrace" | "departed" | "offlineTimeout" | "providerEnded"; /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentSourceView". */ export type EnvironmentSourceView = | { - bindingId: string; + /** + * The create-time idle policy, when one replaced the template's. + */ + idle?: EnvironmentIdlePolicyView | null; + /** + * Absent until the provider first reported the target. + */ + observed?: EnvironmentObservedTargetView | null; providerId: string; + /** + * The public endpoint the provider last reported. + */ + publicEndpoint?: string | null; + /** + * The provider's handle for the target; absent until the provider + * accepted the create. + */ + targetId?: string | null; + templateId: string; type: "provisioned"; } | { - connection: EnvironmentConnectionView; + /** + * `ws://`, `wss://` or `unix://` URL Lightspeed dials envd at. + */ + endpoint: string; type: "external"; } | { + /** + * The daemon's control connection is up. + */ + connected: boolean; + /** + * When the daemon last connected or disconnected. + */ + connectionChangedAtMs: number; daemonId: string; + /** + * The admitting key's identity mode. + */ identityMode: EnvironmentIdentityModeView; + /** + * When the gateway last recorded the daemon's control connection: + * at connect, at disconnect, and at most every five minutes while + * connected. + */ + lastSeenAtMs: number; registrationKeyId: string; type: "registered"; }; /** + * Lifecycle phase of a provider target, as the provider reports it. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentConnectionTransportView". + * via the `definition` "EnvironmentTargetPhaseView". */ -export type EnvironmentConnectionTransportView = - | ("webSocket" | "http" | "stdio" | "ssh") - | { - provider: { - provider_type: string; - }; - }; +export type EnvironmentTargetPhaseView = + "provisioning" | "active" | "closing" | "closed" | "failed" | "unknown"; /** - * What Lightspeed does with a registered environment while its daemon is - * disconnected. Registration-key policy, copied onto each environment. + * Power state of a provisioned environment's target. * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentIdentityModeView". + * via the `definition` "EnvironmentPowerStateView". */ -export type EnvironmentIdentityModeView = "persistent" | "ephemeral"; +export type EnvironmentPowerStateView = "running" | "paused" | "suspended" | "stopped"; /** + * What happens to a registered environment while its daemon is + * disconnected. Policy of the registration key that admitted it. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentLifecycleStatusView". + * via the `definition` "EnvironmentIdentityModeView". */ -export type EnvironmentLifecycleStatusView = - | ("provisioning" | "booting" | "ready" | "closing" | "closed" | "failed" | "unknown") - | "paused" - | "suspended" - | "offline"; +export type EnvironmentIdentityModeView = "persistent" | "ephemeral"; /** + * The lifecycle Lightspeed owns. External and registered environments are + * only ever `open` or `closed`. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "DeploymentEnvironmentProviderTransport". + * via the `definition` "EnvironmentStateView". */ -export type DeploymentEnvironmentProviderTransport = - | { - type: "webSocket"; - } - | { - type: "http"; - } - | { - type: "stdio"; - } - | { - type: "ssh"; - } - | { - providerType: string; - type: "provider"; - }; +export type EnvironmentStateView = ("open" | "closed") | "provisioning" | "closing" | "failed"; /** + * Display status derived from `state` and the latest observation; never + * stored. Tools that use a paused, suspended or stopped environment wake + * it. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentProviderBindingStatusView". + * via the `definition` "EnvironmentStatusView". */ -export type EnvironmentProviderBindingStatusView = "enabled" | "disabled"; +export type EnvironmentStatusView = + | ("provisioning" | "ready" | "closing" | "closed" | "failed") + | "paused" + | "suspended" + | "stopped" + | "unknown" + | "away"; /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentCredentialSourceView". @@ -1466,26 +1496,53 @@ export type EnvironmentCredentialSourceView = type: "directSecret"; }; /** + * How a job's root process ended. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "SessionJobExitView". + */ +export type SessionJobExitView = + | { + code: number; + type: "code"; + } + | { + coreDumped?: boolean; + signal: string; + type: "signal"; + }; +/** + * Job lifecycle. `queued` and `running` are live; the rest are terminal. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobStatusView". */ export type SessionJobStatusView = - | "accepted" - | "queued" | "running" + | "queued" | "succeeded" | "failed" - | "cancelRequested" - | "cancelled" | "timedOut" + | "cancelled" | "dependencyFailed" - | "interrupted" - | "lost"; + | "interrupted"; +/** + * Why Lightspeed stopped a running job. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "SessionJobStoppedByView". + */ +export type SessionJobStoppedByView = "timeout" | "terminate" | "kill"; /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobOutputStreamView". */ export type SessionJobOutputStreamView = "stdout" | "stderr"; +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "SessionJobOutputGapReasonView". + */ +export type SessionJobOutputGapReasonView = "overflow" | "expired"; /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentRegistrationKeyStatusView". @@ -4702,89 +4759,6 @@ export interface DeletedSessionView { displayName?: string | null; sessionId: string; } -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "AgentApiOutcomeOfDeploymentEnvironmentAdoptResponse". - */ -export interface AgentApiOutcomeOfDeploymentEnvironmentAdoptResponse { - notifications?: AgentNotification[]; - result: DeploymentEnvironmentAdoptResponse; -} -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "DeploymentEnvironmentAdoptResponse". - */ -export interface DeploymentEnvironmentAdoptResponse { - environment: EnvironmentView; -} -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentView". - */ -export interface EnvironmentView { - createdAtMs: number; - /** - * Lightspeed-owned power intent; `status` is the observed state. - */ - desiredPower: EnvironmentPowerStateView; - displayName?: string | null; - environmentId: string; - idlePolicy?: EnvironmentIdlePolicyView | null; - incarnation: EnvironmentIncarnationView; - /** - * Registered environments only: when the gateway last saw the daemon's - * control connection. - */ - lastSeenAtMs?: number | null; - metadata?: { - [k: string]: string; - }; - publicEndpoint?: string | null; - publicIngressEnabled: boolean; - requestId: string; - source: EnvironmentSourceView; - status: EnvironmentLifecycleStatusView; - updatedAtMs: number; -} -/** - * Staged idle policy. Thresholds are milliseconds of daemon-reported idle - * time and must be non-decreasing in the order pause, suspend, stop, close. - * Stages whose power state the provider does not support are skipped. - * - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentIdlePolicyView". - */ -export interface EnvironmentIdlePolicyView { - closeAfterMs?: number | null; - pauseAfterMs?: number | null; - stopAfterMs?: number | null; - suspendAfterMs?: number | null; -} -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentIncarnationView". - */ -export interface EnvironmentIncarnationView { - createdAtMs: number; - incarnationId: string; - /** - * Power states the provider reported for this target; empty until - * observed or when the provider offers no power control. - */ - powerStates?: EnvironmentPowerStateView[]; - providerTargetId?: string | null; - provisionRequestId?: string | null; - templateId?: string | null; - updatedAtMs: number; -} -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentConnectionView". - */ -export interface EnvironmentConnectionView { - endpoint: string; - transport: EnvironmentConnectionTransportView; -} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfDeploymentEnvironmentProviderDeleteResponse". @@ -4805,23 +4779,31 @@ export interface DeploymentEnvironmentProviderDeleteResponse { * via the `definition` "DeploymentEnvironmentProviderView". */ export interface DeploymentEnvironmentProviderView { - controllerConnection: DeploymentEnvironmentProviderConnection; + /** + * `ws://`, `wss://` or `unix://` URL of the provider controller. + */ + controllerEndpoint: string; createdAtMs: number; + /** + * Operator-configured `ws://` or `wss://` base of the provider's data + * routes; absent derives it from the controller endpoint. + */ + dataRouteBase?: string | null; displayName?: string | null; - metadata?: { - [k: string]: string; - }; + /** + * The data route base Lightspeed dials: `dataRouteBase`, or the + * controller endpoint's origin followed by `/routes`. Absent only when it + * cannot be derived. + */ + effectiveDataRouteBase?: string | null; + /** + * A bearer credential is stored and presented to the controller and + * data routes; without one no `Authorization` header is sent. + */ + hasCredential: boolean; providerId: string; updatedAtMs: number; } -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "DeploymentEnvironmentProviderConnection". - */ -export interface DeploymentEnvironmentProviderConnection { - endpoint: string; - transport: DeploymentEnvironmentProviderTransport; -} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfDeploymentEnvironmentProviderListResponse". @@ -4883,18 +4865,16 @@ export interface DeploymentProviderBindingDeleteResponse { binding: EnvironmentProviderBindingView; } /** + * A universe's binding to a deployment provider. Creates, templates and + * data routes of the provider need it enabled. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentProviderBindingView". */ export interface EnvironmentProviderBindingView { - bindingId: string; createdAtMs: number; - metadata?: { - [k: string]: string; - }; + enabled: boolean; providerId: string; - revision: number; - status: EnvironmentProviderBindingStatusView; updatedAtMs: number; } /** @@ -5043,6 +5023,92 @@ export interface AgentApiOutcomeOfEnvironmentCloseResponse { export interface EnvironmentCloseResponse { environment: EnvironmentView; } +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentView". + */ +export interface EnvironmentView { + /** + * Present exactly while `state` is `closing` or `closed`. + */ + closeReason?: EnvironmentCloseReasonView | null; + closedAtMs?: number | null; + createdAtMs: number; + /** + * The envd build serving the environment; absent until known. + */ + daemon?: EnvironmentDaemonView | null; + displayName?: string | null; + environmentId: string; + /** + * Caller metadata. + */ + metadata?: { + [k: string]: string; + }; + /** + * The caller's retry identity; absent for registered environments. + */ + requestId?: string | null; + source: EnvironmentSourceView; + state: EnvironmentStateView; + /** + * When `state` last changed. + */ + stateChangedAtMs: number; + /** + * Why the environment failed, or what provisioning or closing is + * waiting on. + */ + stateMessage?: string | null; + status: EnvironmentStatusView; + updatedAtMs: number; +} +/** + * The envd build serving an environment, from its registration or + * Lightspeed's open handshake. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentDaemonView". + */ +export interface EnvironmentDaemonView { + gitSha?: string | null; + protocolVersion: number; + version?: string | null; +} +/** + * Staged idle policy the provider applies. Thresholds are milliseconds of + * envd-reported idle time (running processes, jobs and leftover process + * groups hold it at zero) and must be non-decreasing in the order pause, + * suspend, stop, close. Stages whose power state the template does not + * support are skipped. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentIdlePolicyView". + */ +export interface EnvironmentIdlePolicyView { + closeAfterMs?: number | null; + pauseAfterMs?: number | null; + stopAfterMs?: number | null; + suspendAfterMs?: number | null; +} +/** + * What the provider last reported about a provisioned environment's + * target. Display only: Lightspeed never converges it, and it may be up to + * a few minutes old. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentObservedTargetView". + */ +export interface EnvironmentObservedTargetView { + message?: string | null; + observedAtMs: number; + phase: EnvironmentTargetPhaseView; + /** + * Present only while `phase` is `active`. + */ + power?: EnvironmentPowerStateView | null; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfEnvironmentCreateResponse". @@ -5129,21 +5195,6 @@ export interface AgentApiOutcomeOfEnvironmentExternalCreateResponse { export interface EnvironmentExternalCreateResponse { environment: EnvironmentView; } -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "AgentApiOutcomeOfEnvironmentIdlePolicyPutResponse". - */ -export interface AgentApiOutcomeOfEnvironmentIdlePolicyPutResponse { - notifications?: AgentNotification[]; - result: EnvironmentIdlePolicyPutResponse; -} -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentIdlePolicyPutResponse". - */ -export interface EnvironmentIdlePolicyPutResponse { - environment: EnvironmentView; -} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfEnvironmentIngressPutResponse". @@ -5192,18 +5243,31 @@ export interface SessionJobHandleView { jobId: string; } /** + * What a job is and how it ended. Instants are the environment's wall + * clock. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobSummaryView". */ export interface SessionJobSummaryView { + /** + * When a cancel reached the job while it ran. + */ + cancelRequestedAtMs?: number | null; createdAtMs: number; dependencies?: string[]; - exitCode?: number | null; + /** + * Run time from start to finish, immune to clock jumps. + */ + durationMs?: number | null; + /** + * How the job's root process ended, once it ran and exited. + */ + exit?: SessionJobExitView | null; failure?: string | null; finishedAtMs?: number | null; jobId: string; name?: string | null; - namespace: string; /** * True when the job's root process exited while descendants it spawned * were still running; the host terminated them best-effort at job @@ -5212,8 +5276,16 @@ export interface SessionJobSummaryView { orphanedDescendants?: boolean; queueKey?: string | null; queuedAtMs?: number | null; + /** + * The start request that created the job; part of its identity. + */ + requestId: string; startedAtMs?: number | null; status: SessionJobStatusView; + /** + * Set when Lightspeed stopped the running job. + */ + stoppedBy?: SessionJobStoppedByView | null; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -5261,37 +5333,59 @@ export interface EnvironmentJobReadResponse { jobs?: SessionJobReadEntryView[]; } /** + * One job's summary and output page, or why it could not be read: an + * entry with `error` and no summary names a job the environment does not + * know (never created, or garbage-collected). + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobReadEntryView". */ export interface SessionJobReadEntryView { - artifacts?: SessionJobArtifactView[]; error?: string | null; handle?: SessionJobHandleView | null; + /** + * Output in offset order. + */ outputChunks?: SessionJobOutputChunkView[]; - outputNextSeq: number; + /** + * Output bytes the job produced so far; final once it is terminal. + */ + outputEnd: number; + /** + * Output ranges between the requested start and `outputNext` that + * cannot be returned. + */ + outputGaps?: SessionJobOutputGapView[]; + /** + * Byte offset to continue reading from: pass it as `after`. + */ + outputNext: number; summary?: SessionJobSummaryView | null; } -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "SessionJobArtifactView". - */ -export interface SessionJobArtifactView { - kind?: string | null; - metadata?: { - [k: string]: string; - }; - path: string; -} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "SessionJobOutputChunkView". */ export interface SessionJobOutputChunkView { dataBase64: string; - seq: number; + /** + * Byte offset of the chunk's first byte in the job's combined output. + */ + offset: number; stream: SessionJobOutputStreamView; } +/** + * Output bytes in `[offset, offset + omittedBytes)` that cannot be + * returned. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "SessionJobOutputGapView". + */ +export interface SessionJobOutputGapView { + offset: number; + omittedBytes: number; + reason: SessionJobOutputGapReasonView; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfEnvironmentListResponse". @@ -5305,7 +5399,14 @@ export interface AgentApiOutcomeOfEnvironmentListResponse { * via the `definition` "EnvironmentListResponse". */ export interface EnvironmentListResponse { + /** + * Environments in environment id order. + */ environments?: EnvironmentView[]; + /** + * Pass as `after` to read the next page; absent on the last page. + */ + nextAfter?: string | null; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -5407,11 +5508,16 @@ export interface EnvironmentRegistrationKeyView { expiresAtMs?: number | null; identityMode: EnvironmentIdentityModeView; /** - * First characters of the secret, for identification only. + * First characters of the key's live secret, for identification only. */ keyPrefix: string; lastRegisteredAtMs?: number | null; maxActiveEnvironments?: number | null; + /** + * Persistent keys only: environments whose daemon has been away longer + * than this are closed; absent keeps them open. + */ + offlineCloseAfterMs?: number | null; /** * Environments ever admitted by this key, closed included. */ @@ -5419,6 +5525,7 @@ export interface EnvironmentRegistrationKeyView { registrationKeyId: string; revokedAtMs?: number | null; status: EnvironmentRegistrationKeyStatusView; + updatedAtMs: number; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -5469,6 +5576,40 @@ export interface EnvironmentRegistrationKeyRevokeResponse { closedEnvironmentIds?: string[]; registrationKey: EnvironmentRegistrationKeyView; } +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "AgentApiOutcomeOfEnvironmentRegistrationKeyRotateResponse". + */ +export interface AgentApiOutcomeOfEnvironmentRegistrationKeyRotateResponse { + notifications?: AgentNotification[]; + result: EnvironmentRegistrationKeyRotateResponse; +} +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentRegistrationKeyRotateResponse". + */ +export interface EnvironmentRegistrationKeyRotateResponse { + registrationKey: EnvironmentRegistrationKeyView; + /** + * The new plaintext secret, returned only here. + */ + secret: EnvironmentRegistrationSecretView; +} +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "AgentApiOutcomeOfEnvironmentRegistrationKeyUpdateResponse". + */ +export interface AgentApiOutcomeOfEnvironmentRegistrationKeyUpdateResponse { + notifications?: AgentNotification[]; + result: EnvironmentRegistrationKeyUpdateResponse; +} +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentRegistrationKeyUpdateResponse". + */ +export interface EnvironmentRegistrationKeyUpdateResponse { + registrationKey: EnvironmentRegistrationKeyView; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfEnvironmentTemplateListResponse". @@ -5489,17 +5630,46 @@ export interface EnvironmentTemplateListResponse { * via the `definition` "EnvironmentTemplateView". */ export interface EnvironmentTemplateView { - bindingId: string; deprecated: boolean; description?: string | null; displayName: string; + /** + * Idle policy targets get when `environments/create` names none. + */ + idle?: EnvironmentIdlePolicyView | null; metadata?: { [k: string]: string; }; + /** + * Power states `environments/power/put` can move targets of this + * template to; empty without power control. + */ + powerStates: EnvironmentPowerStateView[]; providerId: string; + /** + * `environments/ingress/put` can enable public ingress on targets of + * this template. + */ publicIngress: boolean; + /** + * Compute targets of this template get; absent when the provider does + * not fix it. + */ + resources?: EnvironmentTemplateResourcesView | null; templateId: string; } +/** + * A template's compute, in provider-neutral units. Each field is absent + * when the provider does not fix it. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentTemplateResourcesView". + */ +export interface EnvironmentTemplateResourcesView { + diskBytes?: number | null; + memoryBytes?: number | null; + vcpus?: number | null; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfEnvironmentTemplateReadResponse". @@ -5515,6 +5685,21 @@ export interface AgentApiOutcomeOfEnvironmentTemplateReadResponse { export interface EnvironmentTemplateReadResponse { template: EnvironmentTemplateView; } +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "AgentApiOutcomeOfEnvironmentUpdateResponse". + */ +export interface AgentApiOutcomeOfEnvironmentUpdateResponse { + notifications?: AgentNotification[]; + result: EnvironmentUpdateResponse; +} +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentUpdateResponse". + */ +export interface EnvironmentUpdateResponse { + environment: EnvironmentView; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "AgentApiOutcomeOfInitializeResponse". @@ -5603,8 +5788,9 @@ export interface ServerInfo { export interface EnvironmentDaemonInfo { gitSha: string; /** - * Environment protocol version the gateway speaks. A daemon registers - * only when it reports exactly this number. + * The envd protocol version (data plane and registration channel) + * the gateway speaks. A daemon registers only when it reports exactly + * this number. */ protocolVersion: number; /** @@ -7387,32 +7573,6 @@ export interface DeploymentDeletedSessionsListParams { after?: string | null; universeId: string; } -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "DeploymentEnvironmentAdoptParams". - */ -export interface DeploymentEnvironmentAdoptParams { - bindingId: string; - displayName?: string | null; - metadata?: { - [k: string]: string; - }; - /** - * Stable caller-generated retry identity inside the destination universe. - */ - requestId: string; - /** - * Provider-native source reference. The Incus provider accepts - * `/` or an instance name in the `default` project. - */ - sourceTarget: string; - /** - * Required acknowledgement that Lightspeed will move, reconfigure, and - * delete the target as part of its ordinary managed lifecycle. - */ - takeOwnership: boolean; - universeId: string; -} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "DeploymentEnvironmentProviderDeleteParams". @@ -7426,15 +7586,35 @@ export interface DeploymentEnvironmentProviderDeleteParams { */ export interface DeploymentEnvironmentProviderListParams {} /** + * Register or replace a provider. `credential` is the optional bearer + * token Lightspeed presents to the provider's controller and data routes: + * it is sealed on receipt and never returned by any method. `Debug` output + * redacts it; request logging must never echo these params. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "DeploymentEnvironmentProviderPutParams". */ export interface DeploymentEnvironmentProviderPutParams { - controllerConnection: DeploymentEnvironmentProviderConnection; + /** + * Remove the stored credential. Excludes `credential`. + */ + clearCredential?: boolean; + /** + * `ws://`, `wss://` or `unix://` URL of the provider controller. + */ + controllerEndpoint: string; + /** + * A new bearer credential; omit to keep the stored one. A provider needs + * none. Excludes `clearCredential`. + */ + credential?: string | null; + /** + * `ws://` or `wss://` base of the provider's data routes; omit to derive + * it from the controller endpoint. Required for a `unix://` controller, + * and must not be `ws://` for a `wss://` controller. + */ + dataRouteBase?: string | null; displayName?: string | null; - metadata?: { - [k: string]: string; - }; providerId: string; } /** @@ -7449,21 +7629,21 @@ export interface DeploymentEnvironmentProviderReadParams { * via the `definition` "DeploymentProviderBindingDeleteParams". */ export interface DeploymentProviderBindingDeleteParams { - bindingId: string; + providerId: string; universeId: string; } /** + * Create or replace a universe's binding to a deployment provider. + * * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "DeploymentProviderBindingPutParams". */ export interface DeploymentProviderBindingPutParams { - bindingId: string; - expectedRevision?: number | null; - metadata?: { - [k: string]: string; - }; + /** + * Admits creates, template listing and data routes for the universe. + */ + enabled: boolean; providerId: string; - status: EnvironmentProviderBindingStatusView; universeId: string; } /** @@ -7536,21 +7716,27 @@ export interface EnvironmentCloseParams { * via the `definition` "EnvironmentCreateParams". */ export interface EnvironmentCreateParams { - bindingId: string; displayName?: string | null; /** - * Optional staged idle policy applied by the power reaper. + * Idle policy passed to the provider in place of the template's `idle`. + * The provider runs the idle clock; it cannot be changed after create. */ - idlePolicy?: EnvironmentIdlePolicyView | null; + idle?: EnvironmentIdlePolicyView | null; metadata?: { [k: string]: string; }; + /** + * The provider to create the environment with; the universe needs an + * enabled binding to it. + */ + providerId: string; /** * Stable caller-generated retry identity, unique inside the universe. + * Retrying with identical inputs returns the same environment. */ requestId: string; /** - * Immutable provider template-version identity. + * The provider template to create the environment from. */ templateId: string; } @@ -7583,32 +7769,28 @@ export interface EnvironmentCredentialUnbindParams { * via the `definition` "EnvironmentExternalCreateParams". */ export interface EnvironmentExternalCreateParams { + displayName?: string | null; /** - * Connection to an envd instance reachable from Lightspeed. + * `ws://`, `wss://` or `unix://` URL of an envd instance reachable from + * Lightspeed. */ - connection: EnvironmentConnectionView; - displayName?: string | null; + endpoint: string; metadata?: { [k: string]: string; }; - requestId: string; -} -/** - * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema - * via the `definition` "EnvironmentIdlePolicyPutParams". - */ -export interface EnvironmentIdlePolicyPutParams { - environmentId: string; /** - * The complete new policy; omit to clear it. + * Stable caller-generated retry identity, unique inside the universe. */ - idlePolicy?: EnvironmentIdlePolicyView | null; + requestId: string; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentIngressPutParams". */ export interface EnvironmentIngressPutParams { + /** + * Ensure (`true`) or remove (`false`) the provider's public ingress. + */ enabled: boolean; environmentId: string; } @@ -7669,17 +7851,38 @@ export interface SessionJobDependencyInput { * via the `definition` "EnvironmentJobReadParams". */ export interface EnvironmentJobReadParams { - afterSeq?: number | null; - includeArtifacts?: boolean; + /** + * Byte offset to read each job's output from: a previous entry's + * `outputNext`. Omitted with `tailBytes` omitted reads from the start. + */ + after?: number | null; jobs: SessionJobHandleInput[]; - outputBytes?: number | null; + /** + * Most output bytes per job; omit for the server's default page. + */ + maxBytes?: number | null; + /** + * Read each job's last `tailBytes` of output instead; excludes `after`. + */ + tailBytes?: number | null; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentListParams". */ export interface EnvironmentListParams { - bindingId?: string | null; + /** + * Exclusive environment id cursor: a previous page's `nextAfter`. + */ + after?: string | null; + /** + * Also list closed environments; they are omitted by default. + */ + includeClosed?: boolean; + /** + * Most environments per page; omit for 100. + */ + limit?: number | null; /** * Only environments carrying every listed metadata pair (AND * semantics); the same filter `session/list` accepts. @@ -7687,12 +7890,19 @@ export interface EnvironmentListParams { metadata?: { [k: string]: string; }; + /** + * Only provisioned environments of this provider. + */ providerId?: string | null; /** * Only registered environments admitted by this registration key. */ registrationKeyId?: string | null; - status?: EnvironmentLifecycleStatusView | null; + /** + * Only environments in this lifecycle state. `closed` lists closed + * environments without `includeClosed`. + */ + state?: EnvironmentStateView | null; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -7701,8 +7911,8 @@ export interface EnvironmentListParams { export interface EnvironmentPowerPutParams { environmentId: string; /** - * Desired steady power state. Must be one of the provider-reported - * `incarnation.powerStates`. + * Power state to move the target to; one of its template's + * `powerStates`. */ power: EnvironmentPowerStateView; } @@ -7716,7 +7926,7 @@ export interface EnvironmentProviderBindingListParams {} * via the `definition` "EnvironmentProviderBindingReadParams". */ export interface EnvironmentProviderBindingReadParams { - bindingId: string; + providerId: string; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -7739,11 +7949,19 @@ export interface EnvironmentRegistrationKeyCreateParams { */ ephemeralDisconnectGraceMs?: number | null; expiresAtMs?: number | null; + /** + * Immutable once the key exists. + */ identityMode: EnvironmentIdentityModeView; /** * Non-closed environments the key may have at once; omit for unlimited. */ maxActiveEnvironments?: number | null; + /** + * Persistent keys only: close environments whose daemon has been away + * longer than this; omit to keep them open. + */ + offlineCloseAfterMs?: number | null; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema @@ -7768,21 +7986,78 @@ export interface EnvironmentRegistrationKeyRevokeParams { closeEnvironments?: boolean; registrationKeyId: string; } +/** + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentRegistrationKeyRotateParams". + */ +export interface EnvironmentRegistrationKeyRotateParams { + registrationKeyId: string; +} +/** + * The complete editable policy of a registration key; an absent field is + * unset. The identity mode is immutable. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentRegistrationKeyUpdateParams". + */ +export interface EnvironmentRegistrationKeyUpdateParams { + /** + * Group name for the environments this key admits. + */ + displayName: string; + /** + * Disconnect grace for ephemeral environments; omit for the default. + */ + ephemeralDisconnectGraceMs?: number | null; + expiresAtMs?: number | null; + /** + * Non-closed environments the key may have at once; omit for unlimited. + */ + maxActiveEnvironments?: number | null; + /** + * Persistent keys only: close environments whose daemon has been away + * longer than this; omit to keep them open. + */ + offlineCloseAfterMs?: number | null; + registrationKeyId: string; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentTemplateListParams". */ export interface EnvironmentTemplateListParams { - bindingId?: string | null; + /** + * Only this provider's templates; omit for every enabled binding. + */ + providerId?: string | null; } /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "EnvironmentTemplateReadParams". */ export interface EnvironmentTemplateReadParams { - bindingId: string; + providerId: string; templateId: string; } +/** + * Replace an environment's display name and caller metadata. + * + * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema + * via the `definition` "EnvironmentUpdateParams". + */ +export interface EnvironmentUpdateParams { + /** + * The new display name; omit to clear it. + */ + displayName?: string | null; + environmentId: string; + /** + * The complete new caller metadata; omit or `{}` to clear it. + */ + metadata?: { + [k: string]: string; + }; +} /** * This interface was referenced by `LightspeedAgentAPI`'s JSON-Schema * via the `definition` "InitializeParams". diff --git a/crates/api/contract/api-reference.md b/crates/api/contract/api-reference.md index d3a9902cc..c17e5c12d 100644 --- a/crates/api/contract/api-reference.md +++ b/crates/api/contract/api-reference.md @@ -419,7 +419,7 @@ Removes one variable-to-credential mapping without deleting the underlying grant **Create an environment** -Records an idempotent provisioning intent against an enabled universe binding, attributed to the caller. The provider validates its provider-wide template and provisions through its backend asynchronously. +Creates a provisioned environment, attributed to the caller, from a template of a provider enabled for the universe. Idempotent per requestId: same inputs, same environment; others conflict. Provisioning until the target's first envd handshake, then open; idle overrides the template default. - Access: `{"kind":"universe","action":"configure_resource"}` - Group: `environments` @@ -432,7 +432,7 @@ Records an idempotent provisioning intent against an enabled universe binding, a **Read an environment** -Returns the durable universe resource, source binding, logical lifecycle state, and minimal current-incarnation identity. +Returns one environment: its source, lifecycle state, derived display status, state message, close reason, daemon build, and the latest provider observation. - Access: `{"kind":"universe","action":"read"}` - Group: `environments` @@ -441,11 +441,24 @@ Returns the durable universe resource, source binding, logical lifecycle state, - Params: `EnvironmentReadParams` - Result: `AgentApiOutcome` +### `environments/update` + +**Update an environment** + +Replaces the display name and caller metadata of an environment of any kind. Refused for a closed environment. + +- Access: `{"kind":"universe","action":"configure_resource"}` +- Group: `environments` +- Role: `operator` +- Target: `none` +- Params: `EnvironmentUpdateParams` +- Result: `AgentApiOutcome` + ### `environments/list` **List environments** -Lists the universe environments, optionally filtered by provider, binding, or logical lifecycle state. +Lists the universe environments in id order, one page at a time, optionally filtered by provider, registration key, state, or metadata. Closed environments are omitted unless includeClosed is set or state is closed. Continue from nextAfter. - Access: `{"kind":"universe","action":"read"}` - Group: `environments` @@ -458,7 +471,7 @@ Lists the universe environments, optionally filtered by provider, binding, or lo **Close an environment** -Records an asynchronous idempotent close intent. Provider cleanup is resumed by lifecycle reconciliation; quota is released only after Closed. +Closes an environment; idempotent. A provisioned environment moves to closing while Lightspeed asks its provider to remove the target, retrying until it is gone, then closed. External and registered environments close at once, and a connected registered daemon is told its environment closed. - Access: `{"kind":"universe","action":"configure_resource"}` - Group: `environments` @@ -471,7 +484,7 @@ Records an asynchronous idempotent close intent. Provider cleanup is resumed by **Register an external environment** -Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, attributed to the caller. Reachability is checked on demand. +Creates an open environment backed by a Lightspeed-reachable envd endpoint, attributed to the caller. Idempotent per requestId. Reachability is checked on use. - Access: `{"kind":"universe","action":"configure_resource"}` - Group: `environments` @@ -484,7 +497,7 @@ Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, **Configure environment public ingress** -Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health. +Synchronously ensures or removes public ingress of an open provisioned environment through its provider; source.publicEndpoint carries the reported endpoint. Rejected when the provider offers no ingress; an internal error, safe to retry, when the provider is unreachable or times out. The provider owns hostname, guest port, routing, TLS, and health. - Access: `{"kind":"universe","action":"configure_resource"}` - Group: `environments` @@ -495,9 +508,9 @@ Synchronously enables or disables one provider-authorized HTTPS endpoint for a p ### `environments/power/put` -**Set environment power intent** +**Set environment power** -Records the desired power state (running, paused, suspended, or stopped) of a provisioned environment; the lifecycle reconciler converges the provider target asynchronously. Powered-down environments wake transparently on their next use. Rejected when the provider does not support the state. +Synchronously moves an open provisioned environment's target to running, paused, suspended, or stopped via its provider and records the observation; the provider may also change power on idle. Rejected without provider power control or when the template lacks the state; an internal error, safe to retry, when the provider is unreachable or times out. - Access: `{"kind":"universe","action":"configure_resource"}` - Group: `environments` @@ -506,24 +519,11 @@ Records the desired power state (running, paused, suspended, or stopped) of a pr - Params: `EnvironmentPowerPutParams` - Result: `AgentApiOutcome` -### `environments/idle-policy/put` - -**Set environment idle policy** - -Replaces or clears the staged idle policy of a provisioned environment. The power reaper measures the daemon's idle duration against the pause/suspend/stop/close thresholds and escalates through the stages the provider supports. - -- Access: `{"kind":"universe","action":"configure_resource"}` -- Group: `environments` -- Role: `operator` -- Target: `none` -- Params: `EnvironmentIdlePolicyPutParams` -- Result: `AgentApiOutcome` - ### `environments/provider-bindings/list` **List environment provider bindings** -Lists this universe's revisioned routing and admission bindings to deployment-scoped physical providers. +Lists this universe's bindings to deployment providers and whether each is enabled. - Access: `{"kind":"universe","action":"read"}` - Group: `environments` @@ -536,7 +536,7 @@ Lists this universe's revisioned routing and admission bindings to deployment-sc **Read an environment provider binding** -Returns one universe routing and admission binding. Provider-wide templates and physical resource, network, and ingress policy remain provider-owned. +Returns this universe's binding to one provider. Templates and physical resource, network, and ingress policy remain provider-owned. - Access: `{"kind":"universe","action":"read"}` - Group: `environments` @@ -549,7 +549,7 @@ Returns one universe routing and admission binding. Provider-wide templates and **List environment templates** -Reads immutable templates directly from the selected bound provider controller. +Reads templates live from the controllers of the universe's enabled provider bindings, or of one provider: compute, power states, public ingress, and default idle policy. - Access: `{"kind":"universe","action":"read"}` - Group: `environments` @@ -562,7 +562,7 @@ Reads immutable templates directly from the selected bound provider controller. **Read an environment template** -Returns one immutable template version from the selected bound provider controller. +Returns one template, read live from a bound provider's controller. - Access: `{"kind":"universe","action":"read"}` - Group: `environments` @@ -575,7 +575,7 @@ Returns one immutable template version from the selected bound provider controll **Create environment jobs** -Starts a dependency-aware job group on one environment instance, injecting the environment's configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance rather than a session. A powered-down environment is woken on use: the call fails with environment_not_ready while the wake is in progress; retry with backoff. +Starts a dependency-aware job group on one environment instance, injecting its configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance, not a session. A powered-down environment wakes on its data route; the call fails with environment_not_ready while it wakes or reconnects; retry with backoff. - Access: `{"kind":"universe","action":"use_resource"}` - Group: `environments` @@ -588,7 +588,7 @@ Starts a dependency-aware job group on one environment instance, injecting the e **Read environment jobs** -Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling. +Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status. - Access: `{"kind":"universe","action":"read"}` - Group: `environments` @@ -614,7 +614,7 @@ Requests cancellation for selected jobs, optionally including dependents. Force **Mint an environment registration key** -Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode, active limit, disconnect grace, and expiry are the key's policy. Treat the secret like a cluster-join credential. +Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode (immutable), active limit, disconnect grace, offline close-after, and expiry are the key's policy. Treat the secret like a cluster-join credential. - Access: `{"kind":"universe","action":"configure_resource"}` - Group: `environments` @@ -649,6 +649,32 @@ Lists this universe's registration keys with policy, status, and derived counts. - Params: `EnvironmentRegistrationKeyListParams` - Result: `AgentApiOutcome` +### `environments/registration-keys/update` + +**Update an environment registration key** + +Replaces a key's editable policy: display name, active limit, disconnect grace, offline close-after, and expiry; an omitted field is unset. The identity mode is immutable. The active limit and expiry apply to later admissions; a changed disconnect grace or offline close-after applies at the next sweep to environments whose daemon is away. + +- Access: `{"kind":"universe","action":"configure_resource"}` +- Group: `environments` +- Role: `operator` +- Target: `none` +- Params: `EnvironmentRegistrationKeyUpdateParams` +- Result: `AgentApiOutcome` + +### `environments/registration-keys/rotate` + +**Rotate an environment registration key** + +Replaces the key's secret and display prefix and retires the old secret at once. The key keeps its id, group, policy, and the environments it admitted; daemons already registered are unaffected because they never present the key again. Returns the new secret once. Refused for a revoked key. + +- Access: `{"kind":"universe","action":"configure_resource"}` +- Group: `environments` +- Role: `operator` +- Target: `none` +- Params: `EnvironmentRegistrationKeyRotateParams` +- Result: `AgentApiOutcome` + ### `environments/registration-keys/revoke` **Revoke an environment registration key** @@ -1747,7 +1773,7 @@ Revokes the key with this display prefix; revoking a revoked key keeps its first **Put an environment provider** -Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed. +Registers or replaces a deployment provider: controller endpoint, optional data route base, and optional bearer credential for controller and data routes (write-only: kept when omitted, removed by clearCredential, never returned). The controller is probed: a definitive refusal is invalid_request, unreachable is accepted. The provider never calls Lightspeed. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` @@ -1760,7 +1786,7 @@ Registers or replaces one deployment provider and its controller connection. The **List environment providers** -Returns every deployment-registered deployment provider and its controller connection. +Returns every deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` @@ -1773,7 +1799,7 @@ Returns every deployment-registered deployment provider and its controller conne **Read an environment provider** -Returns one deployment-registered deployment provider and its controller connection. +Returns one deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` @@ -1786,7 +1812,7 @@ Returns one deployment-registered deployment provider and its controller connect **Delete an environment provider** -Deletes a deployment provider only when no universe binding references it. +Deletes a deployment provider only when no universe binding and no non-closed environment references it. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` @@ -1799,7 +1825,7 @@ Deletes a deployment provider only when no universe binding references it. **Put an environment provider binding** -Creates or replaces one universe's complete revisioned routing and admission binding. A deployment provider may have at most one binding in a universe. +Creates or replaces one universe's binding to a deployment provider. Only an enabled binding admits creates, template listing, and data routes for the universe. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` @@ -1812,7 +1838,7 @@ Creates or replaces one universe's complete revisioned routing and admission bin **Delete an environment provider binding** -Deletes a universe provider binding only after every referencing environment has reached Closed. +Deletes a universe's binding to a provider only when the universe has no non-closed environment of that provider. - Access: `{"kind":"deployment"}` - Group: `deployment/environment-providers` @@ -1821,19 +1847,6 @@ Deletes a universe provider binding only after every referencing environment has - Params: `DeploymentProviderBindingDeleteParams` - Result: `AgentApiOutcome` -### `deployment/environments/adopt` - -**Adopt a provider environment** - -Creates a universe environment by transferring an existing provider target into Lightspeed's managed lifecycle. The caller must explicitly accept ownership transfer. - -- Access: `{"kind":"deployment"}` -- Group: `deployment/environment-providers` -- Role: `none` -- Target: `none` -- Params: `DeploymentEnvironmentAdoptParams` -- Result: `AgentApiOutcome` - ### `deployment/channels/accounts/list` **List channel accounts across universes** diff --git a/crates/api/contract/api.schema.json b/crates/api/contract/api.schema.json index d2bce1b0a..2fa7322b8 100644 --- a/crates/api/contract/api.schema.json +++ b/crates/api/contract/api.schema.json @@ -139,7 +139,7 @@ }, { "const": "environment_not_ready", - "description": "The environment exists but is not reachable yet: it is still\nprovisioning/booting, or it was powered down and a wake (desired\npower `running`) has been initiated by this request. Distinct from\n`Rejected` so clients retry with backoff instead of failing —\npolling and automation callers lean on this for wake-on-use.", + "description": "The environment exists but is not reachable yet: it is still\nprovisioning, its provider is waking or booting it after a data\nroute dial, or its registered daemon is reconnecting. Distinct from\n`Rejected` so clients retry with backoff instead of failing —\npolling and automation callers lean on this for wake-on-use.", "type": "string" }, { @@ -1084,23 +1084,6 @@ ], "type": "object" }, - "AgentApiOutcomeOfDeploymentEnvironmentAdoptResponse": { - "properties": { - "notifications": { - "items": { - "$ref": "#/definitions/AgentNotification" - }, - "type": "array" - }, - "result": { - "$ref": "#/definitions/DeploymentEnvironmentAdoptResponse" - } - }, - "required": [ - "result" - ], - "type": "object" - }, "AgentApiOutcomeOfDeploymentEnvironmentProviderDeleteResponse": { "properties": { "notifications": { @@ -1390,23 +1373,6 @@ ], "type": "object" }, - "AgentApiOutcomeOfEnvironmentIdlePolicyPutResponse": { - "properties": { - "notifications": { - "items": { - "$ref": "#/definitions/AgentNotification" - }, - "type": "array" - }, - "result": { - "$ref": "#/definitions/EnvironmentIdlePolicyPutResponse" - } - }, - "required": [ - "result" - ], - "type": "object" - }, "AgentApiOutcomeOfEnvironmentIngressPutResponse": { "properties": { "notifications": { @@ -1628,6 +1594,40 @@ ], "type": "object" }, + "AgentApiOutcomeOfEnvironmentRegistrationKeyRotateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/definitions/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/definitions/EnvironmentRegistrationKeyRotateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, + "AgentApiOutcomeOfEnvironmentRegistrationKeyUpdateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/definitions/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/definitions/EnvironmentRegistrationKeyUpdateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, "AgentApiOutcomeOfEnvironmentTemplateListResponse": { "properties": { "notifications": { @@ -1662,6 +1662,23 @@ ], "type": "object" }, + "AgentApiOutcomeOfEnvironmentUpdateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/definitions/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/definitions/EnvironmentUpdateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, "AgentApiOutcomeOfInitializeResponse": { "properties": { "notifications": { @@ -8710,74 +8727,6 @@ ], "type": "object" }, - "DeploymentEnvironmentAdoptParams": { - "properties": { - "bindingId": { - "type": "string" - }, - "displayName": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "requestId": { - "description": "Stable caller-generated retry identity inside the destination universe.", - "type": "string" - }, - "sourceTarget": { - "description": "Provider-native source reference. The Incus provider accepts\n`/` or an instance name in the `default` project.", - "type": "string" - }, - "takeOwnership": { - "description": "Required acknowledgement that Lightspeed will move, reconfigure, and\ndelete the target as part of its ordinary managed lifecycle.", - "type": "boolean" - }, - "universeId": { - "type": "string" - } - }, - "required": [ - "universeId", - "requestId", - "bindingId", - "sourceTarget", - "takeOwnership" - ], - "type": "object" - }, - "DeploymentEnvironmentAdoptResponse": { - "properties": { - "environment": { - "$ref": "#/definitions/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, - "DeploymentEnvironmentProviderConnection": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/definitions/DeploymentEnvironmentProviderTransport" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "DeploymentEnvironmentProviderDeleteParams": { "properties": { "providerId": { @@ -8818,21 +8767,35 @@ "type": "object" }, "DeploymentEnvironmentProviderPutParams": { + "description": "Register or replace a provider. `credential` is the optional bearer\ntoken Lightspeed presents to the provider's controller and data routes:\nit is sealed on receipt and never returned by any method. `Debug` output\nredacts it; request logging must never echo these params.", "properties": { - "controllerConnection": { - "$ref": "#/definitions/DeploymentEnvironmentProviderConnection" + "clearCredential": { + "description": "Remove the stored credential. Excludes `credential`.", + "type": "boolean" }, - "displayName": { + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" + }, + "credential": { + "description": "A new bearer credential; omit to keep the stored one. A provider needs\nnone. Excludes `clearCredential`.", "type": [ "string", "null" ] }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "dataRouteBase": { + "description": "`ws://` or `wss://` base of the provider's data routes; omit to derive\nit from the controller endpoint. Required for a `unix://` controller,\nand must not be `ws://` for a `wss://` controller.", + "type": [ + "string", + "null" + ] + }, + "displayName": { + "type": [ + "string", + "null" + ] }, "providerId": { "type": "string" @@ -8840,7 +8803,7 @@ }, "required": [ "providerId", - "controllerConnection" + "controllerEndpoint" ], "type": "object" }, @@ -8877,94 +8840,39 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderTransport": { - "oneOf": [ - { - "properties": { - "type": { - "const": "webSocket", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "http", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "stdio", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "ssh", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "providerType": { - "type": "string" - }, - "type": { - "const": "provider", - "type": "string" - } - }, - "required": [ - "type", - "providerType" - ], - "type": "object" - } - ] - }, "DeploymentEnvironmentProviderView": { "properties": { - "controllerConnection": { - "$ref": "#/definitions/DeploymentEnvironmentProviderConnection" + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" }, "createdAtMs": { "format": "int64", "type": "integer" }, + "dataRouteBase": { + "description": "Operator-configured `ws://` or `wss://` base of the provider's data\nroutes; absent derives it from the controller endpoint.", + "type": [ + "string", + "null" + ] + }, "displayName": { "type": [ "string", "null" ] }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "effectiveDataRouteBase": { + "description": "The data route base Lightspeed dials: `dataRouteBase`, or the\ncontroller endpoint's origin followed by `/routes`. Absent only when it\ncannot be derived.", + "type": [ + "string", + "null" + ] + }, + "hasCredential": { + "description": "A bearer credential is stored and presented to the controller and\ndata routes; without one no `Authorization` header is sent.", + "type": "boolean" }, "providerId": { "type": "string" @@ -8976,7 +8884,8 @@ }, "required": [ "providerId", - "controllerConnection", + "controllerEndpoint", + "hasCredential", "createdAtMs", "updatedAtMs" ], @@ -8984,7 +8893,7 @@ }, "DeploymentProviderBindingDeleteParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" }, "universeId": { @@ -8993,7 +8902,7 @@ }, "required": [ "universeId", - "bindingId" + "providerId" ], "type": "object" }, @@ -9009,39 +8918,23 @@ "type": "object" }, "DeploymentProviderBindingPutParams": { + "description": "Create or replace a universe's binding to a deployment provider.", "properties": { - "bindingId": { - "type": "string" - }, - "expectedRevision": { - "format": "uint64", - "minimum": 0, - "type": [ - "integer", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "enabled": { + "description": "Admits creates, template listing and data routes for the universe.", + "type": "boolean" }, "providerId": { "type": "string" }, - "status": { - "$ref": "#/definitions/EnvironmentProviderBindingStatusView" - }, "universeId": { "type": "string" } }, "required": [ "universeId", - "bindingId", "providerId", - "status" + "enabled" ], "type": "object" }, @@ -9328,77 +9221,61 @@ ], "type": "object" }, - "EnvironmentCloseResponse": { - "properties": { - "environment": { - "$ref": "#/definitions/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, - "EnvironmentConnectionTransportView": { + "EnvironmentCloseReasonView": { + "description": "Why an environment is closing or closed.", "oneOf": [ { - "enum": [ - "webSocket", - "http", - "stdio", - "ssh" - ], + "const": "requested", + "description": "`environments/close`.", "type": "string" }, { - "additionalProperties": false, - "properties": { - "provider": { - "properties": { - "provider_type": { - "type": "string" - } - }, - "required": [ - "provider_type" - ], - "type": "object" - } - }, - "required": [ - "provider" - ], - "type": "object" + "const": "keyRevoked", + "description": "Its registration key was revoked with `closeEnvironments`.", + "type": "string" + }, + { + "const": "ephemeralGrace", + "description": "Its ephemeral daemon stayed away longer than the key's disconnect\ngrace.", + "type": "string" + }, + { + "const": "departed", + "description": "Its ephemeral daemon said goodbye.", + "type": "string" + }, + { + "const": "offlineTimeout", + "description": "Its persistent daemon stayed away longer than the key's\n`offlineCloseAfterMs`.", + "type": "string" + }, + { + "const": "providerEnded", + "description": "The provider reported the target closed or gone.", + "type": "string" } ] }, - "EnvironmentConnectionView": { + "EnvironmentCloseResponse": { "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/definitions/EnvironmentConnectionTransportView" + "environment": { + "$ref": "#/definitions/EnvironmentView" } }, "required": [ - "endpoint", - "transport" + "environment" ], "type": "object" }, "EnvironmentCreateParams": { "properties": { - "bindingId": { - "type": "string" - }, "displayName": { "type": [ "string", "null" ] }, - "idlePolicy": { + "idle": { "anyOf": [ { "$ref": "#/definitions/EnvironmentIdlePolicyView" @@ -9407,7 +9284,7 @@ "type": "null" } ], - "description": "Optional staged idle policy applied by the power reaper." + "description": "Idle policy passed to the provider in place of the template's `idle`.\nThe provider runs the idle clock; it cannot be changed after create." }, "metadata": { "additionalProperties": { @@ -9415,18 +9292,22 @@ }, "type": "object" }, + "providerId": { + "description": "The provider to create the environment with; the universe needs an\nenabled binding to it.", + "type": "string" + }, "requestId": { - "description": "Stable caller-generated retry identity, unique inside the universe.", + "description": "Stable caller-generated retry identity, unique inside the universe.\nRetrying with identical inputs returns the same environment.", "type": "string" }, "templateId": { - "description": "Immutable provider template-version identity.", + "description": "The provider template to create the environment from.", "type": "string" } }, "required": [ "requestId", - "bindingId", + "providerId", "templateId" ], "type": "object" @@ -9609,7 +9490,7 @@ "type": "string" }, "protocolVersion": { - "description": "Environment protocol version the gateway speaks. A daemon registers\nonly when it reports exactly this number.", + "description": "The envd protocol version (data plane and registration channel)\nthe gateway speaks. A daemon registers only when it reports exactly\nthis number.", "format": "uint32", "minimum": 0, "type": "integer" @@ -9633,22 +9514,44 @@ ], "type": "object" }, - "EnvironmentExternalCreateParams": { + "EnvironmentDaemonView": { + "description": "The envd build serving an environment, from its registration or\nLightspeed's open handshake.", "properties": { - "connection": { - "allOf": [ - { - "$ref": "#/definitions/EnvironmentConnectionView" - } - ], - "description": "Connection to an envd instance reachable from Lightspeed." + "gitSha": { + "type": [ + "string", + "null" + ] + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" }, + "version": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "protocolVersion" + ], + "type": "object" + }, + "EnvironmentExternalCreateParams": { + "properties": { "displayName": { "type": [ "string", "null" ] }, + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of an envd instance reachable from\nLightspeed.", + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" @@ -9656,12 +9559,13 @@ "type": "object" }, "requestId": { + "description": "Stable caller-generated retry identity, unique inside the universe.", "type": "string" } }, "required": [ "requestId", - "connection" + "endpoint" ], "type": "object" }, @@ -9677,55 +9581,23 @@ "type": "object" }, "EnvironmentIdentityModeView": { - "description": "What Lightspeed does with a registered environment while its daemon is\ndisconnected. Registration-key policy, copied onto each environment.", + "description": "What happens to a registered environment while its daemon is\ndisconnected. Policy of the registration key that admitted it.", "oneOf": [ { "const": "persistent", - "description": "Stays offline until explicitly closed.", + "description": "Stays open: tools wait up to two minutes for the daemon to reconnect.\nClosed explicitly, or after the key's `offlineCloseAfterMs`.", "type": "string" }, { "const": "ephemeral", - "description": "Closed once the daemon has been away longer than the key's\ndisconnect grace.", + "description": "Unusable while disconnected; closed once the daemon has been away\nlonger than the key's disconnect grace, or at once when it says\ngoodbye.", "type": "string" } ] }, - "EnvironmentIdlePolicyPutParams": { - "properties": { - "environmentId": { - "type": "string" - }, - "idlePolicy": { - "anyOf": [ - { - "$ref": "#/definitions/EnvironmentIdlePolicyView" - }, - { - "type": "null" - } - ], - "description": "The complete new policy; omit to clear it." - } - }, - "required": [ - "environmentId" - ], - "type": "object" - }, - "EnvironmentIdlePolicyPutResponse": { - "properties": { - "environment": { - "$ref": "#/definitions/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, "EnvironmentIdlePolicyView": { - "description": "Staged idle policy. Thresholds are milliseconds of daemon-reported idle\ntime and must be non-decreasing in the order pause, suspend, stop, close.\nStages whose power state the provider does not support are skipped.", + "additionalProperties": false, + "description": "Staged idle policy the provider applies. Thresholds are milliseconds of\nenvd-reported idle time (running processes, jobs and leftover process\ngroups hold it at zero) and must be non-decreasing in the order pause,\nsuspend, stop, close. Stages whose power state the template does not\nsupport are skipped.", "properties": { "closeAfterMs": { "format": "uint64", @@ -9762,55 +9634,10 @@ }, "type": "object" }, - "EnvironmentIncarnationView": { - "properties": { - "createdAtMs": { - "format": "int64", - "type": "integer" - }, - "incarnationId": { - "type": "string" - }, - "powerStates": { - "description": "Power states the provider reported for this target; empty until\nobserved or when the provider offers no power control.", - "items": { - "$ref": "#/definitions/EnvironmentPowerStateView" - }, - "type": "array" - }, - "providerTargetId": { - "type": [ - "string", - "null" - ] - }, - "provisionRequestId": { - "type": [ - "string", - "null" - ] - }, - "templateId": { - "type": [ - "string", - "null" - ] - }, - "updatedAtMs": { - "format": "int64", - "type": "integer" - } - }, - "required": [ - "incarnationId", - "createdAtMs", - "updatedAtMs" - ], - "type": "object" - }, "EnvironmentIngressPutParams": { "properties": { "enabled": { + "description": "Ensure (`true`) or remove (`false`) the provider's public ingress.", "type": "boolean" }, "environmentId": { @@ -9918,7 +9745,8 @@ }, "EnvironmentJobReadParams": { "properties": { - "afterSeq": { + "after": { + "description": "Byte offset to read each job's output from: a previous entry's\n`outputNext`. Omitted with `tailBytes` omitted reads from the start.", "format": "uint64", "minimum": 0, "type": [ @@ -9926,18 +9754,24 @@ "null" ] }, - "includeArtifacts": { - "default": false, - "type": "boolean" - }, "jobs": { "items": { "$ref": "#/definitions/SessionJobHandleInput" }, "type": "array" }, - "outputBytes": { - "format": "uint", + "maxBytes": { + "description": "Most output bytes per job; omit for the server's default page.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "tailBytes": { + "description": "Read each job's last `tailBytes` of output instead; excludes `after`.", + "format": "uint64", "minimum": 0, "type": [ "integer", @@ -9962,45 +9796,29 @@ }, "type": "object" }, - "EnvironmentLifecycleStatusView": { - "oneOf": [ - { - "enum": [ - "provisioning", - "booting", - "ready", - "closing", - "closed", - "failed", - "unknown" - ], - "type": "string" - }, - { - "const": "paused", - "description": "Execution frozen; wakes on next use.", - "type": "string" - }, - { - "const": "suspended", - "description": "Execution state saved to disk; wakes on next use.", - "type": "string" - }, - { - "const": "offline", - "description": "Powered off; provisioned environments wake on next use when the\nprovider supports power control.", - "type": "string" - } - ] - }, "EnvironmentListParams": { "properties": { - "bindingId": { + "after": { + "description": "Exclusive environment id cursor: a previous page's `nextAfter`.", "type": [ "string", "null" ] }, + "includeClosed": { + "description": "Also list closed environments; they are omitted by default.", + "type": "boolean" + }, + "limit": { + "description": "Most environments per page; omit for 100.", + "format": "uint32", + "maximum": 500, + "minimum": 1, + "type": [ + "integer", + "null" + ] + }, "metadata": { "additionalProperties": { "type": "string" @@ -10009,6 +9827,7 @@ "type": "object" }, "providerId": { + "description": "Only provisioned environments of this provider.", "type": [ "string", "null" @@ -10021,15 +9840,16 @@ "null" ] }, - "status": { + "state": { "anyOf": [ { - "$ref": "#/definitions/EnvironmentLifecycleStatusView" + "$ref": "#/definitions/EnvironmentStateView" }, { "type": "null" } - ] + ], + "description": "Only environments in this lifecycle state. `closed` lists closed\nenvironments without `includeClosed`." } }, "type": "object" @@ -10038,12 +9858,54 @@ "properties": { "environments": { "default": [], + "description": "Environments in environment id order.", "items": { "$ref": "#/definitions/EnvironmentView" }, "type": "array" + }, + "nextAfter": { + "description": "Pass as `after` to read the next page; absent on the last page.", + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, + "EnvironmentObservedTargetView": { + "description": "What the provider last reported about a provisioned environment's\ntarget. Display only: Lightspeed never converges it, and it may be up to\na few minutes old.", + "properties": { + "message": { + "type": [ + "string", + "null" + ] + }, + "observedAtMs": { + "format": "int64", + "type": "integer" + }, + "phase": { + "$ref": "#/definitions/EnvironmentTargetPhaseView" + }, + "power": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentPowerStateView" + }, + { + "type": "null" + } + ], + "description": "Present only while `phase` is `active`." } }, + "required": [ + "phase", + "observedAtMs" + ], "type": "object" }, "EnvironmentPowerPutParams": { @@ -10057,7 +9919,7 @@ "$ref": "#/definitions/EnvironmentPowerStateView" } ], - "description": "Desired steady power state. Must be one of the provider-reported\n`incarnation.powerStates`." + "description": "Power state to move the target to; one of its template's\n`powerStates`." } }, "required": [ @@ -10078,7 +9940,7 @@ "type": "object" }, "EnvironmentPowerStateView": { - "description": "Steady power state of a provisioned environment.", + "description": "Power state of a provisioned environment's target.", "oneOf": [ { "enum": [ @@ -10140,12 +10002,12 @@ }, "EnvironmentProviderBindingReadParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" } }, "required": [ - "bindingId" + "providerId" ], "type": "object" }, @@ -10160,49 +10022,27 @@ ], "type": "object" }, - "EnvironmentProviderBindingStatusView": { - "enum": [ - "enabled", - "disabled" - ], - "type": "string" - }, "EnvironmentProviderBindingView": { + "description": "A universe's binding to a deployment provider. Creates, templates and\ndata routes of the provider need it enabled.", "properties": { - "bindingId": { - "type": "string" - }, "createdAtMs": { "format": "int64", "type": "integer" }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "enabled": { + "type": "boolean" }, "providerId": { "type": "string" }, - "revision": { - "format": "uint64", - "minimum": 0, - "type": "integer" - }, - "status": { - "$ref": "#/definitions/EnvironmentProviderBindingStatusView" - }, - "updatedAtMs": { - "format": "int64", + "updatedAtMs": { + "format": "int64", "type": "integer" } }, "required": [ - "bindingId", "providerId", - "status", - "revision", + "enabled", "createdAtMs", "updatedAtMs" ], @@ -10253,7 +10093,12 @@ ] }, "identityMode": { - "$ref": "#/definitions/EnvironmentIdentityModeView" + "allOf": [ + { + "$ref": "#/definitions/EnvironmentIdentityModeView" + } + ], + "description": "Immutable once the key exists." }, "maxActiveEnvironments": { "description": "Non-closed environments the key may have at once; omit for unlimited.", @@ -10263,6 +10108,15 @@ "integer", "null" ] + }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: close environments whose daemon has been away\nlonger than this; omit to keep them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] } }, "required": [ @@ -10362,6 +10216,37 @@ ], "type": "object" }, + "EnvironmentRegistrationKeyRotateParams": { + "properties": { + "registrationKeyId": { + "type": "string" + } + }, + "required": [ + "registrationKeyId" + ], + "type": "object" + }, + "EnvironmentRegistrationKeyRotateResponse": { + "properties": { + "registrationKey": { + "$ref": "#/definitions/EnvironmentRegistrationKeyView" + }, + "secret": { + "allOf": [ + { + "$ref": "#/definitions/EnvironmentRegistrationSecretView" + } + ], + "description": "The new plaintext secret, returned only here." + } + }, + "required": [ + "registrationKey", + "secret" + ], + "type": "object" + }, "EnvironmentRegistrationKeyStatusView": { "enum": [ "active", @@ -10370,6 +10255,68 @@ ], "type": "string" }, + "EnvironmentRegistrationKeyUpdateParams": { + "description": "The complete editable policy of a registration key; an absent field is\nunset. The identity mode is immutable.", + "properties": { + "displayName": { + "description": "Group name for the environments this key admits.", + "type": "string" + }, + "ephemeralDisconnectGraceMs": { + "description": "Disconnect grace for ephemeral environments; omit for the default.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "expiresAtMs": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "maxActiveEnvironments": { + "description": "Non-closed environments the key may have at once; omit for unlimited.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: close environments whose daemon has been away\nlonger than this; omit to keep them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "registrationKeyId": { + "type": "string" + } + }, + "required": [ + "registrationKeyId", + "displayName" + ], + "type": "object" + }, + "EnvironmentRegistrationKeyUpdateResponse": { + "properties": { + "registrationKey": { + "$ref": "#/definitions/EnvironmentRegistrationKeyView" + } + }, + "required": [ + "registrationKey" + ], + "type": "object" + }, "EnvironmentRegistrationKeyView": { "properties": { "activeEnvironmentCount": { @@ -10403,7 +10350,7 @@ "$ref": "#/definitions/EnvironmentIdentityModeView" }, "keyPrefix": { - "description": "First characters of the secret, for identification only.", + "description": "First characters of the key's live secret, for identification only.", "type": "string" }, "lastRegisteredAtMs": { @@ -10421,6 +10368,15 @@ "null" ] }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: environments whose daemon has been away longer\nthan this are closed; absent keeps them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "registeredEnvironmentCount": { "description": "Environments ever admitted by this key, closed included.", "format": "uint64", @@ -10439,6 +10395,10 @@ }, "status": { "$ref": "#/definitions/EnvironmentRegistrationKeyStatusView" + }, + "updatedAtMs": { + "format": "int64", + "type": "integer" } }, "required": [ @@ -10450,7 +10410,8 @@ "status", "registeredEnvironmentCount", "activeEnvironmentCount", - "createdAtMs" + "createdAtMs", + "updatedAtMs" ], "type": "object" }, @@ -10479,13 +10440,50 @@ "EnvironmentSourceView": { "oneOf": [ { + "description": "A target a deployment provider created from a template. The provider\nowns its power, idle policy and ingress.", "properties": { - "bindingId": { - "type": "string" + "idle": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentIdlePolicyView" + }, + { + "type": "null" + } + ], + "description": "The create-time idle policy, when one replaced the template's." + }, + "observed": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentObservedTargetView" + }, + { + "type": "null" + } + ], + "description": "Absent until the provider first reported the target." }, "providerId": { "type": "string" }, + "publicEndpoint": { + "description": "The public endpoint the provider last reported.", + "type": [ + "string", + "null" + ] + }, + "targetId": { + "description": "The provider's handle for the target; absent until the provider\naccepted the create.", + "type": [ + "string", + "null" + ] + }, + "templateId": { + "type": "string" + }, "type": { "const": "provisioned", "type": "string" @@ -10494,14 +10492,15 @@ "required": [ "type", "providerId", - "bindingId" + "templateId" ], "type": "object" }, { "properties": { - "connection": { - "$ref": "#/definitions/EnvironmentConnectionView" + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL Lightspeed dials envd at.", + "type": "string" }, "type": { "const": "external", @@ -10510,18 +10509,37 @@ }, "required": [ "type", - "connection" + "endpoint" ], "type": "object" }, { "description": "An envd that dialed the gateway outbound and was admitted by a\nregistration key. The key is the environment's group; the daemon id\nis derived from the daemon's public key and is its identity.", "properties": { + "connected": { + "description": "The daemon's control connection is up.", + "type": "boolean" + }, + "connectionChangedAtMs": { + "description": "When the daemon last connected or disconnected.", + "format": "int64", + "type": "integer" + }, "daemonId": { "type": "string" }, "identityMode": { - "$ref": "#/definitions/EnvironmentIdentityModeView" + "allOf": [ + { + "$ref": "#/definitions/EnvironmentIdentityModeView" + } + ], + "description": "The admitting key's identity mode." + }, + "lastSeenAtMs": { + "description": "When the gateway last recorded the daemon's control connection:\nat connect, at disconnect, and at most every five minutes while\nconnected.", + "format": "int64", + "type": "integer" }, "registrationKeyId": { "type": "string" @@ -10535,15 +10553,98 @@ "type", "registrationKeyId", "daemonId", - "identityMode" + "identityMode", + "connected", + "connectionChangedAtMs", + "lastSeenAtMs" ], "type": "object" } ] }, + "EnvironmentStateView": { + "description": "The lifecycle Lightspeed owns. External and registered environments are\nonly ever `open` or `closed`.", + "oneOf": [ + { + "enum": [ + "open", + "closed" + ], + "type": "string" + }, + { + "const": "provisioning", + "description": "The provider is creating the target; the environment opens after\nLightspeed's first handshake with its envd.", + "type": "string" + }, + { + "const": "closing", + "description": "The provider is removing the target.", + "type": "string" + }, + { + "const": "failed", + "description": "Provisioning or the provider failed; `stateMessage` says why.", + "type": "string" + } + ] + }, + "EnvironmentStatusView": { + "description": "Display status derived from `state` and the latest observation; never\nstored. Tools that use a paused, suspended or stopped environment wake\nit.", + "oneOf": [ + { + "enum": [ + "provisioning", + "ready", + "closing", + "closed", + "failed" + ], + "type": "string" + }, + { + "const": "paused", + "description": "Execution frozen with RAM resident.", + "type": "string" + }, + { + "const": "suspended", + "description": "Execution state saved to disk.", + "type": "string" + }, + { + "const": "stopped", + "description": "Powered off with disk retained.", + "type": "string" + }, + { + "const": "unknown", + "description": "Open, but the provider's latest observation is inconclusive.", + "type": "string" + }, + { + "const": "away", + "description": "Registered, open, and its daemon is not connected.", + "type": "string" + } + ] + }, + "EnvironmentTargetPhaseView": { + "description": "Lifecycle phase of a provider target, as the provider reports it.", + "enum": [ + "provisioning", + "active", + "closing", + "closed", + "failed", + "unknown" + ], + "type": "string" + }, "EnvironmentTemplateListParams": { "properties": { - "bindingId": { + "providerId": { + "description": "Only this provider's templates; omit for every enabled binding.", "type": [ "string", "null" @@ -10568,7 +10669,7 @@ }, "EnvironmentTemplateReadParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" }, "templateId": { @@ -10576,7 +10677,7 @@ } }, "required": [ - "bindingId", + "providerId", "templateId" ], "type": "object" @@ -10592,11 +10693,38 @@ ], "type": "object" }, - "EnvironmentTemplateView": { + "EnvironmentTemplateResourcesView": { + "description": "A template's compute, in provider-neutral units. Each field is absent\nwhen the provider does not fix it.", "properties": { - "bindingId": { - "type": "string" + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "EnvironmentTemplateView": { + "properties": { "deprecated": { "type": "boolean" }, @@ -10609,99 +10737,177 @@ "displayName": { "type": "string" }, + "idle": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentIdlePolicyView" + }, + { + "type": "null" + } + ], + "description": "Idle policy targets get when `environments/create` names none." + }, + "metadata": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "powerStates": { + "description": "Power states `environments/power/put` can move targets of this\ntemplate to; empty without power control.", + "items": { + "$ref": "#/definitions/EnvironmentPowerStateView" + }, + "type": "array" + }, + "providerId": { + "type": "string" + }, + "publicIngress": { + "description": "`environments/ingress/put` can enable public ingress on targets of\nthis template.", + "type": "boolean" + }, + "resources": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentTemplateResourcesView" + }, + { + "type": "null" + } + ], + "description": "Compute targets of this template get; absent when the provider does\nnot fix it." + }, + "templateId": { + "type": "string" + } + }, + "required": [ + "templateId", + "providerId", + "displayName", + "publicIngress", + "deprecated", + "powerStates" + ], + "type": "object" + }, + "EnvironmentUpdateParams": { + "description": "Replace an environment's display name and caller metadata.", + "properties": { + "displayName": { + "description": "The new display name; omit to clear it.", + "type": [ + "string", + "null" + ] + }, + "environmentId": { + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" }, + "description": "The complete new caller metadata; omit or `{}` to clear it.", "type": "object" - }, - "providerId": { - "type": "string" - }, - "publicIngress": { - "type": "boolean" - }, - "templateId": { - "type": "string" } }, "required": [ - "templateId", - "providerId", - "bindingId", - "displayName", - "publicIngress", - "deprecated" + "environmentId" + ], + "type": "object" + }, + "EnvironmentUpdateResponse": { + "properties": { + "environment": { + "$ref": "#/definitions/EnvironmentView" + } + }, + "required": [ + "environment" ], "type": "object" }, "EnvironmentView": { "properties": { - "createdAtMs": { - "format": "int64", - "type": "integer" - }, - "desiredPower": { - "allOf": [ + "closeReason": { + "anyOf": [ { - "$ref": "#/definitions/EnvironmentPowerStateView" + "$ref": "#/definitions/EnvironmentCloseReasonView" + }, + { + "type": "null" } ], - "description": "Lightspeed-owned power intent; `status` is the observed state." + "description": "Present exactly while `state` is `closing` or `closed`." }, - "displayName": { + "closedAtMs": { + "format": "int64", "type": [ - "string", + "integer", "null" ] }, - "environmentId": { - "type": "string" + "createdAtMs": { + "format": "int64", + "type": "integer" }, - "idlePolicy": { + "daemon": { "anyOf": [ { - "$ref": "#/definitions/EnvironmentIdlePolicyView" + "$ref": "#/definitions/EnvironmentDaemonView" }, { "type": "null" } - ] - }, - "incarnation": { - "$ref": "#/definitions/EnvironmentIncarnationView" + ], + "description": "The envd build serving the environment; absent until known." }, - "lastSeenAtMs": { - "description": "Registered environments only: when the gateway last saw the daemon's\ncontrol connection.", - "format": "int64", + "displayName": { "type": [ - "integer", + "string", "null" ] }, + "environmentId": { + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" }, + "description": "Caller metadata.", "type": "object" }, - "publicEndpoint": { + "requestId": { + "description": "The caller's retry identity; absent for registered environments.", "type": [ "string", "null" ] }, - "publicIngressEnabled": { - "type": "boolean" - }, - "requestId": { - "type": "string" - }, "source": { "$ref": "#/definitions/EnvironmentSourceView" }, + "state": { + "$ref": "#/definitions/EnvironmentStateView" + }, + "stateChangedAtMs": { + "description": "When `state` last changed.", + "format": "int64", + "type": "integer" + }, + "stateMessage": { + "description": "Why the environment failed, or what provisioning or closing is\nwaiting on.", + "type": [ + "string", + "null" + ] + }, "status": { - "$ref": "#/definitions/EnvironmentLifecycleStatusView" + "$ref": "#/definitions/EnvironmentStatusView" }, "updatedAtMs": { "format": "int64", @@ -10710,12 +10916,10 @@ }, "required": [ "environmentId", - "requestId", "source", + "state", "status", - "desiredPower", - "incarnation", - "publicIngressEnabled", + "stateChangedAtMs", "createdAtMs", "updatedAtMs" ], @@ -12286,7 +12490,7 @@ }, { "const": "deployment/environment-providers", - "description": "Environment providers, their bindings and adoption.", + "description": "Environment providers and their universe bindings.", "type": "string" }, { @@ -15777,29 +15981,6 @@ ], "type": "object" }, - "SessionJobArtifactView": { - "properties": { - "kind": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "type": "object" - }, "SessionJobCancelEntryView": { "properties": { "error": { @@ -15862,6 +16043,49 @@ ], "type": "string" }, + "SessionJobExitView": { + "description": "How a job's root process ended.", + "oneOf": [ + { + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "type": { + "const": "code", + "type": "string" + } + }, + "required": [ + "type", + "code" + ], + "type": "object" + }, + { + "description": "Killed by this signal, named like `SIGSEGV`.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "signal": { + "type": "string" + }, + "type": { + "const": "signal", + "type": "string" + } + }, + "required": [ + "type", + "signal" + ], + "type": "object" + } + ] + }, "SessionJobHandleInput": { "properties": { "environmentId": { @@ -15897,7 +16121,8 @@ "dataBase64": { "type": "string" }, - "seq": { + "offset": { + "description": "Byte offset of the chunk's first byte in the job's combined output.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15907,12 +16132,50 @@ } }, "required": [ - "seq", + "offset", "stream", "dataBase64" ], "type": "object" }, + "SessionJobOutputGapReasonView": { + "oneOf": [ + { + "const": "overflow", + "description": "Dropped from the middle because the job's retained output exceeded\nits cap; the head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Released after it was delivered.", + "type": "string" + } + ] + }, + "SessionJobOutputGapView": { + "description": "Output bytes in `[offset, offset + omittedBytes)` that cannot be\nreturned.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/definitions/SessionJobOutputGapReasonView" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, "SessionJobOutputStreamView": { "enum": [ "stdout", @@ -15921,13 +16184,8 @@ "type": "string" }, "SessionJobReadEntryView": { + "description": "One job's summary and output page, or why it could not be read: an\nentry with `error` and no summary names a job the environment does not\nknow (never created, or garbage-collected).", "properties": { - "artifacts": { - "items": { - "$ref": "#/definitions/SessionJobArtifactView" - }, - "type": "array" - }, "error": { "type": [ "string", @@ -15945,12 +16203,27 @@ ] }, "outputChunks": { + "description": "Output in offset order.", "items": { "$ref": "#/definitions/SessionJobOutputChunkView" }, "type": "array" }, - "outputNextSeq": { + "outputEnd": { + "description": "Output bytes the job produced so far; final once it is terminal.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "outputGaps": { + "description": "Output ranges between the requested start and `outputNext` that\ncannot be returned.", + "items": { + "$ref": "#/definitions/SessionJobOutputGapView" + }, + "type": "array" + }, + "outputNext": { + "description": "Byte offset to continue reading from: pass it as `after`.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15967,7 +16240,8 @@ } }, "required": [ - "outputNextSeq" + "outputNext", + "outputEnd" ], "type": "object" }, @@ -16087,26 +16361,84 @@ "type": "object" }, "SessionJobStatusView": { - "enum": [ - "accepted", - "queued", - "running", - "succeeded", - "failed", - "cancelRequested", - "cancelled", - "timedOut", - "dependencyFailed", - "interrupted", - "lost" - ], - "type": "string" + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its timeout expired and it was stopped.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of a cancel.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The environment daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "SessionJobStoppedByView": { + "description": "Why Lightspeed stopped a running job.", + "oneOf": [ + { + "const": "timeout", + "description": "Its timeout expired: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "terminate", + "description": "A cancel without `force`: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "kill", + "description": "A cancel with `force`: killed at once.", + "type": "string" + } + ] }, "SessionJobSummaryView": { + "description": "What a job is and how it ended. Instants are the environment's wall\nclock.", "properties": { + "cancelRequestedAtMs": { + "description": "When a cancel reached the job while it ran.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "createdAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": "integer" }, "dependencies": { @@ -16115,13 +16447,26 @@ }, "type": "array" }, - "exitCode": { - "format": "int32", + "durationMs": { + "description": "Run time from start to finish, immune to clock jumps.", + "format": "uint64", + "minimum": 0, "type": [ "integer", "null" ] }, + "exit": { + "anyOf": [ + { + "$ref": "#/definitions/SessionJobExitView" + }, + { + "type": "null" + } + ], + "description": "How the job's root process ended, once it ran and exited." + }, "failure": { "type": [ "string", @@ -16129,8 +16474,7 @@ ] }, "finishedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16145,9 +16489,6 @@ "null" ] }, - "namespace": { - "type": "string" - }, "orphanedDescendants": { "description": "True when the job's root process exited while descendants it spawned\nwere still running; the host terminated them best-effort at job\ncompletion.", "type": "boolean" @@ -16159,16 +16500,18 @@ ] }, "queuedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" ] }, + "requestId": { + "description": "The start request that created the job; part of its identity.", + "type": "string" + }, "startedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16176,11 +16519,22 @@ }, "status": { "$ref": "#/definitions/SessionJobStatusView" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/definitions/SessionJobStoppedByView" + }, + { + "type": "null" + } + ], + "description": "Set when Lightspeed stopped the running job." } }, "required": [ - "namespace", "jobId", + "requestId", "status", "createdAtMs" ], diff --git a/crates/api/contract/methods.json b/crates/api/contract/methods.json index ad1d9c68d..3f3e262ba 100644 --- a/crates/api/contract/methods.json +++ b/crates/api/contract/methods.json @@ -779,7 +779,7 @@ "action": "configure_resource", "kind": "universe" }, - "description": "Records an idempotent provisioning intent against an enabled universe binding, attributed to the caller. The provider validates its provider-wide template and provisions through its backend asynchronously.", + "description": "Creates a provisioned environment, attributed to the caller, from a template of a provider enabled for the universe. Idempotent per requestId: same inputs, same environment; others conflict. Provisioning until the target's first envd handshake, then open; idle overrides the template default.", "group": "environments", "method": "environments/create", "params": { @@ -804,7 +804,7 @@ "action": "read", "kind": "universe" }, - "description": "Returns the durable universe resource, source binding, logical lifecycle state, and minimal current-incarnation identity.", + "description": "Returns one environment: its source, lifecycle state, derived display status, state message, close reason, daemon build, and the latest provider observation.", "group": "environments", "method": "environments/read", "params": { @@ -824,12 +824,37 @@ "summary": "Read an environment", "target": null }, + { + "access": { + "action": "configure_resource", + "kind": "universe" + }, + "description": "Replaces the display name and caller metadata of an environment of any kind. Refused for a closed environment.", + "group": "environments", + "method": "environments/update", + "params": { + "schema": { + "$ref": "#/definitions/EnvironmentUpdateParams" + }, + "type": "EnvironmentUpdateParams" + }, + "result": { + "schema": { + "$ref": "#/definitions/AgentApiOutcomeOfEnvironmentUpdateResponse" + }, + "type": "AgentApiOutcome" + }, + "role": "operator", + "scope": "universe", + "summary": "Update an environment", + "target": null + }, { "access": { "action": "read", "kind": "universe" }, - "description": "Lists the universe environments, optionally filtered by provider, binding, or logical lifecycle state.", + "description": "Lists the universe environments in id order, one page at a time, optionally filtered by provider, registration key, state, or metadata. Closed environments are omitted unless includeClosed is set or state is closed. Continue from nextAfter.", "group": "environments", "method": "environments/list", "params": { @@ -854,7 +879,7 @@ "action": "configure_resource", "kind": "universe" }, - "description": "Records an asynchronous idempotent close intent. Provider cleanup is resumed by lifecycle reconciliation; quota is released only after Closed.", + "description": "Closes an environment; idempotent. A provisioned environment moves to closing while Lightspeed asks its provider to remove the target, retrying until it is gone, then closed. External and registered environments close at once, and a connected registered daemon is told its environment closed.", "group": "environments", "method": "environments/close", "params": { @@ -879,7 +904,7 @@ "action": "configure_resource", "kind": "universe" }, - "description": "Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, attributed to the caller. Reachability is checked on demand.", + "description": "Creates an open environment backed by a Lightspeed-reachable envd endpoint, attributed to the caller. Idempotent per requestId. Reachability is checked on use.", "group": "environments", "method": "environments/external/create", "params": { @@ -904,7 +929,7 @@ "action": "configure_resource", "kind": "universe" }, - "description": "Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health.", + "description": "Synchronously ensures or removes public ingress of an open provisioned environment through its provider; source.publicEndpoint carries the reported endpoint. Rejected when the provider offers no ingress; an internal error, safe to retry, when the provider is unreachable or times out. The provider owns hostname, guest port, routing, TLS, and health.", "group": "environments", "method": "environments/ingress/put", "params": { @@ -929,7 +954,7 @@ "action": "configure_resource", "kind": "universe" }, - "description": "Records the desired power state (running, paused, suspended, or stopped) of a provisioned environment; the lifecycle reconciler converges the provider target asynchronously. Powered-down environments wake transparently on their next use. Rejected when the provider does not support the state.", + "description": "Synchronously moves an open provisioned environment's target to running, paused, suspended, or stopped via its provider and records the observation; the provider may also change power on idle. Rejected without provider power control or when the template lacks the state; an internal error, safe to retry, when the provider is unreachable or times out.", "group": "environments", "method": "environments/power/put", "params": { @@ -946,32 +971,7 @@ }, "role": "operator", "scope": "universe", - "summary": "Set environment power intent", - "target": null - }, - { - "access": { - "action": "configure_resource", - "kind": "universe" - }, - "description": "Replaces or clears the staged idle policy of a provisioned environment. The power reaper measures the daemon's idle duration against the pause/suspend/stop/close thresholds and escalates through the stages the provider supports.", - "group": "environments", - "method": "environments/idle-policy/put", - "params": { - "schema": { - "$ref": "#/definitions/EnvironmentIdlePolicyPutParams" - }, - "type": "EnvironmentIdlePolicyPutParams" - }, - "result": { - "schema": { - "$ref": "#/definitions/AgentApiOutcomeOfEnvironmentIdlePolicyPutResponse" - }, - "type": "AgentApiOutcome" - }, - "role": "operator", - "scope": "universe", - "summary": "Set environment idle policy", + "summary": "Set environment power", "target": null }, { @@ -979,7 +979,7 @@ "action": "read", "kind": "universe" }, - "description": "Lists this universe's revisioned routing and admission bindings to deployment-scoped physical providers.", + "description": "Lists this universe's bindings to deployment providers and whether each is enabled.", "group": "environments", "method": "environments/provider-bindings/list", "params": { @@ -1004,7 +1004,7 @@ "action": "read", "kind": "universe" }, - "description": "Returns one universe routing and admission binding. Provider-wide templates and physical resource, network, and ingress policy remain provider-owned.", + "description": "Returns this universe's binding to one provider. Templates and physical resource, network, and ingress policy remain provider-owned.", "group": "environments", "method": "environments/provider-bindings/read", "params": { @@ -1029,7 +1029,7 @@ "action": "read", "kind": "universe" }, - "description": "Reads immutable templates directly from the selected bound provider controller.", + "description": "Reads templates live from the controllers of the universe's enabled provider bindings, or of one provider: compute, power states, public ingress, and default idle policy.", "group": "environments", "method": "environments/templates/list", "params": { @@ -1054,7 +1054,7 @@ "action": "read", "kind": "universe" }, - "description": "Returns one immutable template version from the selected bound provider controller.", + "description": "Returns one template, read live from a bound provider's controller.", "group": "environments", "method": "environments/templates/read", "params": { @@ -1079,7 +1079,7 @@ "action": "use_resource", "kind": "universe" }, - "description": "Starts a dependency-aware job group on one environment instance, injecting the environment's configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance rather than a session. A powered-down environment is woken on use: the call fails with environment_not_ready while the wake is in progress; retry with backoff.", + "description": "Starts a dependency-aware job group on one environment instance, injecting its configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance, not a session. A powered-down environment wakes on its data route; the call fails with environment_not_ready while it wakes or reconnects; retry with backoff.", "group": "environments", "method": "environments/jobs/create", "params": { @@ -1104,7 +1104,7 @@ "action": "read", "kind": "universe" }, - "description": "Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling.", + "description": "Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status.", "group": "environments", "method": "environments/jobs/read", "params": { @@ -1154,7 +1154,7 @@ "action": "configure_resource", "kind": "universe" }, - "description": "Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode, active limit, disconnect grace, and expiry are the key's policy. Treat the secret like a cluster-join credential.", + "description": "Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode (immutable), active limit, disconnect grace, offline close-after, and expiry are the key's policy. Treat the secret like a cluster-join credential.", "group": "environments", "method": "environments/registration-keys/create", "params": { @@ -1224,6 +1224,56 @@ "summary": "List environment registration keys", "target": null }, + { + "access": { + "action": "configure_resource", + "kind": "universe" + }, + "description": "Replaces a key's editable policy: display name, active limit, disconnect grace, offline close-after, and expiry; an omitted field is unset. The identity mode is immutable. The active limit and expiry apply to later admissions; a changed disconnect grace or offline close-after applies at the next sweep to environments whose daemon is away.", + "group": "environments", + "method": "environments/registration-keys/update", + "params": { + "schema": { + "$ref": "#/definitions/EnvironmentRegistrationKeyUpdateParams" + }, + "type": "EnvironmentRegistrationKeyUpdateParams" + }, + "result": { + "schema": { + "$ref": "#/definitions/AgentApiOutcomeOfEnvironmentRegistrationKeyUpdateResponse" + }, + "type": "AgentApiOutcome" + }, + "role": "operator", + "scope": "universe", + "summary": "Update an environment registration key", + "target": null + }, + { + "access": { + "action": "configure_resource", + "kind": "universe" + }, + "description": "Replaces the key's secret and display prefix and retires the old secret at once. The key keeps its id, group, policy, and the environments it admitted; daemons already registered are unaffected because they never present the key again. Returns the new secret once. Refused for a revoked key.", + "group": "environments", + "method": "environments/registration-keys/rotate", + "params": { + "schema": { + "$ref": "#/definitions/EnvironmentRegistrationKeyRotateParams" + }, + "type": "EnvironmentRegistrationKeyRotateParams" + }, + "result": { + "schema": { + "$ref": "#/definitions/AgentApiOutcomeOfEnvironmentRegistrationKeyRotateResponse" + }, + "type": "AgentApiOutcome" + }, + "role": "operator", + "scope": "universe", + "summary": "Rotate an environment registration key", + "target": null + }, { "access": { "action": "configure_resource", @@ -3289,7 +3339,7 @@ "access": { "kind": "deployment" }, - "description": "Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed.", + "description": "Registers or replaces a deployment provider: controller endpoint, optional data route base, and optional bearer credential for controller and data routes (write-only: kept when omitted, removed by clearCredential, never returned). The controller is probed: a definitive refusal is invalid_request, unreachable is accepted. The provider never calls Lightspeed.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/put", "params": { @@ -3313,7 +3363,7 @@ "access": { "kind": "deployment" }, - "description": "Returns every deployment-registered deployment provider and its controller connection.", + "description": "Returns every deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/list", "params": { @@ -3337,7 +3387,7 @@ "access": { "kind": "deployment" }, - "description": "Returns one deployment-registered deployment provider and its controller connection.", + "description": "Returns one deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/read", "params": { @@ -3361,7 +3411,7 @@ "access": { "kind": "deployment" }, - "description": "Deletes a deployment provider only when no universe binding references it.", + "description": "Deletes a deployment provider only when no universe binding and no non-closed environment references it.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/delete", "params": { @@ -3385,7 +3435,7 @@ "access": { "kind": "deployment" }, - "description": "Creates or replaces one universe's complete revisioned routing and admission binding. A deployment provider may have at most one binding in a universe.", + "description": "Creates or replaces one universe's binding to a deployment provider. Only an enabled binding admits creates, template listing, and data routes for the universe.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/bindings/put", "params": { @@ -3409,7 +3459,7 @@ "access": { "kind": "deployment" }, - "description": "Deletes a universe provider binding only after every referencing environment has reached Closed.", + "description": "Deletes a universe's binding to a provider only when the universe has no non-closed environment of that provider.", "group": "deployment/environment-providers", "method": "deployment/environment-providers/bindings/delete", "params": { @@ -3429,30 +3479,6 @@ "summary": "Delete an environment provider binding", "target": null }, - { - "access": { - "kind": "deployment" - }, - "description": "Creates a universe environment by transferring an existing provider target into Lightspeed's managed lifecycle. The caller must explicitly accept ownership transfer.", - "group": "deployment/environment-providers", - "method": "deployment/environments/adopt", - "params": { - "schema": { - "$ref": "#/definitions/DeploymentEnvironmentAdoptParams" - }, - "type": "DeploymentEnvironmentAdoptParams" - }, - "result": { - "schema": { - "$ref": "#/definitions/AgentApiOutcomeOfDeploymentEnvironmentAdoptResponse" - }, - "type": "AgentApiOutcome" - }, - "role": null, - "scope": "deployment", - "summary": "Adopt a provider environment", - "target": null - }, { "access": { "kind": "deployment" diff --git a/crates/api/contract/openrpc.json b/crates/api/contract/openrpc.json index 36c83ce67..8065eaf39 100644 --- a/crates/api/contract/openrpc.json +++ b/crates/api/contract/openrpc.json @@ -139,7 +139,7 @@ }, { "const": "environment_not_ready", - "description": "The environment exists but is not reachable yet: it is still\nprovisioning/booting, or it was powered down and a wake (desired\npower `running`) has been initiated by this request. Distinct from\n`Rejected` so clients retry with backoff instead of failing —\npolling and automation callers lean on this for wake-on-use.", + "description": "The environment exists but is not reachable yet: it is still\nprovisioning, its provider is waking or booting it after a data\nroute dial, or its registered daemon is reconnecting. Distinct from\n`Rejected` so clients retry with backoff instead of failing —\npolling and automation callers lean on this for wake-on-use.", "type": "string" }, { @@ -1084,23 +1084,6 @@ ], "type": "object" }, - "AgentApiOutcomeOfDeploymentEnvironmentAdoptResponse": { - "properties": { - "notifications": { - "items": { - "$ref": "#/components/schemas/AgentNotification" - }, - "type": "array" - }, - "result": { - "$ref": "#/components/schemas/DeploymentEnvironmentAdoptResponse" - } - }, - "required": [ - "result" - ], - "type": "object" - }, "AgentApiOutcomeOfDeploymentEnvironmentProviderDeleteResponse": { "properties": { "notifications": { @@ -1390,23 +1373,6 @@ ], "type": "object" }, - "AgentApiOutcomeOfEnvironmentIdlePolicyPutResponse": { - "properties": { - "notifications": { - "items": { - "$ref": "#/components/schemas/AgentNotification" - }, - "type": "array" - }, - "result": { - "$ref": "#/components/schemas/EnvironmentIdlePolicyPutResponse" - } - }, - "required": [ - "result" - ], - "type": "object" - }, "AgentApiOutcomeOfEnvironmentIngressPutResponse": { "properties": { "notifications": { @@ -1628,6 +1594,40 @@ ], "type": "object" }, + "AgentApiOutcomeOfEnvironmentRegistrationKeyRotateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/components/schemas/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/components/schemas/EnvironmentRegistrationKeyRotateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, + "AgentApiOutcomeOfEnvironmentRegistrationKeyUpdateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/components/schemas/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/components/schemas/EnvironmentRegistrationKeyUpdateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, "AgentApiOutcomeOfEnvironmentTemplateListResponse": { "properties": { "notifications": { @@ -1662,6 +1662,23 @@ ], "type": "object" }, + "AgentApiOutcomeOfEnvironmentUpdateResponse": { + "properties": { + "notifications": { + "items": { + "$ref": "#/components/schemas/AgentNotification" + }, + "type": "array" + }, + "result": { + "$ref": "#/components/schemas/EnvironmentUpdateResponse" + } + }, + "required": [ + "result" + ], + "type": "object" + }, "AgentApiOutcomeOfInitializeResponse": { "properties": { "notifications": { @@ -8710,74 +8727,6 @@ ], "type": "object" }, - "DeploymentEnvironmentAdoptParams": { - "properties": { - "bindingId": { - "type": "string" - }, - "displayName": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "requestId": { - "description": "Stable caller-generated retry identity inside the destination universe.", - "type": "string" - }, - "sourceTarget": { - "description": "Provider-native source reference. The Incus provider accepts\n`/` or an instance name in the `default` project.", - "type": "string" - }, - "takeOwnership": { - "description": "Required acknowledgement that Lightspeed will move, reconfigure, and\ndelete the target as part of its ordinary managed lifecycle.", - "type": "boolean" - }, - "universeId": { - "type": "string" - } - }, - "required": [ - "universeId", - "requestId", - "bindingId", - "sourceTarget", - "takeOwnership" - ], - "type": "object" - }, - "DeploymentEnvironmentAdoptResponse": { - "properties": { - "environment": { - "$ref": "#/components/schemas/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, - "DeploymentEnvironmentProviderConnection": { - "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/components/schemas/DeploymentEnvironmentProviderTransport" - } - }, - "required": [ - "endpoint", - "transport" - ], - "type": "object" - }, "DeploymentEnvironmentProviderDeleteParams": { "properties": { "providerId": { @@ -8818,21 +8767,35 @@ "type": "object" }, "DeploymentEnvironmentProviderPutParams": { + "description": "Register or replace a provider. `credential` is the optional bearer\ntoken Lightspeed presents to the provider's controller and data routes:\nit is sealed on receipt and never returned by any method. `Debug` output\nredacts it; request logging must never echo these params.", "properties": { - "controllerConnection": { - "$ref": "#/components/schemas/DeploymentEnvironmentProviderConnection" + "clearCredential": { + "description": "Remove the stored credential. Excludes `credential`.", + "type": "boolean" }, - "displayName": { + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" + }, + "credential": { + "description": "A new bearer credential; omit to keep the stored one. A provider needs\nnone. Excludes `clearCredential`.", "type": [ "string", "null" ] }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "dataRouteBase": { + "description": "`ws://` or `wss://` base of the provider's data routes; omit to derive\nit from the controller endpoint. Required for a `unix://` controller,\nand must not be `ws://` for a `wss://` controller.", + "type": [ + "string", + "null" + ] + }, + "displayName": { + "type": [ + "string", + "null" + ] }, "providerId": { "type": "string" @@ -8840,7 +8803,7 @@ }, "required": [ "providerId", - "controllerConnection" + "controllerEndpoint" ], "type": "object" }, @@ -8877,94 +8840,39 @@ ], "type": "object" }, - "DeploymentEnvironmentProviderTransport": { - "oneOf": [ - { - "properties": { - "type": { - "const": "webSocket", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "http", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "stdio", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "type": { - "const": "ssh", - "type": "string" - } - }, - "required": [ - "type" - ], - "type": "object" - }, - { - "properties": { - "providerType": { - "type": "string" - }, - "type": { - "const": "provider", - "type": "string" - } - }, - "required": [ - "type", - "providerType" - ], - "type": "object" - } - ] - }, "DeploymentEnvironmentProviderView": { "properties": { - "controllerConnection": { - "$ref": "#/components/schemas/DeploymentEnvironmentProviderConnection" + "controllerEndpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of the provider controller.", + "type": "string" }, "createdAtMs": { "format": "int64", "type": "integer" }, + "dataRouteBase": { + "description": "Operator-configured `ws://` or `wss://` base of the provider's data\nroutes; absent derives it from the controller endpoint.", + "type": [ + "string", + "null" + ] + }, "displayName": { "type": [ "string", "null" ] }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "effectiveDataRouteBase": { + "description": "The data route base Lightspeed dials: `dataRouteBase`, or the\ncontroller endpoint's origin followed by `/routes`. Absent only when it\ncannot be derived.", + "type": [ + "string", + "null" + ] + }, + "hasCredential": { + "description": "A bearer credential is stored and presented to the controller and\ndata routes; without one no `Authorization` header is sent.", + "type": "boolean" }, "providerId": { "type": "string" @@ -8976,7 +8884,8 @@ }, "required": [ "providerId", - "controllerConnection", + "controllerEndpoint", + "hasCredential", "createdAtMs", "updatedAtMs" ], @@ -8984,7 +8893,7 @@ }, "DeploymentProviderBindingDeleteParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" }, "universeId": { @@ -8993,7 +8902,7 @@ }, "required": [ "universeId", - "bindingId" + "providerId" ], "type": "object" }, @@ -9009,39 +8918,23 @@ "type": "object" }, "DeploymentProviderBindingPutParams": { + "description": "Create or replace a universe's binding to a deployment provider.", "properties": { - "bindingId": { - "type": "string" - }, - "expectedRevision": { - "format": "uint64", - "minimum": 0, - "type": [ - "integer", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "enabled": { + "description": "Admits creates, template listing and data routes for the universe.", + "type": "boolean" }, "providerId": { "type": "string" }, - "status": { - "$ref": "#/components/schemas/EnvironmentProviderBindingStatusView" - }, "universeId": { "type": "string" } }, "required": [ "universeId", - "bindingId", "providerId", - "status" + "enabled" ], "type": "object" }, @@ -9328,77 +9221,61 @@ ], "type": "object" }, - "EnvironmentCloseResponse": { - "properties": { - "environment": { - "$ref": "#/components/schemas/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, - "EnvironmentConnectionTransportView": { + "EnvironmentCloseReasonView": { + "description": "Why an environment is closing or closed.", "oneOf": [ { - "enum": [ - "webSocket", - "http", - "stdio", - "ssh" - ], + "const": "requested", + "description": "`environments/close`.", "type": "string" }, { - "additionalProperties": false, - "properties": { - "provider": { - "properties": { - "provider_type": { - "type": "string" - } - }, - "required": [ - "provider_type" - ], - "type": "object" - } - }, - "required": [ - "provider" - ], - "type": "object" + "const": "keyRevoked", + "description": "Its registration key was revoked with `closeEnvironments`.", + "type": "string" + }, + { + "const": "ephemeralGrace", + "description": "Its ephemeral daemon stayed away longer than the key's disconnect\ngrace.", + "type": "string" + }, + { + "const": "departed", + "description": "Its ephemeral daemon said goodbye.", + "type": "string" + }, + { + "const": "offlineTimeout", + "description": "Its persistent daemon stayed away longer than the key's\n`offlineCloseAfterMs`.", + "type": "string" + }, + { + "const": "providerEnded", + "description": "The provider reported the target closed or gone.", + "type": "string" } ] }, - "EnvironmentConnectionView": { + "EnvironmentCloseResponse": { "properties": { - "endpoint": { - "type": "string" - }, - "transport": { - "$ref": "#/components/schemas/EnvironmentConnectionTransportView" + "environment": { + "$ref": "#/components/schemas/EnvironmentView" } }, "required": [ - "endpoint", - "transport" + "environment" ], "type": "object" }, "EnvironmentCreateParams": { "properties": { - "bindingId": { - "type": "string" - }, "displayName": { "type": [ "string", "null" ] }, - "idlePolicy": { + "idle": { "anyOf": [ { "$ref": "#/components/schemas/EnvironmentIdlePolicyView" @@ -9407,7 +9284,7 @@ "type": "null" } ], - "description": "Optional staged idle policy applied by the power reaper." + "description": "Idle policy passed to the provider in place of the template's `idle`.\nThe provider runs the idle clock; it cannot be changed after create." }, "metadata": { "additionalProperties": { @@ -9415,18 +9292,22 @@ }, "type": "object" }, + "providerId": { + "description": "The provider to create the environment with; the universe needs an\nenabled binding to it.", + "type": "string" + }, "requestId": { - "description": "Stable caller-generated retry identity, unique inside the universe.", + "description": "Stable caller-generated retry identity, unique inside the universe.\nRetrying with identical inputs returns the same environment.", "type": "string" }, "templateId": { - "description": "Immutable provider template-version identity.", + "description": "The provider template to create the environment from.", "type": "string" } }, "required": [ "requestId", - "bindingId", + "providerId", "templateId" ], "type": "object" @@ -9609,7 +9490,7 @@ "type": "string" }, "protocolVersion": { - "description": "Environment protocol version the gateway speaks. A daemon registers\nonly when it reports exactly this number.", + "description": "The envd protocol version (data plane and registration channel)\nthe gateway speaks. A daemon registers only when it reports exactly\nthis number.", "format": "uint32", "minimum": 0, "type": "integer" @@ -9633,22 +9514,44 @@ ], "type": "object" }, - "EnvironmentExternalCreateParams": { + "EnvironmentDaemonView": { + "description": "The envd build serving an environment, from its registration or\nLightspeed's open handshake.", "properties": { - "connection": { - "allOf": [ - { - "$ref": "#/components/schemas/EnvironmentConnectionView" - } - ], - "description": "Connection to an envd instance reachable from Lightspeed." + "gitSha": { + "type": [ + "string", + "null" + ] }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "version": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "protocolVersion" + ], + "type": "object" + }, + "EnvironmentExternalCreateParams": { + "properties": { "displayName": { "type": [ "string", "null" ] }, + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL of an envd instance reachable from\nLightspeed.", + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" @@ -9656,12 +9559,13 @@ "type": "object" }, "requestId": { + "description": "Stable caller-generated retry identity, unique inside the universe.", "type": "string" } }, "required": [ "requestId", - "connection" + "endpoint" ], "type": "object" }, @@ -9677,55 +9581,23 @@ "type": "object" }, "EnvironmentIdentityModeView": { - "description": "What Lightspeed does with a registered environment while its daemon is\ndisconnected. Registration-key policy, copied onto each environment.", + "description": "What happens to a registered environment while its daemon is\ndisconnected. Policy of the registration key that admitted it.", "oneOf": [ { "const": "persistent", - "description": "Stays offline until explicitly closed.", + "description": "Stays open: tools wait up to two minutes for the daemon to reconnect.\nClosed explicitly, or after the key's `offlineCloseAfterMs`.", "type": "string" }, { "const": "ephemeral", - "description": "Closed once the daemon has been away longer than the key's\ndisconnect grace.", + "description": "Unusable while disconnected; closed once the daemon has been away\nlonger than the key's disconnect grace, or at once when it says\ngoodbye.", "type": "string" } ] }, - "EnvironmentIdlePolicyPutParams": { - "properties": { - "environmentId": { - "type": "string" - }, - "idlePolicy": { - "anyOf": [ - { - "$ref": "#/components/schemas/EnvironmentIdlePolicyView" - }, - { - "type": "null" - } - ], - "description": "The complete new policy; omit to clear it." - } - }, - "required": [ - "environmentId" - ], - "type": "object" - }, - "EnvironmentIdlePolicyPutResponse": { - "properties": { - "environment": { - "$ref": "#/components/schemas/EnvironmentView" - } - }, - "required": [ - "environment" - ], - "type": "object" - }, "EnvironmentIdlePolicyView": { - "description": "Staged idle policy. Thresholds are milliseconds of daemon-reported idle\ntime and must be non-decreasing in the order pause, suspend, stop, close.\nStages whose power state the provider does not support are skipped.", + "additionalProperties": false, + "description": "Staged idle policy the provider applies. Thresholds are milliseconds of\nenvd-reported idle time (running processes, jobs and leftover process\ngroups hold it at zero) and must be non-decreasing in the order pause,\nsuspend, stop, close. Stages whose power state the template does not\nsupport are skipped.", "properties": { "closeAfterMs": { "format": "uint64", @@ -9762,55 +9634,10 @@ }, "type": "object" }, - "EnvironmentIncarnationView": { - "properties": { - "createdAtMs": { - "format": "int64", - "type": "integer" - }, - "incarnationId": { - "type": "string" - }, - "powerStates": { - "description": "Power states the provider reported for this target; empty until\nobserved or when the provider offers no power control.", - "items": { - "$ref": "#/components/schemas/EnvironmentPowerStateView" - }, - "type": "array" - }, - "providerTargetId": { - "type": [ - "string", - "null" - ] - }, - "provisionRequestId": { - "type": [ - "string", - "null" - ] - }, - "templateId": { - "type": [ - "string", - "null" - ] - }, - "updatedAtMs": { - "format": "int64", - "type": "integer" - } - }, - "required": [ - "incarnationId", - "createdAtMs", - "updatedAtMs" - ], - "type": "object" - }, "EnvironmentIngressPutParams": { "properties": { "enabled": { + "description": "Ensure (`true`) or remove (`false`) the provider's public ingress.", "type": "boolean" }, "environmentId": { @@ -9918,7 +9745,8 @@ }, "EnvironmentJobReadParams": { "properties": { - "afterSeq": { + "after": { + "description": "Byte offset to read each job's output from: a previous entry's\n`outputNext`. Omitted with `tailBytes` omitted reads from the start.", "format": "uint64", "minimum": 0, "type": [ @@ -9926,18 +9754,24 @@ "null" ] }, - "includeArtifacts": { - "default": false, - "type": "boolean" - }, "jobs": { "items": { "$ref": "#/components/schemas/SessionJobHandleInput" }, "type": "array" }, - "outputBytes": { - "format": "uint", + "maxBytes": { + "description": "Most output bytes per job; omit for the server's default page.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "tailBytes": { + "description": "Read each job's last `tailBytes` of output instead; excludes `after`.", + "format": "uint64", "minimum": 0, "type": [ "integer", @@ -9962,45 +9796,29 @@ }, "type": "object" }, - "EnvironmentLifecycleStatusView": { - "oneOf": [ - { - "enum": [ - "provisioning", - "booting", - "ready", - "closing", - "closed", - "failed", - "unknown" - ], - "type": "string" - }, - { - "const": "paused", - "description": "Execution frozen; wakes on next use.", - "type": "string" - }, - { - "const": "suspended", - "description": "Execution state saved to disk; wakes on next use.", - "type": "string" - }, - { - "const": "offline", - "description": "Powered off; provisioned environments wake on next use when the\nprovider supports power control.", - "type": "string" - } - ] - }, "EnvironmentListParams": { "properties": { - "bindingId": { + "after": { + "description": "Exclusive environment id cursor: a previous page's `nextAfter`.", "type": [ "string", "null" ] }, + "includeClosed": { + "description": "Also list closed environments; they are omitted by default.", + "type": "boolean" + }, + "limit": { + "description": "Most environments per page; omit for 100.", + "format": "uint32", + "maximum": 500, + "minimum": 1, + "type": [ + "integer", + "null" + ] + }, "metadata": { "additionalProperties": { "type": "string" @@ -10009,6 +9827,7 @@ "type": "object" }, "providerId": { + "description": "Only provisioned environments of this provider.", "type": [ "string", "null" @@ -10021,15 +9840,16 @@ "null" ] }, - "status": { + "state": { "anyOf": [ { - "$ref": "#/components/schemas/EnvironmentLifecycleStatusView" + "$ref": "#/components/schemas/EnvironmentStateView" }, { "type": "null" } - ] + ], + "description": "Only environments in this lifecycle state. `closed` lists closed\nenvironments without `includeClosed`." } }, "type": "object" @@ -10038,14 +9858,56 @@ "properties": { "environments": { "default": [], + "description": "Environments in environment id order.", "items": { "$ref": "#/components/schemas/EnvironmentView" }, "type": "array" + }, + "nextAfter": { + "description": "Pass as `after` to read the next page; absent on the last page.", + "type": [ + "string", + "null" + ] } }, "type": "object" }, + "EnvironmentObservedTargetView": { + "description": "What the provider last reported about a provisioned environment's\ntarget. Display only: Lightspeed never converges it, and it may be up to\na few minutes old.", + "properties": { + "message": { + "type": [ + "string", + "null" + ] + }, + "observedAtMs": { + "format": "int64", + "type": "integer" + }, + "phase": { + "$ref": "#/components/schemas/EnvironmentTargetPhaseView" + }, + "power": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentPowerStateView" + }, + { + "type": "null" + } + ], + "description": "Present only while `phase` is `active`." + } + }, + "required": [ + "phase", + "observedAtMs" + ], + "type": "object" + }, "EnvironmentPowerPutParams": { "properties": { "environmentId": { @@ -10057,7 +9919,7 @@ "$ref": "#/components/schemas/EnvironmentPowerStateView" } ], - "description": "Desired steady power state. Must be one of the provider-reported\n`incarnation.powerStates`." + "description": "Power state to move the target to; one of its template's\n`powerStates`." } }, "required": [ @@ -10078,7 +9940,7 @@ "type": "object" }, "EnvironmentPowerStateView": { - "description": "Steady power state of a provisioned environment.", + "description": "Power state of a provisioned environment's target.", "oneOf": [ { "enum": [ @@ -10140,12 +10002,12 @@ }, "EnvironmentProviderBindingReadParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" } }, "required": [ - "bindingId" + "providerId" ], "type": "object" }, @@ -10160,49 +10022,27 @@ ], "type": "object" }, - "EnvironmentProviderBindingStatusView": { - "enum": [ - "enabled", - "disabled" - ], - "type": "string" - }, "EnvironmentProviderBindingView": { + "description": "A universe's binding to a deployment provider. Creates, templates and\ndata routes of the provider need it enabled.", "properties": { - "bindingId": { - "type": "string" - }, "createdAtMs": { "format": "int64", "type": "integer" }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "enabled": { + "type": "boolean" }, "providerId": { "type": "string" }, - "revision": { - "format": "uint64", - "minimum": 0, - "type": "integer" - }, - "status": { - "$ref": "#/components/schemas/EnvironmentProviderBindingStatusView" - }, "updatedAtMs": { "format": "int64", "type": "integer" } }, "required": [ - "bindingId", "providerId", - "status", - "revision", + "enabled", "createdAtMs", "updatedAtMs" ], @@ -10253,7 +10093,12 @@ ] }, "identityMode": { - "$ref": "#/components/schemas/EnvironmentIdentityModeView" + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentIdentityModeView" + } + ], + "description": "Immutable once the key exists." }, "maxActiveEnvironments": { "description": "Non-closed environments the key may have at once; omit for unlimited.", @@ -10263,6 +10108,15 @@ "integer", "null" ] + }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: close environments whose daemon has been away\nlonger than this; omit to keep them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] } }, "required": [ @@ -10362,6 +10216,37 @@ ], "type": "object" }, + "EnvironmentRegistrationKeyRotateParams": { + "properties": { + "registrationKeyId": { + "type": "string" + } + }, + "required": [ + "registrationKeyId" + ], + "type": "object" + }, + "EnvironmentRegistrationKeyRotateResponse": { + "properties": { + "registrationKey": { + "$ref": "#/components/schemas/EnvironmentRegistrationKeyView" + }, + "secret": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentRegistrationSecretView" + } + ], + "description": "The new plaintext secret, returned only here." + } + }, + "required": [ + "registrationKey", + "secret" + ], + "type": "object" + }, "EnvironmentRegistrationKeyStatusView": { "enum": [ "active", @@ -10370,6 +10255,68 @@ ], "type": "string" }, + "EnvironmentRegistrationKeyUpdateParams": { + "description": "The complete editable policy of a registration key; an absent field is\nunset. The identity mode is immutable.", + "properties": { + "displayName": { + "description": "Group name for the environments this key admits.", + "type": "string" + }, + "ephemeralDisconnectGraceMs": { + "description": "Disconnect grace for ephemeral environments; omit for the default.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "expiresAtMs": { + "format": "int64", + "type": [ + "integer", + "null" + ] + }, + "maxActiveEnvironments": { + "description": "Non-closed environments the key may have at once; omit for unlimited.", + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: close environments whose daemon has been away\nlonger than this; omit to keep them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "registrationKeyId": { + "type": "string" + } + }, + "required": [ + "registrationKeyId", + "displayName" + ], + "type": "object" + }, + "EnvironmentRegistrationKeyUpdateResponse": { + "properties": { + "registrationKey": { + "$ref": "#/components/schemas/EnvironmentRegistrationKeyView" + } + }, + "required": [ + "registrationKey" + ], + "type": "object" + }, "EnvironmentRegistrationKeyView": { "properties": { "activeEnvironmentCount": { @@ -10403,7 +10350,7 @@ "$ref": "#/components/schemas/EnvironmentIdentityModeView" }, "keyPrefix": { - "description": "First characters of the secret, for identification only.", + "description": "First characters of the key's live secret, for identification only.", "type": "string" }, "lastRegisteredAtMs": { @@ -10421,6 +10368,15 @@ "null" ] }, + "offlineCloseAfterMs": { + "description": "Persistent keys only: environments whose daemon has been away longer\nthan this are closed; absent keeps them open.", + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, "registeredEnvironmentCount": { "description": "Environments ever admitted by this key, closed included.", "format": "uint64", @@ -10439,6 +10395,10 @@ }, "status": { "$ref": "#/components/schemas/EnvironmentRegistrationKeyStatusView" + }, + "updatedAtMs": { + "format": "int64", + "type": "integer" } }, "required": [ @@ -10450,7 +10410,8 @@ "status", "registeredEnvironmentCount", "activeEnvironmentCount", - "createdAtMs" + "createdAtMs", + "updatedAtMs" ], "type": "object" }, @@ -10479,13 +10440,50 @@ "EnvironmentSourceView": { "oneOf": [ { + "description": "A target a deployment provider created from a template. The provider\nowns its power, idle policy and ingress.", "properties": { - "bindingId": { - "type": "string" + "idle": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentIdlePolicyView" + }, + { + "type": "null" + } + ], + "description": "The create-time idle policy, when one replaced the template's." + }, + "observed": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentObservedTargetView" + }, + { + "type": "null" + } + ], + "description": "Absent until the provider first reported the target." }, "providerId": { "type": "string" }, + "publicEndpoint": { + "description": "The public endpoint the provider last reported.", + "type": [ + "string", + "null" + ] + }, + "targetId": { + "description": "The provider's handle for the target; absent until the provider\naccepted the create.", + "type": [ + "string", + "null" + ] + }, + "templateId": { + "type": "string" + }, "type": { "const": "provisioned", "type": "string" @@ -10494,14 +10492,15 @@ "required": [ "type", "providerId", - "bindingId" + "templateId" ], "type": "object" }, { "properties": { - "connection": { - "$ref": "#/components/schemas/EnvironmentConnectionView" + "endpoint": { + "description": "`ws://`, `wss://` or `unix://` URL Lightspeed dials envd at.", + "type": "string" }, "type": { "const": "external", @@ -10510,18 +10509,37 @@ }, "required": [ "type", - "connection" + "endpoint" ], "type": "object" }, { "description": "An envd that dialed the gateway outbound and was admitted by a\nregistration key. The key is the environment's group; the daemon id\nis derived from the daemon's public key and is its identity.", "properties": { + "connected": { + "description": "The daemon's control connection is up.", + "type": "boolean" + }, + "connectionChangedAtMs": { + "description": "When the daemon last connected or disconnected.", + "format": "int64", + "type": "integer" + }, "daemonId": { "type": "string" }, "identityMode": { - "$ref": "#/components/schemas/EnvironmentIdentityModeView" + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentIdentityModeView" + } + ], + "description": "The admitting key's identity mode." + }, + "lastSeenAtMs": { + "description": "When the gateway last recorded the daemon's control connection:\nat connect, at disconnect, and at most every five minutes while\nconnected.", + "format": "int64", + "type": "integer" }, "registrationKeyId": { "type": "string" @@ -10535,15 +10553,98 @@ "type", "registrationKeyId", "daemonId", - "identityMode" + "identityMode", + "connected", + "connectionChangedAtMs", + "lastSeenAtMs" ], "type": "object" } ] }, + "EnvironmentStateView": { + "description": "The lifecycle Lightspeed owns. External and registered environments are\nonly ever `open` or `closed`.", + "oneOf": [ + { + "enum": [ + "open", + "closed" + ], + "type": "string" + }, + { + "const": "provisioning", + "description": "The provider is creating the target; the environment opens after\nLightspeed's first handshake with its envd.", + "type": "string" + }, + { + "const": "closing", + "description": "The provider is removing the target.", + "type": "string" + }, + { + "const": "failed", + "description": "Provisioning or the provider failed; `stateMessage` says why.", + "type": "string" + } + ] + }, + "EnvironmentStatusView": { + "description": "Display status derived from `state` and the latest observation; never\nstored. Tools that use a paused, suspended or stopped environment wake\nit.", + "oneOf": [ + { + "enum": [ + "provisioning", + "ready", + "closing", + "closed", + "failed" + ], + "type": "string" + }, + { + "const": "paused", + "description": "Execution frozen with RAM resident.", + "type": "string" + }, + { + "const": "suspended", + "description": "Execution state saved to disk.", + "type": "string" + }, + { + "const": "stopped", + "description": "Powered off with disk retained.", + "type": "string" + }, + { + "const": "unknown", + "description": "Open, but the provider's latest observation is inconclusive.", + "type": "string" + }, + { + "const": "away", + "description": "Registered, open, and its daemon is not connected.", + "type": "string" + } + ] + }, + "EnvironmentTargetPhaseView": { + "description": "Lifecycle phase of a provider target, as the provider reports it.", + "enum": [ + "provisioning", + "active", + "closing", + "closed", + "failed", + "unknown" + ], + "type": "string" + }, "EnvironmentTemplateListParams": { "properties": { - "bindingId": { + "providerId": { + "description": "Only this provider's templates; omit for every enabled binding.", "type": [ "string", "null" @@ -10568,7 +10669,7 @@ }, "EnvironmentTemplateReadParams": { "properties": { - "bindingId": { + "providerId": { "type": "string" }, "templateId": { @@ -10576,7 +10677,7 @@ } }, "required": [ - "bindingId", + "providerId", "templateId" ], "type": "object" @@ -10592,11 +10693,38 @@ ], "type": "object" }, - "EnvironmentTemplateView": { + "EnvironmentTemplateResourcesView": { + "description": "A template's compute, in provider-neutral units. Each field is absent\nwhen the provider does not fix it.", "properties": { - "bindingId": { - "type": "string" + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": [ + "integer", + "null" + ] + } + }, + "type": "object" + }, + "EnvironmentTemplateView": { + "properties": { "deprecated": { "type": "boolean" }, @@ -10609,18 +10737,48 @@ "displayName": { "type": "string" }, + "idle": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentIdlePolicyView" + }, + { + "type": "null" + } + ], + "description": "Idle policy targets get when `environments/create` names none." + }, "metadata": { "additionalProperties": { "type": "string" }, "type": "object" }, + "powerStates": { + "description": "Power states `environments/power/put` can move targets of this\ntemplate to; empty without power control.", + "items": { + "$ref": "#/components/schemas/EnvironmentPowerStateView" + }, + "type": "array" + }, "providerId": { "type": "string" }, "publicIngress": { + "description": "`environments/ingress/put` can enable public ingress on targets of\nthis template.", "type": "boolean" }, + "resources": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentTemplateResourcesView" + }, + { + "type": "null" + } + ], + "description": "Compute targets of this template get; absent when the provider does\nnot fix it." + }, "templateId": { "type": "string" } @@ -10628,28 +10786,18 @@ "required": [ "templateId", "providerId", - "bindingId", "displayName", "publicIngress", - "deprecated" + "deprecated", + "powerStates" ], "type": "object" }, - "EnvironmentView": { + "EnvironmentUpdateParams": { + "description": "Replace an environment's display name and caller metadata.", "properties": { - "createdAtMs": { - "format": "int64", - "type": "integer" - }, - "desiredPower": { - "allOf": [ - { - "$ref": "#/components/schemas/EnvironmentPowerStateView" - } - ], - "description": "Lightspeed-owned power intent; `status` is the observed state." - }, "displayName": { + "description": "The new display name; omit to clear it.", "type": [ "string", "null" @@ -10658,50 +10806,108 @@ "environmentId": { "type": "string" }, - "idlePolicy": { + "metadata": { + "additionalProperties": { + "type": "string" + }, + "description": "The complete new caller metadata; omit or `{}` to clear it.", + "type": "object" + } + }, + "required": [ + "environmentId" + ], + "type": "object" + }, + "EnvironmentUpdateResponse": { + "properties": { + "environment": { + "$ref": "#/components/schemas/EnvironmentView" + } + }, + "required": [ + "environment" + ], + "type": "object" + }, + "EnvironmentView": { + "properties": { + "closeReason": { "anyOf": [ { - "$ref": "#/components/schemas/EnvironmentIdlePolicyView" + "$ref": "#/components/schemas/EnvironmentCloseReasonView" }, { "type": "null" } - ] - }, - "incarnation": { - "$ref": "#/components/schemas/EnvironmentIncarnationView" + ], + "description": "Present exactly while `state` is `closing` or `closed`." }, - "lastSeenAtMs": { - "description": "Registered environments only: when the gateway last saw the daemon's\ncontrol connection.", + "closedAtMs": { "format": "int64", "type": [ "integer", "null" ] }, + "createdAtMs": { + "format": "int64", + "type": "integer" + }, + "daemon": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentDaemonView" + }, + { + "type": "null" + } + ], + "description": "The envd build serving the environment; absent until known." + }, + "displayName": { + "type": [ + "string", + "null" + ] + }, + "environmentId": { + "type": "string" + }, "metadata": { "additionalProperties": { "type": "string" }, + "description": "Caller metadata.", "type": "object" }, - "publicEndpoint": { + "requestId": { + "description": "The caller's retry identity; absent for registered environments.", "type": [ "string", "null" ] }, - "publicIngressEnabled": { - "type": "boolean" - }, - "requestId": { - "type": "string" - }, "source": { "$ref": "#/components/schemas/EnvironmentSourceView" }, + "state": { + "$ref": "#/components/schemas/EnvironmentStateView" + }, + "stateChangedAtMs": { + "description": "When `state` last changed.", + "format": "int64", + "type": "integer" + }, + "stateMessage": { + "description": "Why the environment failed, or what provisioning or closing is\nwaiting on.", + "type": [ + "string", + "null" + ] + }, "status": { - "$ref": "#/components/schemas/EnvironmentLifecycleStatusView" + "$ref": "#/components/schemas/EnvironmentStatusView" }, "updatedAtMs": { "format": "int64", @@ -10710,12 +10916,10 @@ }, "required": [ "environmentId", - "requestId", "source", + "state", "status", - "desiredPower", - "incarnation", - "publicIngressEnabled", + "stateChangedAtMs", "createdAtMs", "updatedAtMs" ], @@ -12286,7 +12490,7 @@ }, { "const": "deployment/environment-providers", - "description": "Environment providers, their bindings and adoption.", + "description": "Environment providers and their universe bindings.", "type": "string" }, { @@ -15777,29 +15981,6 @@ ], "type": "object" }, - "SessionJobArtifactView": { - "properties": { - "kind": { - "type": [ - "string", - "null" - ] - }, - "metadata": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "path": { - "type": "string" - } - }, - "required": [ - "path" - ], - "type": "object" - }, "SessionJobCancelEntryView": { "properties": { "error": { @@ -15852,15 +16033,58 @@ "null" ] } - }, - "type": "object" - }, - "SessionJobDependencyPolicyView": { - "enum": [ - "allSucceeded", - "allTerminal" - ], - "type": "string" + }, + "type": "object" + }, + "SessionJobDependencyPolicyView": { + "enum": [ + "allSucceeded", + "allTerminal" + ], + "type": "string" + }, + "SessionJobExitView": { + "description": "How a job's root process ended.", + "oneOf": [ + { + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "type": { + "const": "code", + "type": "string" + } + }, + "required": [ + "type", + "code" + ], + "type": "object" + }, + { + "description": "Killed by this signal, named like `SIGSEGV`.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "signal": { + "type": "string" + }, + "type": { + "const": "signal", + "type": "string" + } + }, + "required": [ + "type", + "signal" + ], + "type": "object" + } + ] }, "SessionJobHandleInput": { "properties": { @@ -15897,7 +16121,8 @@ "dataBase64": { "type": "string" }, - "seq": { + "offset": { + "description": "Byte offset of the chunk's first byte in the job's combined output.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15907,12 +16132,50 @@ } }, "required": [ - "seq", + "offset", "stream", "dataBase64" ], "type": "object" }, + "SessionJobOutputGapReasonView": { + "oneOf": [ + { + "const": "overflow", + "description": "Dropped from the middle because the job's retained output exceeded\nits cap; the head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Released after it was delivered.", + "type": "string" + } + ] + }, + "SessionJobOutputGapView": { + "description": "Output bytes in `[offset, offset + omittedBytes)` that cannot be\nreturned.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/components/schemas/SessionJobOutputGapReasonView" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, "SessionJobOutputStreamView": { "enum": [ "stdout", @@ -15921,13 +16184,8 @@ "type": "string" }, "SessionJobReadEntryView": { + "description": "One job's summary and output page, or why it could not be read: an\nentry with `error` and no summary names a job the environment does not\nknow (never created, or garbage-collected).", "properties": { - "artifacts": { - "items": { - "$ref": "#/components/schemas/SessionJobArtifactView" - }, - "type": "array" - }, "error": { "type": [ "string", @@ -15945,12 +16203,27 @@ ] }, "outputChunks": { + "description": "Output in offset order.", "items": { "$ref": "#/components/schemas/SessionJobOutputChunkView" }, "type": "array" }, - "outputNextSeq": { + "outputEnd": { + "description": "Output bytes the job produced so far; final once it is terminal.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "outputGaps": { + "description": "Output ranges between the requested start and `outputNext` that\ncannot be returned.", + "items": { + "$ref": "#/components/schemas/SessionJobOutputGapView" + }, + "type": "array" + }, + "outputNext": { + "description": "Byte offset to continue reading from: pass it as `after`.", "format": "uint64", "minimum": 0, "type": "integer" @@ -15967,7 +16240,8 @@ } }, "required": [ - "outputNextSeq" + "outputNext", + "outputEnd" ], "type": "object" }, @@ -16087,26 +16361,84 @@ "type": "object" }, "SessionJobStatusView": { - "enum": [ - "accepted", - "queued", - "running", - "succeeded", - "failed", - "cancelRequested", - "cancelled", - "timedOut", - "dependencyFailed", - "interrupted", - "lost" - ], - "type": "string" + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its timeout expired and it was stopped.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of a cancel.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The environment daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "SessionJobStoppedByView": { + "description": "Why Lightspeed stopped a running job.", + "oneOf": [ + { + "const": "timeout", + "description": "Its timeout expired: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "terminate", + "description": "A cancel without `force`: terminate, a grace period, then kill.", + "type": "string" + }, + { + "const": "kill", + "description": "A cancel with `force`: killed at once.", + "type": "string" + } + ] }, "SessionJobSummaryView": { + "description": "What a job is and how it ended. Instants are the environment's wall\nclock.", "properties": { + "cancelRequestedAtMs": { + "description": "When a cancel reached the job while it ran.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "createdAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": "integer" }, "dependencies": { @@ -16115,13 +16447,26 @@ }, "type": "array" }, - "exitCode": { - "format": "int32", + "durationMs": { + "description": "Run time from start to finish, immune to clock jumps.", + "format": "uint64", + "minimum": 0, "type": [ "integer", "null" ] }, + "exit": { + "anyOf": [ + { + "$ref": "#/components/schemas/SessionJobExitView" + }, + { + "type": "null" + } + ], + "description": "How the job's root process ended, once it ran and exited." + }, "failure": { "type": [ "string", @@ -16129,8 +16474,7 @@ ] }, "finishedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16145,9 +16489,6 @@ "null" ] }, - "namespace": { - "type": "string" - }, "orphanedDescendants": { "description": "True when the job's root process exited while descendants it spawned\nwere still running; the host terminated them best-effort at job\ncompletion.", "type": "boolean" @@ -16159,16 +16500,18 @@ ] }, "queuedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" ] }, + "requestId": { + "description": "The start request that created the job; part of its identity.", + "type": "string" + }, "startedAtMs": { - "format": "uint64", - "minimum": 0, + "format": "int64", "type": [ "integer", "null" @@ -16176,11 +16519,22 @@ }, "status": { "$ref": "#/components/schemas/SessionJobStatusView" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/components/schemas/SessionJobStoppedByView" + }, + { + "type": "null" + } + ], + "description": "Set when Lightspeed stopped the running job." } }, "required": [ - "namespace", "jobId", + "requestId", "status", "createdAtMs" ], @@ -19765,7 +20119,7 @@ "x-lightspeed-target": null }, { - "description": "Records an idempotent provisioning intent against an enabled universe binding, attributed to the caller. The provider validates its provider-wide template and provisions through its backend asynchronously.", + "description": "Creates a provisioned environment, attributed to the caller, from a template of a provider enabled for the universe. Idempotent per requestId: same inputs, same environment; others conflict. Provisioning until the target's first envd handshake, then open; idle overrides the template default.", "name": "environments/create", "paramStructure": "by-name", "params": [ @@ -19793,7 +20147,7 @@ "x-lightspeed-target": null }, { - "description": "Returns the durable universe resource, source binding, logical lifecycle state, and minimal current-incarnation identity.", + "description": "Returns one environment: its source, lifecycle state, derived display status, state message, close reason, daemon build, and the latest provider observation.", "name": "environments/read", "paramStructure": "by-name", "params": [ @@ -19821,81 +20175,81 @@ "x-lightspeed-target": null }, { - "description": "Lists the universe environments, optionally filtered by provider, binding, or logical lifecycle state.", - "name": "environments/list", + "description": "Replaces the display name and caller metadata of an environment of any kind. Refused for a closed environment.", + "name": "environments/update", "paramStructure": "by-name", "params": [ { "name": "params", "required": true, "schema": { - "$ref": "#/components/schemas/EnvironmentListParams" + "$ref": "#/components/schemas/EnvironmentUpdateParams" } } ], "result": { "name": "result", "schema": { - "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentListResponse" + "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentUpdateResponse" } }, - "summary": "List environments", + "summary": "Update an environment", "x-lightspeed-access": { - "action": "read", + "action": "configure_resource", "kind": "universe" }, "x-lightspeed-group": "environments", - "x-lightspeed-role": "viewer", + "x-lightspeed-role": "operator", "x-lightspeed-target": null }, { - "description": "Records an asynchronous idempotent close intent. Provider cleanup is resumed by lifecycle reconciliation; quota is released only after Closed.", - "name": "environments/close", + "description": "Lists the universe environments in id order, one page at a time, optionally filtered by provider, registration key, state, or metadata. Closed environments are omitted unless includeClosed is set or state is closed. Continue from nextAfter.", + "name": "environments/list", "paramStructure": "by-name", "params": [ { "name": "params", "required": true, "schema": { - "$ref": "#/components/schemas/EnvironmentCloseParams" + "$ref": "#/components/schemas/EnvironmentListParams" } } ], "result": { "name": "result", "schema": { - "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentCloseResponse" + "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentListResponse" } }, - "summary": "Close an environment", + "summary": "List environments", "x-lightspeed-access": { - "action": "configure_resource", + "action": "read", "kind": "universe" }, "x-lightspeed-group": "environments", - "x-lightspeed-role": "operator", + "x-lightspeed-role": "viewer", "x-lightspeed-target": null }, { - "description": "Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, attributed to the caller. Reachability is checked on demand.", - "name": "environments/external/create", + "description": "Closes an environment; idempotent. A provisioned environment moves to closing while Lightspeed asks its provider to remove the target, retrying until it is gone, then closed. External and registered environments close at once, and a connected registered daemon is told its environment closed.", + "name": "environments/close", "paramStructure": "by-name", "params": [ { "name": "params", "required": true, "schema": { - "$ref": "#/components/schemas/EnvironmentExternalCreateParams" + "$ref": "#/components/schemas/EnvironmentCloseParams" } } ], "result": { "name": "result", "schema": { - "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentExternalCreateResponse" + "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentCloseResponse" } }, - "summary": "Register an external environment", + "summary": "Close an environment", "x-lightspeed-access": { "action": "configure_resource", "kind": "universe" @@ -19905,25 +20259,25 @@ "x-lightspeed-target": null }, { - "description": "Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health.", - "name": "environments/ingress/put", + "description": "Creates an open environment backed by a Lightspeed-reachable envd endpoint, attributed to the caller. Idempotent per requestId. Reachability is checked on use.", + "name": "environments/external/create", "paramStructure": "by-name", "params": [ { "name": "params", "required": true, "schema": { - "$ref": "#/components/schemas/EnvironmentIngressPutParams" + "$ref": "#/components/schemas/EnvironmentExternalCreateParams" } } ], "result": { "name": "result", "schema": { - "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentIngressPutResponse" + "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentExternalCreateResponse" } }, - "summary": "Configure environment public ingress", + "summary": "Register an external environment", "x-lightspeed-access": { "action": "configure_resource", "kind": "universe" @@ -19933,25 +20287,25 @@ "x-lightspeed-target": null }, { - "description": "Records the desired power state (running, paused, suspended, or stopped) of a provisioned environment; the lifecycle reconciler converges the provider target asynchronously. Powered-down environments wake transparently on their next use. Rejected when the provider does not support the state.", - "name": "environments/power/put", + "description": "Synchronously ensures or removes public ingress of an open provisioned environment through its provider; source.publicEndpoint carries the reported endpoint. Rejected when the provider offers no ingress; an internal error, safe to retry, when the provider is unreachable or times out. The provider owns hostname, guest port, routing, TLS, and health.", + "name": "environments/ingress/put", "paramStructure": "by-name", "params": [ { "name": "params", "required": true, "schema": { - "$ref": "#/components/schemas/EnvironmentPowerPutParams" + "$ref": "#/components/schemas/EnvironmentIngressPutParams" } } ], "result": { "name": "result", "schema": { - "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentPowerPutResponse" + "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentIngressPutResponse" } }, - "summary": "Set environment power intent", + "summary": "Configure environment public ingress", "x-lightspeed-access": { "action": "configure_resource", "kind": "universe" @@ -19961,25 +20315,25 @@ "x-lightspeed-target": null }, { - "description": "Replaces or clears the staged idle policy of a provisioned environment. The power reaper measures the daemon's idle duration against the pause/suspend/stop/close thresholds and escalates through the stages the provider supports.", - "name": "environments/idle-policy/put", + "description": "Synchronously moves an open provisioned environment's target to running, paused, suspended, or stopped via its provider and records the observation; the provider may also change power on idle. Rejected without provider power control or when the template lacks the state; an internal error, safe to retry, when the provider is unreachable or times out.", + "name": "environments/power/put", "paramStructure": "by-name", "params": [ { "name": "params", "required": true, "schema": { - "$ref": "#/components/schemas/EnvironmentIdlePolicyPutParams" + "$ref": "#/components/schemas/EnvironmentPowerPutParams" } } ], "result": { "name": "result", "schema": { - "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentIdlePolicyPutResponse" + "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentPowerPutResponse" } }, - "summary": "Set environment idle policy", + "summary": "Set environment power", "x-lightspeed-access": { "action": "configure_resource", "kind": "universe" @@ -19989,7 +20343,7 @@ "x-lightspeed-target": null }, { - "description": "Lists this universe's revisioned routing and admission bindings to deployment-scoped physical providers.", + "description": "Lists this universe's bindings to deployment providers and whether each is enabled.", "name": "environments/provider-bindings/list", "paramStructure": "by-name", "params": [ @@ -20017,7 +20371,7 @@ "x-lightspeed-target": null }, { - "description": "Returns one universe routing and admission binding. Provider-wide templates and physical resource, network, and ingress policy remain provider-owned.", + "description": "Returns this universe's binding to one provider. Templates and physical resource, network, and ingress policy remain provider-owned.", "name": "environments/provider-bindings/read", "paramStructure": "by-name", "params": [ @@ -20045,7 +20399,7 @@ "x-lightspeed-target": null }, { - "description": "Reads immutable templates directly from the selected bound provider controller.", + "description": "Reads templates live from the controllers of the universe's enabled provider bindings, or of one provider: compute, power states, public ingress, and default idle policy.", "name": "environments/templates/list", "paramStructure": "by-name", "params": [ @@ -20073,7 +20427,7 @@ "x-lightspeed-target": null }, { - "description": "Returns one immutable template version from the selected bound provider controller.", + "description": "Returns one template, read live from a bound provider's controller.", "name": "environments/templates/read", "paramStructure": "by-name", "params": [ @@ -20101,7 +20455,7 @@ "x-lightspeed-target": null }, { - "description": "Starts a dependency-aware job group on one environment instance, injecting the environment's configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance rather than a session. A powered-down environment is woken on use: the call fails with environment_not_ready while the wake is in progress; retry with backoff.", + "description": "Starts a dependency-aware job group on one environment instance, injecting its configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance, not a session. A powered-down environment wakes on its data route; the call fails with environment_not_ready while it wakes or reconnects; retry with backoff.", "name": "environments/jobs/create", "paramStructure": "by-name", "params": [ @@ -20129,7 +20483,7 @@ "x-lightspeed-target": null }, { - "description": "Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling.", + "description": "Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status.", "name": "environments/jobs/read", "paramStructure": "by-name", "params": [ @@ -20185,7 +20539,7 @@ "x-lightspeed-target": null }, { - "description": "Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode, active limit, disconnect grace, and expiry are the key's policy. Treat the secret like a cluster-join credential.", + "description": "Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode (immutable), active limit, disconnect grace, offline close-after, and expiry are the key's policy. Treat the secret like a cluster-join credential.", "name": "environments/registration-keys/create", "paramStructure": "by-name", "params": [ @@ -20268,6 +20622,62 @@ "x-lightspeed-role": "operator", "x-lightspeed-target": null }, + { + "description": "Replaces a key's editable policy: display name, active limit, disconnect grace, offline close-after, and expiry; an omitted field is unset. The identity mode is immutable. The active limit and expiry apply to later admissions; a changed disconnect grace or offline close-after applies at the next sweep to environments whose daemon is away.", + "name": "environments/registration-keys/update", + "paramStructure": "by-name", + "params": [ + { + "name": "params", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentRegistrationKeyUpdateParams" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentRegistrationKeyUpdateResponse" + } + }, + "summary": "Update an environment registration key", + "x-lightspeed-access": { + "action": "configure_resource", + "kind": "universe" + }, + "x-lightspeed-group": "environments", + "x-lightspeed-role": "operator", + "x-lightspeed-target": null + }, + { + "description": "Replaces the key's secret and display prefix and retires the old secret at once. The key keeps its id, group, policy, and the environments it admitted; daemons already registered are unaffected because they never present the key again. Returns the new secret once. Refused for a revoked key.", + "name": "environments/registration-keys/rotate", + "paramStructure": "by-name", + "params": [ + { + "name": "params", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentRegistrationKeyRotateParams" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/AgentApiOutcomeOfEnvironmentRegistrationKeyRotateResponse" + } + }, + "summary": "Rotate an environment registration key", + "x-lightspeed-access": { + "action": "configure_resource", + "kind": "universe" + }, + "x-lightspeed-group": "environments", + "x-lightspeed-role": "operator", + "x-lightspeed-target": null + }, { "description": "Stops the key from admitting new daemon identities; already registered daemons keep reconnecting. With closeEnvironments, also closes every non-closed environment the key admitted. Idempotent.", "name": "environments/registration-keys/revoke", @@ -22579,7 +22989,7 @@ "x-lightspeed-target": null }, { - "description": "Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed.", + "description": "Registers or replaces a deployment provider: controller endpoint, optional data route base, and optional bearer credential for controller and data routes (write-only: kept when omitted, removed by clearCredential, never returned). The controller is probed: a definitive refusal is invalid_request, unreachable is accepted. The provider never calls Lightspeed.", "name": "deployment/environment-providers/put", "paramStructure": "by-name", "params": [ @@ -22606,7 +23016,7 @@ "x-lightspeed-target": null }, { - "description": "Returns every deployment-registered deployment provider and its controller connection.", + "description": "Returns every deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored.", "name": "deployment/environment-providers/list", "paramStructure": "by-name", "params": [ @@ -22633,7 +23043,7 @@ "x-lightspeed-target": null }, { - "description": "Returns one deployment-registered deployment provider and its controller connection.", + "description": "Returns one deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored.", "name": "deployment/environment-providers/read", "paramStructure": "by-name", "params": [ @@ -22660,7 +23070,7 @@ "x-lightspeed-target": null }, { - "description": "Deletes a deployment provider only when no universe binding references it.", + "description": "Deletes a deployment provider only when no universe binding and no non-closed environment references it.", "name": "deployment/environment-providers/delete", "paramStructure": "by-name", "params": [ @@ -22687,7 +23097,7 @@ "x-lightspeed-target": null }, { - "description": "Creates or replaces one universe's complete revisioned routing and admission binding. A deployment provider may have at most one binding in a universe.", + "description": "Creates or replaces one universe's binding to a deployment provider. Only an enabled binding admits creates, template listing, and data routes for the universe.", "name": "deployment/environment-providers/bindings/put", "paramStructure": "by-name", "params": [ @@ -22714,7 +23124,7 @@ "x-lightspeed-target": null }, { - "description": "Deletes a universe provider binding only after every referencing environment has reached Closed.", + "description": "Deletes a universe's binding to a provider only when the universe has no non-closed environment of that provider.", "name": "deployment/environment-providers/bindings/delete", "paramStructure": "by-name", "params": [ @@ -22740,33 +23150,6 @@ "x-lightspeed-role": null, "x-lightspeed-target": null }, - { - "description": "Creates a universe environment by transferring an existing provider target into Lightspeed's managed lifecycle. The caller must explicitly accept ownership transfer.", - "name": "deployment/environments/adopt", - "paramStructure": "by-name", - "params": [ - { - "name": "params", - "required": true, - "schema": { - "$ref": "#/components/schemas/DeploymentEnvironmentAdoptParams" - } - } - ], - "result": { - "name": "result", - "schema": { - "$ref": "#/components/schemas/AgentApiOutcomeOfDeploymentEnvironmentAdoptResponse" - } - }, - "summary": "Adopt a provider environment", - "x-lightspeed-access": { - "kind": "deployment" - }, - "x-lightspeed-group": "deployment/environment-providers", - "x-lightspeed-role": null, - "x-lightspeed-target": null - }, { "description": "The connector host's discovery call: every enabled provider account of the deployment with its universe id and credential grant reference. Re-poll to pick up accounts created or disabled since.", "name": "deployment/channels/accounts/list", diff --git a/crates/api/src/access.rs b/crates/api/src/access.rs index feac11115..ea560607c 100644 --- a/crates/api/src/access.rs +++ b/crates/api/src/access.rs @@ -85,7 +85,7 @@ pub enum MethodGroup { DeploymentUniverses, #[serde(rename = "deployment/api-keys")] DeploymentApiKeys, - /// Environment providers, their bindings and adoption. + /// Environment providers and their universe bindings. #[serde(rename = "deployment/environment-providers")] DeploymentEnvironmentProviders, /// `deployment/channels/accounts/list`: a connector host's discovery. @@ -208,7 +208,7 @@ impl MethodGroup { Self::DeploymentUniverses } else if group("deployment/api-keys/") { Self::DeploymentApiKeys - } else if group("deployment/environment-provider") || group("deployment/environments/") { + } else if group("deployment/environment-provider") { Self::DeploymentEnvironmentProviders } else if group("deployment/channels/") { Self::DeploymentChannels @@ -514,10 +514,6 @@ mod tests { "deployment/environment-provider-bindings/list", Some(MethodGroup::DeploymentEnvironmentProviders), ), - ( - "deployment/environments/adopt", - Some(MethodGroup::DeploymentEnvironmentProviders), - ), ( "deployment/channels/accounts/list", Some(MethodGroup::DeploymentChannels), diff --git a/crates/api/src/constants.rs b/crates/api/src/constants.rs index 8d3b51385..0ba4b711a 100644 --- a/crates/api/src/constants.rs +++ b/crates/api/src/constants.rs @@ -98,12 +98,12 @@ pub const METHOD_MCP_SERVERS_DELETE: &str = "mcp/servers/delete"; pub const METHOD_ENVIRONMENTS_CREATE: &str = "environments/create"; pub const METHOD_ENVIRONMENTS_READ: &str = "environments/read"; +pub const METHOD_ENVIRONMENTS_UPDATE: &str = "environments/update"; pub const METHOD_ENVIRONMENTS_LIST: &str = "environments/list"; pub const METHOD_ENVIRONMENTS_CLOSE: &str = "environments/close"; pub const METHOD_ENVIRONMENTS_EXTERNAL_CREATE: &str = "environments/external/create"; pub const METHOD_ENVIRONMENTS_INGRESS_PUT: &str = "environments/ingress/put"; pub const METHOD_ENVIRONMENTS_POWER_PUT: &str = "environments/power/put"; -pub const METHOD_ENVIRONMENTS_IDLE_POLICY_PUT: &str = "environments/idle-policy/put"; pub const METHOD_ENVIRONMENTS_PROVIDER_BINDINGS_LIST: &str = "environments/provider-bindings/list"; pub const METHOD_ENVIRONMENTS_PROVIDER_BINDINGS_READ: &str = "environments/provider-bindings/read"; pub const METHOD_ENVIRONMENTS_TEMPLATES_LIST: &str = "environments/templates/list"; @@ -115,6 +115,10 @@ pub const METHOD_ENVIRONMENTS_REGISTRATION_KEYS_CREATE: &str = "environments/registration-keys/create"; pub const METHOD_ENVIRONMENTS_REGISTRATION_KEYS_READ: &str = "environments/registration-keys/read"; pub const METHOD_ENVIRONMENTS_REGISTRATION_KEYS_LIST: &str = "environments/registration-keys/list"; +pub const METHOD_ENVIRONMENTS_REGISTRATION_KEYS_UPDATE: &str = + "environments/registration-keys/update"; +pub const METHOD_ENVIRONMENTS_REGISTRATION_KEYS_ROTATE: &str = + "environments/registration-keys/rotate"; pub const METHOD_ENVIRONMENTS_REGISTRATION_KEYS_REVOKE: &str = "environments/registration-keys/revoke"; diff --git a/crates/api/src/deployment.rs b/crates/api/src/deployment.rs index 393efbce6..b5581efbe 100644 --- a/crates/api/src/deployment.rs +++ b/crates/api/src/deployment.rs @@ -37,7 +37,6 @@ pub const METHOD_DEPLOYMENT_PROVIDER_BINDINGS_PUT: &str = "deployment/environment-providers/bindings/put"; pub const METHOD_DEPLOYMENT_PROVIDER_BINDINGS_DELETE: &str = "deployment/environment-providers/bindings/delete"; -pub const METHOD_DEPLOYMENT_ENVIRONMENTS_ADOPT: &str = "deployment/environments/adopt"; pub const METHOD_DEPLOYMENT_CHANNELS_ACCOUNTS_LIST: &str = "deployment/channels/accounts/list"; pub const METHOD_DEPLOYMENT_SESSIONS_DELETED_LIST: &str = "deployment/sessions/deleted/list"; @@ -272,48 +271,70 @@ pub struct DeploymentApiKeyRevokeResponse { pub api_key: DeploymentApiKeyView, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum DeploymentEnvironmentProviderTransport { - WebSocket, - Http, - Stdio, - Ssh, - Provider { - #[serde(rename = "providerType")] - provider_type: String, - }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct DeploymentEnvironmentProviderConnection { - pub endpoint: String, - pub transport: DeploymentEnvironmentProviderTransport, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DeploymentEnvironmentProviderView { pub provider_id: EnvironmentProviderId, #[serde(default, skip_serializing_if = "Option::is_none")] pub display_name: Option, - pub controller_connection: DeploymentEnvironmentProviderConnection, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, + /// `ws://`, `wss://` or `unix://` URL of the provider controller. + pub controller_endpoint: String, + /// Operator-configured `ws://` or `wss://` base of the provider's data + /// routes; absent derives it from the controller endpoint. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_route_base: Option, + /// The data route base Lightspeed dials: `dataRouteBase`, or the + /// controller endpoint's origin followed by `/routes`. Absent only when it + /// cannot be derived. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effective_data_route_base: Option, + /// A bearer credential is stored and presented to the controller and + /// data routes; without one no `Authorization` header is sent. + pub has_credential: bool, pub created_at_ms: i64, pub updated_at_ms: i64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +/// Register or replace a provider. `credential` is the optional bearer +/// token Lightspeed presents to the provider's controller and data routes: +/// it is sealed on receipt and never returned by any method. `Debug` output +/// redacts it; request logging must never echo these params. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DeploymentEnvironmentProviderPutParams { pub provider_id: EnvironmentProviderId, #[serde(default, skip_serializing_if = "Option::is_none")] pub display_name: Option, - pub controller_connection: DeploymentEnvironmentProviderConnection, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, + /// `ws://`, `wss://` or `unix://` URL of the provider controller. + pub controller_endpoint: String, + /// `ws://` or `wss://` base of the provider's data routes; omit to derive + /// it from the controller endpoint. Required for a `unix://` controller, + /// and must not be `ws://` for a `wss://` controller. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_route_base: Option, + /// A new bearer credential; omit to keep the stored one. A provider needs + /// none. Excludes `clearCredential`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credential: Option, + /// Remove the stored credential. Excludes `credential`. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub clear_credential: bool, +} + +impl fmt::Debug for DeploymentEnvironmentProviderPutParams { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DeploymentEnvironmentProviderPutParams") + .field("provider_id", &self.provider_id) + .field("display_name", &self.display_name) + .field("controller_endpoint", &self.controller_endpoint) + .field("data_route_base", &self.data_route_base) + .field( + "credential", + &self.credential.as_ref().map(|_| ""), + ) + .field("clear_credential", &self.clear_credential) + .finish() + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -356,17 +377,14 @@ pub struct DeploymentEnvironmentProviderDeleteResponse { pub provider: DeploymentEnvironmentProviderView, } +/// Create or replace a universe's binding to a deployment provider. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct DeploymentProviderBindingPutParams { pub universe_id: String, - pub binding_id: EnvironmentProviderBindingId, pub provider_id: EnvironmentProviderId, - pub status: EnvironmentProviderBindingStatusView, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub expected_revision: Option, + /// Admits creates, template listing and data routes for the universe. + pub enabled: bool, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -379,7 +397,7 @@ pub struct DeploymentProviderBindingPutResponse { #[serde(rename_all = "camelCase")] pub struct DeploymentProviderBindingDeleteParams { pub universe_id: String, - pub binding_id: EnvironmentProviderBindingId, + pub provider_id: EnvironmentProviderId, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -388,31 +406,6 @@ pub struct DeploymentProviderBindingDeleteResponse { pub binding: EnvironmentProviderBindingView, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct DeploymentEnvironmentAdoptParams { - pub universe_id: String, - /// Stable caller-generated retry identity inside the destination universe. - pub request_id: EnvironmentProvisionRequestId, - pub binding_id: EnvironmentProviderBindingId, - /// Provider-native source reference. The Incus provider accepts - /// `/` or an instance name in the `default` project. - pub source_target: String, - /// Required acknowledgement that Lightspeed will move, reconfigure, and - /// delete the target as part of its ordinary managed lifecycle. - pub take_ownership: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub display_name: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct DeploymentEnvironmentAdoptResponse { - pub environment: EnvironmentView, -} - /// One channel account of any universe, for the connector host's discovery /// pass. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -566,15 +559,6 @@ pub trait DeploymentApiService: Send + Sync { )) } - async fn adopt_environment( - &self, - _params: DeploymentEnvironmentAdoptParams, - ) -> Result, AgentApiError> { - Err(AgentApiError::internal( - "managed environment adoption is unavailable", - )) - } - async fn list_deployment_channel_accounts( &self, _params: DeploymentChannelAccountListParams, @@ -663,19 +647,17 @@ deployment_api_methods! { METHOD_DEPLOYMENT_API_KEYS_REVOKE => revoke_api_key(DeploymentApiKeyRevokeParams) -> DeploymentApiKeyRevokeResponse => ["Revoke a scoped API key", "Revokes the key with this display prefix; revoking a revoked key keeps its first revocation time. An unknown prefix is not found."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_PUT => put_environment_provider(DeploymentEnvironmentProviderPutParams) -> DeploymentEnvironmentProviderPutResponse => - ["Put an environment provider", "Registers or replaces one deployment provider and its controller connection. The provider does not call this API or require access to Lightspeed."], access: MethodAccess::Deployment, + ["Put an environment provider", "Registers or replaces a deployment provider: controller endpoint, optional data route base, and optional bearer credential for controller and data routes (write-only: kept when omitted, removed by clearCredential, never returned). The controller is probed: a definitive refusal is invalid_request, unreachable is accepted. The provider never calls Lightspeed."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_LIST => list_environment_providers(DeploymentEnvironmentProviderListParams) -> DeploymentEnvironmentProviderListResponse => - ["List environment providers", "Returns every deployment-registered deployment provider and its controller connection."], access: MethodAccess::Deployment, + ["List environment providers", "Returns every deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_READ => read_environment_provider(DeploymentEnvironmentProviderReadParams) -> DeploymentEnvironmentProviderReadResponse => - ["Read an environment provider", "Returns one deployment-registered deployment provider and its controller connection."], access: MethodAccess::Deployment, + ["Read an environment provider", "Returns one deployment-registered provider with its controller endpoint, data route base, and whether a credential is stored."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_DELETE => delete_environment_provider(DeploymentEnvironmentProviderDeleteParams) -> DeploymentEnvironmentProviderDeleteResponse => - ["Delete an environment provider", "Deletes a deployment provider only when no universe binding references it."], access: MethodAccess::Deployment, + ["Delete an environment provider", "Deletes a deployment provider only when no universe binding and no non-closed environment references it."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_PROVIDER_BINDINGS_PUT => put_environment_provider_binding(DeploymentProviderBindingPutParams) -> DeploymentProviderBindingPutResponse => - ["Put an environment provider binding", "Creates or replaces one universe's complete revisioned routing and admission binding. A deployment provider may have at most one binding in a universe."], access: MethodAccess::Deployment, + ["Put an environment provider binding", "Creates or replaces one universe's binding to a deployment provider. Only an enabled binding admits creates, template listing, and data routes for the universe."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_PROVIDER_BINDINGS_DELETE => delete_environment_provider_binding(DeploymentProviderBindingDeleteParams) -> DeploymentProviderBindingDeleteResponse => - ["Delete an environment provider binding", "Deletes a universe provider binding only after every referencing environment has reached Closed."], access: MethodAccess::Deployment, - METHOD_DEPLOYMENT_ENVIRONMENTS_ADOPT => adopt_environment(DeploymentEnvironmentAdoptParams) -> DeploymentEnvironmentAdoptResponse => - ["Adopt a provider environment", "Creates a universe environment by transferring an existing provider target into Lightspeed's managed lifecycle. The caller must explicitly accept ownership transfer."], access: MethodAccess::Deployment, + ["Delete an environment provider binding", "Deletes a universe's binding to a provider only when the universe has no non-closed environment of that provider."], access: MethodAccess::Deployment, METHOD_DEPLOYMENT_CHANNELS_ACCOUNTS_LIST => list_deployment_channel_accounts(DeploymentChannelAccountListParams) -> DeploymentChannelAccountListResponse => ["List channel accounts across universes", "The connector host's discovery call: every enabled provider account of the deployment with its universe id and credential grant reference. Re-poll to pick up accounts created or disabled since."], access: MethodAccess::Deployment, } diff --git a/crates/api/src/environments.rs b/crates/api/src/environments.rs index d53a85c5d..3d17dcc65 100644 --- a/crates/api/src/environments.rs +++ b/crates/api/src/environments.rs @@ -29,17 +29,21 @@ pub struct SessionEnvironmentDeactivateResponse { #[serde(rename_all = "camelCase")] pub struct EnvironmentCreateParams { /// Stable caller-generated retry identity, unique inside the universe. + /// Retrying with identical inputs returns the same environment. pub request_id: EnvironmentProvisionRequestId, - pub binding_id: EnvironmentProviderBindingId, - /// Immutable provider template-version identity. + /// The provider to create the environment with; the universe needs an + /// enabled binding to it. + pub provider_id: EnvironmentProviderId, + /// The provider template to create the environment from. pub template_id: EnvironmentTemplateId, #[serde(default, skip_serializing_if = "Option::is_none")] pub display_name: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, - /// Optional staged idle policy applied by the power reaper. + /// Idle policy passed to the provider in place of the template's `idle`. + /// The provider runs the idle clock; it cannot be changed after create. #[serde(default, skip_serializing_if = "Option::is_none")] - pub idle_policy: Option, + pub idle: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -60,29 +64,63 @@ pub struct EnvironmentReadResponse { pub environment: EnvironmentView, } +/// Replace an environment's display name and caller metadata. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentUpdateParams { + pub environment_id: EnvironmentId, + /// The new display name; omit to clear it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display_name: Option, + /// The complete new caller metadata; omit or `{}` to clear it. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub metadata: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentUpdateResponse { + pub environment: EnvironmentView, +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentListParams { + /// Only provisioned environments of this provider. #[serde(default, skip_serializing_if = "Option::is_none")] pub provider_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub binding_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, /// Only registered environments admitted by this registration key. #[serde(default, skip_serializing_if = "Option::is_none")] pub registration_key_id: Option, + /// Only environments in this lifecycle state. `closed` lists closed + /// environments without `includeClosed`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state: Option, + /// Also list closed environments; they are omitted by default. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub include_closed: bool, /// Only environments carrying every listed metadata pair (AND /// semantics); the same filter `session/list` accepts. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, + /// Most environments per page; omit for 100. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1, max = 500))] + pub limit: Option, + /// Exclusive environment id cursor: a previous page's `nextAfter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub after: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentListResponse { + /// Environments in environment id order. #[serde(default)] pub environments: Vec, + /// Pass as `after` to read the next page; absent on the last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_after: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -101,6 +139,7 @@ pub struct EnvironmentCloseResponse { #[serde(rename_all = "camelCase")] pub struct EnvironmentIngressPutParams { pub environment_id: EnvironmentId, + /// Ensure (`true`) or remove (`false`) the provider's public ingress. pub enabled: bool, } @@ -110,7 +149,7 @@ pub struct EnvironmentIngressPutResponse { pub environment: EnvironmentView, } -/// Steady power state of a provisioned environment. +/// Power state of a provisioned environment's target. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum EnvironmentPowerStateView { @@ -123,11 +162,13 @@ pub enum EnvironmentPowerStateView { Stopped, } -/// Staged idle policy. Thresholds are milliseconds of daemon-reported idle -/// time and must be non-decreasing in the order pause, suspend, stop, close. -/// Stages whose power state the provider does not support are skipped. +/// Staged idle policy the provider applies. Thresholds are milliseconds of +/// envd-reported idle time (running processes, jobs and leftover process +/// groups hold it at zero) and must be non-decreasing in the order pause, +/// suspend, stop, close. Stages whose power state the template does not +/// support are skipped. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct EnvironmentIdlePolicyView { #[serde(default, skip_serializing_if = "Option::is_none")] pub pause_after_ms: Option, @@ -143,8 +184,8 @@ pub struct EnvironmentIdlePolicyView { #[serde(rename_all = "camelCase")] pub struct EnvironmentPowerPutParams { pub environment_id: EnvironmentId, - /// Desired steady power state. Must be one of the provider-reported - /// `incarnation.powerStates`. + /// Power state to move the target to; one of its template's + /// `powerStates`. pub power: EnvironmentPowerStateView, } @@ -154,27 +195,14 @@ pub struct EnvironmentPowerPutResponse { pub environment: EnvironmentView, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentIdlePolicyPutParams { - pub environment_id: EnvironmentId, - /// The complete new policy; omit to clear it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub idle_policy: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentIdlePolicyPutResponse { - pub environment: EnvironmentView, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentExternalCreateParams { + /// Stable caller-generated retry identity, unique inside the universe. pub request_id: EnvironmentProvisionRequestId, - /// Connection to an envd instance reachable from Lightspeed. - pub connection: EnvironmentConnectionView, + /// `ws://`, `wss://` or `unix://` URL of an envd instance reachable from + /// Lightspeed. + pub endpoint: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub display_name: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] @@ -187,25 +215,104 @@ pub struct EnvironmentExternalCreateResponse { pub environment: EnvironmentView, } +/// The lifecycle Lightspeed owns. External and registered environments are +/// only ever `open` or `closed`. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub enum EnvironmentLifecycleStatusView { +pub enum EnvironmentStateView { + /// The provider is creating the target; the environment opens after + /// Lightspeed's first handshake with its envd. + Provisioning, + Open, + /// The provider is removing the target. + Closing, + Closed, + /// Provisioning or the provider failed; `stateMessage` says why. + Failed, +} + +/// Display status derived from `state` and the latest observation; never +/// stored. Tools that use a paused, suspended or stopped environment wake +/// it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum EnvironmentStatusView { Provisioning, - Booting, Ready, - /// Execution frozen; wakes on next use. + /// Execution frozen with RAM resident. Paused, - /// Execution state saved to disk; wakes on next use. + /// Execution state saved to disk. Suspended, - /// Powered off; provisioned environments wake on next use when the - /// provider supports power control. - Offline, + /// Powered off with disk retained. + Stopped, + /// Open, but the provider's latest observation is inconclusive. + Unknown, + /// Registered, open, and its daemon is not connected. + Away, + Closing, + Closed, + Failed, +} + +/// Why an environment is closing or closed. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum EnvironmentCloseReasonView { + /// `environments/close`. + Requested, + /// Its registration key was revoked with `closeEnvironments`. + KeyRevoked, + /// Its ephemeral daemon stayed away longer than the key's disconnect + /// grace. + EphemeralGrace, + /// Its ephemeral daemon said goodbye. + Departed, + /// Its persistent daemon stayed away longer than the key's + /// `offlineCloseAfterMs`. + OfflineTimeout, + /// The provider reported the target closed or gone. + ProviderEnded, +} + +/// Lifecycle phase of a provider target, as the provider reports it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum EnvironmentTargetPhaseView { + Provisioning, + Active, Closing, Closed, Failed, Unknown, } +/// What the provider last reported about a provisioned environment's +/// target. Display only: Lightspeed never converges it, and it may be up to +/// a few minutes old. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentObservedTargetView { + pub phase: EnvironmentTargetPhaseView, + /// Present only while `phase` is `active`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub power: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, + pub observed_at_ms: i64, +} + +/// The envd build serving an environment, from its registration or +/// Lightspeed's open handshake. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentDaemonView { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub git_sha: Option, + pub protocol_version: u32, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde( tag = "type", @@ -213,12 +320,28 @@ pub enum EnvironmentLifecycleStatusView { rename_all_fields = "camelCase" )] pub enum EnvironmentSourceView { + /// A target a deployment provider created from a template. The provider + /// owns its power, idle policy and ingress. Provisioned { provider_id: EnvironmentProviderId, - binding_id: EnvironmentProviderBindingId, + template_id: EnvironmentTemplateId, + /// The provider's handle for the target; absent until the provider + /// accepted the create. + #[serde(default, skip_serializing_if = "Option::is_none")] + target_id: Option, + /// The create-time idle policy, when one replaced the template's. + #[serde(default, skip_serializing_if = "Option::is_none")] + idle: Option, + /// Absent until the provider first reported the target. + #[serde(default, skip_serializing_if = "Option::is_none")] + observed: Option, + /// The public endpoint the provider last reported. + #[serde(default, skip_serializing_if = "Option::is_none")] + public_endpoint: Option, }, External { - connection: EnvironmentConnectionView, + /// `ws://`, `wss://` or `unix://` URL Lightspeed dials envd at. + endpoint: String, }, /// An envd that dialed the gateway outbound and was admitted by a /// registration key. The key is the environment's group; the daemon id @@ -226,19 +349,30 @@ pub enum EnvironmentSourceView { Registered { registration_key_id: EnvironmentRegistrationKeyId, daemon_id: EnvironmentDaemonId, + /// The admitting key's identity mode. identity_mode: EnvironmentIdentityModeView, + /// The daemon's control connection is up. + connected: bool, + /// When the daemon last connected or disconnected. + connection_changed_at_ms: i64, + /// When the gateway last recorded the daemon's control connection: + /// at connect, at disconnect, and at most every five minutes while + /// connected. + last_seen_at_ms: i64, }, } -/// What Lightspeed does with a registered environment while its daemon is -/// disconnected. Registration-key policy, copied onto each environment. +/// What happens to a registered environment while its daemon is +/// disconnected. Policy of the registration key that admitted it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum EnvironmentIdentityModeView { - /// Stays offline until explicitly closed. + /// Stays open: tools wait up to two minutes for the daemon to reconnect. + /// Closed explicitly, or after the key's `offlineCloseAfterMs`. Persistent, - /// Closed once the daemon has been away longer than the key's - /// disconnect grace. + /// Unusable while disconnected; closed once the daemon has been away + /// longer than the key's disconnect grace, or at once when it says + /// goodbye. Ephemeral, } @@ -256,13 +390,17 @@ pub struct EnvironmentRegistrationKeyView { pub registration_key_id: EnvironmentRegistrationKeyId, /// The group name shown wherever registered environments are listed. pub display_name: String, - /// First characters of the secret, for identification only. + /// First characters of the key's live secret, for identification only. pub key_prefix: String, pub identity_mode: EnvironmentIdentityModeView, #[serde(default, skip_serializing_if = "Option::is_none")] pub max_active_environments: Option, /// Effective disconnect grace for ephemeral environments. pub ephemeral_disconnect_grace_ms: u64, + /// Persistent keys only: environments whose daemon has been away longer + /// than this are closed; absent keeps them open. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offline_close_after_ms: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub expires_at_ms: Option, pub status: EnvironmentRegistrationKeyStatusView, @@ -273,6 +411,7 @@ pub struct EnvironmentRegistrationKeyView { #[serde(default, skip_serializing_if = "Option::is_none")] pub last_registered_at_ms: Option, pub created_at_ms: i64, + pub updated_at_ms: i64, #[serde(default, skip_serializing_if = "Option::is_none")] pub revoked_at_ms: Option, } @@ -294,6 +433,7 @@ impl fmt::Debug for EnvironmentRegistrationSecretView { pub struct EnvironmentRegistrationKeyCreateParams { /// Group name for the environments this key admits. pub display_name: String, + /// Immutable once the key exists. pub identity_mode: EnvironmentIdentityModeView, /// Non-closed environments the key may have at once; omit for unlimited. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -301,6 +441,10 @@ pub struct EnvironmentRegistrationKeyCreateParams { /// Disconnect grace for ephemeral environments; omit for the default. #[serde(default, skip_serializing_if = "Option::is_none")] pub ephemeral_disconnect_grace_ms: Option, + /// Persistent keys only: close environments whose daemon has been away + /// longer than this; omit to keep them open. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offline_close_after_ms: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub expires_at_ms: Option, } @@ -313,6 +457,48 @@ pub struct EnvironmentRegistrationKeyCreateResponse { pub secret: EnvironmentRegistrationSecretView, } +/// The complete editable policy of a registration key; an absent field is +/// unset. The identity mode is immutable. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentRegistrationKeyUpdateParams { + pub registration_key_id: EnvironmentRegistrationKeyId, + /// Group name for the environments this key admits. + pub display_name: String, + /// Non-closed environments the key may have at once; omit for unlimited. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_active_environments: Option, + /// Disconnect grace for ephemeral environments; omit for the default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ephemeral_disconnect_grace_ms: Option, + /// Persistent keys only: close environments whose daemon has been away + /// longer than this; omit to keep them open. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offline_close_after_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires_at_ms: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentRegistrationKeyUpdateResponse { + pub registration_key: EnvironmentRegistrationKeyView, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentRegistrationKeyRotateParams { + pub registration_key_id: EnvironmentRegistrationKeyId, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentRegistrationKeyRotateResponse { + pub registration_key: EnvironmentRegistrationKeyView, + /// The new plaintext secret, returned only here. + pub secret: EnvironmentRegistrationSecretView, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentRegistrationKeyReadParams { @@ -354,84 +540,46 @@ pub struct EnvironmentRegistrationKeyRevokeResponse { pub closed_environment_ids: Vec, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentConnectionView { - pub endpoint: String, - pub transport: EnvironmentConnectionTransportView, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub enum EnvironmentConnectionTransportView { - WebSocket, - Http, - Stdio, - Ssh, - Provider { provider_type: String }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentIncarnationView { - pub incarnation_id: EnvironmentIncarnationId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provision_request_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_target_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub template_id: Option, - /// Power states the provider reported for this target; empty until - /// observed or when the provider offers no power control. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub power_states: Vec, - pub created_at_ms: i64, - pub updated_at_ms: i64, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentView { pub environment_id: EnvironmentId, - pub request_id: EnvironmentProvisionRequestId, + /// The caller's retry identity; absent for registered environments. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_id: Option, pub source: EnvironmentSourceView, #[serde(default, skip_serializing_if = "Option::is_none")] pub display_name: Option, - pub status: EnvironmentLifecycleStatusView, - /// Lightspeed-owned power intent; `status` is the observed state. - pub desired_power: EnvironmentPowerStateView, + pub state: EnvironmentStateView, + pub status: EnvironmentStatusView, + /// Why the environment failed, or what provisioning or closing is + /// waiting on. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state_message: Option, + /// Present exactly while `state` is `closing` or `closed`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub idle_policy: Option, - pub incarnation: EnvironmentIncarnationView, - pub public_ingress_enabled: bool, + pub close_reason: Option, + /// When `state` last changed. + pub state_changed_at_ms: i64, + /// The envd build serving the environment; absent until known. #[serde(default, skip_serializing_if = "Option::is_none")] - pub public_endpoint: Option, + pub daemon: Option, + /// Caller metadata. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, - /// Registered environments only: when the gateway last saw the daemon's - /// control connection. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub last_seen_at_ms: Option, pub created_at_ms: i64, pub updated_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub closed_at_ms: Option, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub enum EnvironmentProviderBindingStatusView { - Enabled, - Disabled, -} - +/// A universe's binding to a deployment provider. Creates, templates and +/// data routes of the provider need it enabled. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentProviderBindingView { - pub binding_id: EnvironmentProviderBindingId, pub provider_id: EnvironmentProviderId, - pub status: EnvironmentProviderBindingStatusView, - pub revision: u64, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, + pub enabled: bool, pub created_at_ms: i64, pub updated_at_ms: i64, } @@ -449,7 +597,7 @@ pub struct EnvironmentProviderBindingListResponse { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentProviderBindingReadParams { - pub binding_id: EnvironmentProviderBindingId, + pub provider_id: EnvironmentProviderId, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -463,21 +611,46 @@ pub struct EnvironmentProviderBindingReadResponse { pub struct EnvironmentTemplateView { pub template_id: EnvironmentTemplateId, pub provider_id: EnvironmentProviderId, - pub binding_id: EnvironmentProviderBindingId, pub display_name: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, + /// `environments/ingress/put` can enable public ingress on targets of + /// this template. pub public_ingress: bool, pub deprecated: bool, + /// Compute targets of this template get; absent when the provider does + /// not fix it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resources: Option, + /// Power states `environments/power/put` can move targets of this + /// template to; empty without power control. + pub power_states: Vec, + /// Idle policy targets get when `environments/create` names none. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub idle: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub metadata: BTreeMap, } +/// A template's compute, in provider-neutral units. Each field is absent +/// when the provider does not fix it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct EnvironmentTemplateResourcesView { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vcpus: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub disk_bytes: Option, +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentTemplateListParams { + /// Only this provider's templates; omit for every enabled binding. #[serde(default, skip_serializing_if = "Option::is_none")] - pub binding_id: Option, + pub provider_id: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -489,7 +662,7 @@ pub struct EnvironmentTemplateListResponse { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct EnvironmentTemplateReadParams { - pub binding_id: EnvironmentProviderBindingId, + pub provider_id: EnvironmentProviderId, pub template_id: EnvironmentTemplateId, } @@ -632,6 +805,9 @@ pub struct SessionJobStartedView { pub queue_key: Option, } +/// One job's summary and output page, or why it could not be read: an +/// entry with `error` and no summary names a job the environment does not +/// know (never created, or garbage-collected). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SessionJobReadEntryView { @@ -639,11 +815,17 @@ pub struct SessionJobReadEntryView { pub handle: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub summary: Option, + /// Output in offset order. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub output_chunks: Vec, - pub output_next_seq: u64, + /// Byte offset to continue reading from: pass it as `after`. + pub output_next: u64, + /// Output bytes the job produced so far; final once it is terminal. + pub output_end: u64, + /// Output ranges between the requested start and `outputNext` that + /// cannot be returned. #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub artifacts: Vec, + pub output_gaps: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, } @@ -667,25 +849,38 @@ pub enum SessionJobCancelScopeView { Dependents, } +/// What a job is and how it ended. Instants are the environment's wall +/// clock. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SessionJobSummaryView { - pub namespace: String, pub job_id: String, + /// The start request that created the job; part of its identity. + pub request_id: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, pub status: SessionJobStatusView, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub dependencies: Vec, - pub created_at_ms: u64, + pub created_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queued_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at_ms: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub queued_at_ms: Option, + pub finished_at_ms: Option, + /// When a cancel reached the job while it ran. #[serde(default, skip_serializing_if = "Option::is_none")] - pub started_at_ms: Option, + pub cancel_requested_at_ms: Option, + /// Run time from start to finish, immune to clock jumps. #[serde(default, skip_serializing_if = "Option::is_none")] - pub finished_at_ms: Option, + pub duration_ms: Option, + /// How the job's root process ended, once it ran and exited. #[serde(default, skip_serializing_if = "Option::is_none")] - pub exit_code: Option, + pub exit: Option, + /// Set when Lightspeed stopped the running job. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stopped_by: Option, /// True when the job's root process exited while descendants it spawned /// were still running; the host terminated them best-effort at job /// completion. @@ -697,45 +892,91 @@ pub struct SessionJobSummaryView { pub queue_key: Option, } +/// Job lifecycle. `queued` and `running` are live; the rest are terminal. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum SessionJobStatusView { - Accepted, + /// Waiting for dependencies or its queue. Queued, Running, + /// Exited with code 0. Succeeded, + /// Exited otherwise, or could not start. Failed, - CancelRequested, - Cancelled, + /// Its timeout expired and it was stopped. TimedOut, + /// Stopped or never started because of a cancel. + Cancelled, + /// A dependency did not finish as the dependency policy requires. DependencyFailed, + /// The environment daemon stopped while the job was queued or running. Interrupted, - Lost, +} + +/// How a job's root process ended. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde( + tag = "type", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum SessionJobExitView { + /// Normal exit with this status code. + Code { code: i32 }, + /// Killed by this signal, named like `SIGSEGV`. + Signal { + signal: String, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + core_dumped: bool, + }, +} + +/// Why Lightspeed stopped a running job. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum SessionJobStoppedByView { + /// Its timeout expired: terminate, a grace period, then kill. + Timeout, + /// A cancel without `force`: terminate, a grace period, then kill. + Terminate, + /// A cancel with `force`: killed at once. + Kill, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SessionJobOutputChunkView { - pub seq: u64, + /// Byte offset of the chunk's first byte in the job's combined output. + pub offset: u64, pub stream: SessionJobOutputStreamView, pub data_base64: String, } +/// Output bytes in `[offset, offset + omittedBytes)` that cannot be +/// returned. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub enum SessionJobOutputStreamView { - Stdout, - Stderr, +pub struct SessionJobOutputGapView { + pub offset: u64, + pub omitted_bytes: u64, + pub reason: SessionJobOutputGapReasonView, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] -pub struct SessionJobArtifactView { - pub path: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub kind: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, +pub enum SessionJobOutputGapReasonView { + /// Dropped from the middle because the job's retained output exceeded + /// its cap; the head and the newest bytes are kept. + Overflow, + /// Released after it was delivered. + Expired, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub enum SessionJobOutputStreamView { + Stdout, + Stderr, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -759,12 +1000,16 @@ pub struct EnvironmentJobCreateResponse { #[serde(rename_all = "camelCase")] pub struct EnvironmentJobReadParams { pub jobs: Vec, + /// Most output bytes per job; omit for the server's default page. #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_bytes: Option, + pub max_bytes: Option, + /// Byte offset to read each job's output from: a previous entry's + /// `outputNext`. Omitted with `tailBytes` omitted reads from the start. #[serde(default, skip_serializing_if = "Option::is_none")] - pub after_seq: Option, - #[serde(default)] - pub include_artifacts: bool, + pub after: Option, + /// Read each job's last `tailBytes` of output instead; excludes `after`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tail_bytes: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] diff --git a/crates/api/src/handshake.rs b/crates/api/src/handshake.rs index 33a9c7c2a..c20746ba4 100644 --- a/crates/api/src/handshake.rs +++ b/crates/api/src/handshake.rs @@ -96,8 +96,9 @@ pub struct ServerInfo { pub struct EnvironmentDaemonInfo { pub version: String, pub git_sha: String, - /// Environment protocol version the gateway speaks. A daemon registers - /// only when it reports exactly this number. + /// The envd protocol version (data plane and registration channel) + /// the gateway speaks. A daemon registers only when it reports exactly + /// this number. pub protocol_version: u32, /// Rust target triples the daemon is published for. pub targets: Vec, diff --git a/crates/api/src/ids.rs b/crates/api/src/ids.rs index e6aaeaca3..09425304c 100644 --- a/crates/api/src/ids.rs +++ b/crates/api/src/ids.rs @@ -6,9 +6,7 @@ pub type ItemId = String; pub type SkillId = String; pub type EnvironmentId = String; pub type EnvironmentProviderId = String; -pub type EnvironmentProviderBindingId = String; pub type EnvironmentProvisionRequestId = String; -pub type EnvironmentIncarnationId = String; pub type EnvironmentTemplateId = String; pub type EnvironmentTargetId = String; pub type EnvironmentJobGroupId = String; diff --git a/crates/api/src/rpc.rs b/crates/api/src/rpc.rs index 08c2d47e5..ad40d6161 100644 --- a/crates/api/src/rpc.rs +++ b/crates/api/src/rpc.rs @@ -22,8 +22,8 @@ pub enum AgentApiErrorKind { /// rather than an ordinary "answer this message" failure. SessionBootstrapFailed, /// The environment exists but is not reachable yet: it is still - /// provisioning/booting, or it was powered down and a wake (desired - /// power `running`) has been initiated by this request. Distinct from + /// provisioning, its provider is waking or booting it after a data + /// route dial, or its registered daemon is reconnecting. Distinct from /// `Rejected` so clients retry with backoff instead of failing — /// polling and automation callers lean on this for wake-on-use. EnvironmentNotReady, @@ -395,41 +395,45 @@ api_methods! { METHOD_ENVIRONMENTS_CREDENTIALS_UNBIND => unbind_environment_credential(EnvironmentCredentialUnbindParams) -> EnvironmentCredentialUnbindResponse => ["Unbind an environment credential", "Removes one variable-to-credential mapping without deleting the underlying grant, provider credential, or secret."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_CREATE => create_environment(EnvironmentCreateParams) -> EnvironmentCreateResponse => - ["Create an environment", "Records an idempotent provisioning intent against an enabled universe binding, attributed to the caller. The provider validates its provider-wide template and provisions through its backend asynchronously."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + ["Create an environment", "Creates a provisioned environment, attributed to the caller, from a template of a provider enabled for the universe. Idempotent per requestId: same inputs, same environment; others conflict. Provisioning until the target's first envd handshake, then open; idle overrides the template default."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_READ => read_environment(EnvironmentReadParams) -> EnvironmentReadResponse => - ["Read an environment", "Returns the durable universe resource, source binding, logical lifecycle state, and minimal current-incarnation identity."], access: MethodAccess::Universe(UniverseAction::Read), + ["Read an environment", "Returns one environment: its source, lifecycle state, derived display status, state message, close reason, daemon build, and the latest provider observation."], access: MethodAccess::Universe(UniverseAction::Read), + METHOD_ENVIRONMENTS_UPDATE => update_environment(EnvironmentUpdateParams) -> EnvironmentUpdateResponse => + ["Update an environment", "Replaces the display name and caller metadata of an environment of any kind. Refused for a closed environment."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_LIST => list_environments(EnvironmentListParams) -> EnvironmentListResponse => - ["List environments", "Lists the universe environments, optionally filtered by provider, binding, or logical lifecycle state."], access: MethodAccess::Universe(UniverseAction::Read), + ["List environments", "Lists the universe environments in id order, one page at a time, optionally filtered by provider, registration key, state, or metadata. Closed environments are omitted unless includeClosed is set or state is closed. Continue from nextAfter."], access: MethodAccess::Universe(UniverseAction::Read), METHOD_ENVIRONMENTS_CLOSE => close_environment(EnvironmentCloseParams) -> EnvironmentCloseResponse => - ["Close an environment", "Records an asynchronous idempotent close intent. Provider cleanup is resumed by lifecycle reconciliation; quota is released only after Closed."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + ["Close an environment", "Closes an environment; idempotent. A provisioned environment moves to closing while Lightspeed asks its provider to remove the target, retrying until it is gone, then closed. External and registered environments close at once, and a connected registered daemon is told its environment closed."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_EXTERNAL_CREATE => create_external_environment(EnvironmentExternalCreateParams) -> EnvironmentExternalCreateResponse => - ["Register an external environment", "Creates an environment backed by a Lightspeed-reachable envd WebSocket endpoint, attributed to the caller. Reachability is checked on demand."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + ["Register an external environment", "Creates an open environment backed by a Lightspeed-reachable envd endpoint, attributed to the caller. Idempotent per requestId. Reachability is checked on use."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_INGRESS_PUT => put_environment_ingress(EnvironmentIngressPutParams) -> EnvironmentIngressPutResponse => - ["Configure environment public ingress", "Synchronously enables or disables one provider-authorized HTTPS endpoint for a provisioned environment. The provider owns hostname allocation, the approved guest port, routing, TLS, and health."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + ["Configure environment public ingress", "Synchronously ensures or removes public ingress of an open provisioned environment through its provider; source.publicEndpoint carries the reported endpoint. Rejected when the provider offers no ingress; an internal error, safe to retry, when the provider is unreachable or times out. The provider owns hostname, guest port, routing, TLS, and health."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_POWER_PUT => put_environment_power(EnvironmentPowerPutParams) -> EnvironmentPowerPutResponse => - ["Set environment power intent", "Records the desired power state (running, paused, suspended, or stopped) of a provisioned environment; the lifecycle reconciler converges the provider target asynchronously. Powered-down environments wake transparently on their next use. Rejected when the provider does not support the state."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), - METHOD_ENVIRONMENTS_IDLE_POLICY_PUT => put_environment_idle_policy(EnvironmentIdlePolicyPutParams) -> EnvironmentIdlePolicyPutResponse => - ["Set environment idle policy", "Replaces or clears the staged idle policy of a provisioned environment. The power reaper measures the daemon's idle duration against the pause/suspend/stop/close thresholds and escalates through the stages the provider supports."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + ["Set environment power", "Synchronously moves an open provisioned environment's target to running, paused, suspended, or stopped via its provider and records the observation; the provider may also change power on idle. Rejected without provider power control or when the template lacks the state; an internal error, safe to retry, when the provider is unreachable or times out."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_PROVIDER_BINDINGS_LIST => list_environment_provider_bindings(EnvironmentProviderBindingListParams) -> EnvironmentProviderBindingListResponse => - ["List environment provider bindings", "Lists this universe's revisioned routing and admission bindings to deployment-scoped physical providers."], access: MethodAccess::Universe(UniverseAction::Read), + ["List environment provider bindings", "Lists this universe's bindings to deployment providers and whether each is enabled."], access: MethodAccess::Universe(UniverseAction::Read), METHOD_ENVIRONMENTS_PROVIDER_BINDINGS_READ => read_environment_provider_binding(EnvironmentProviderBindingReadParams) -> EnvironmentProviderBindingReadResponse => - ["Read an environment provider binding", "Returns one universe routing and admission binding. Provider-wide templates and physical resource, network, and ingress policy remain provider-owned."], access: MethodAccess::Universe(UniverseAction::Read), + ["Read an environment provider binding", "Returns this universe's binding to one provider. Templates and physical resource, network, and ingress policy remain provider-owned."], access: MethodAccess::Universe(UniverseAction::Read), METHOD_ENVIRONMENTS_TEMPLATES_LIST => list_environment_templates(EnvironmentTemplateListParams) -> EnvironmentTemplateListResponse => - ["List environment templates", "Reads immutable templates directly from the selected bound provider controller."], access: MethodAccess::Universe(UniverseAction::Read), + ["List environment templates", "Reads templates live from the controllers of the universe's enabled provider bindings, or of one provider: compute, power states, public ingress, and default idle policy."], access: MethodAccess::Universe(UniverseAction::Read), METHOD_ENVIRONMENTS_TEMPLATES_READ => read_environment_template(EnvironmentTemplateReadParams) -> EnvironmentTemplateReadResponse => - ["Read an environment template", "Returns one immutable template version from the selected bound provider controller."], access: MethodAccess::Universe(UniverseAction::Read), + ["Read an environment template", "Returns one template, read live from a bound provider's controller."], access: MethodAccess::Universe(UniverseAction::Read), METHOD_ENVIRONMENTS_JOBS_CREATE => create_environment_jobs(EnvironmentJobCreateParams) -> EnvironmentJobCreateResponse => - ["Create environment jobs", "Starts a dependency-aware job group on one environment instance, injecting the environment's configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance rather than a session. A powered-down environment is woken on use: the call fails with environment_not_ready while the wake is in progress; retry with backoff."], access: MethodAccess::Universe(UniverseAction::UseResource), + ["Create environment jobs", "Starts a dependency-aware job group on one environment instance, injecting its configured credentials at provider start. requestId is the retry identity; jobs are owned by the instance, not a session. A powered-down environment wakes on its data route; the call fails with environment_not_ready while it wakes or reconnects; retry with backoff."], access: MethodAccess::Universe(UniverseAction::UseResource), METHOD_ENVIRONMENTS_JOBS_READ => read_environment_jobs(EnvironmentJobReadParams) -> EnvironmentJobReadResponse => - ["Read environment jobs", "Reads selected job handles with bounded output, optional sequence continuation, and optional artifacts; use returned status/sequence data for polling."], access: MethodAccess::Universe(UniverseAction::Read), + ["Read environment jobs", "Reads selected job handles with bounded output from a byte offset or the output tail; continue from each entry's outputNext and poll on the returned status."], access: MethodAccess::Universe(UniverseAction::Read), METHOD_ENVIRONMENTS_JOBS_CANCEL => cancel_environment_jobs(EnvironmentJobCancelParams) -> EnvironmentJobCancelResponse => ["Cancel environment jobs", "Requests cancellation for selected jobs, optionally including dependents. Force is provider-specific escalation; inspect each per-job result."], access: MethodAccess::Universe(UniverseAction::UseResource), METHOD_ENVIRONMENTS_REGISTRATION_KEYS_CREATE => create_environment_registration_key(EnvironmentRegistrationKeyCreateParams) -> EnvironmentRegistrationKeyCreateResponse => - ["Mint an environment registration key", "Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode, active limit, disconnect grace, and expiry are the key's policy. Treat the secret like a cluster-join credential."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + ["Mint an environment registration key", "Creates a reusable universe-scoped key that lets outbound envd daemons register as environments. The plaintext secret is returned exactly once; only its hash is stored. Identity mode (immutable), active limit, disconnect grace, offline close-after, and expiry are the key's policy. Treat the secret like a cluster-join credential."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_REGISTRATION_KEYS_READ => read_environment_registration_key(EnvironmentRegistrationKeyReadParams) -> EnvironmentRegistrationKeyReadResponse => ["Read an environment registration key", "Returns the key's display prefix, policy, status, and derived environment counts; never the secret or its hash."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_REGISTRATION_KEYS_LIST => list_environment_registration_keys(EnvironmentRegistrationKeyListParams) -> EnvironmentRegistrationKeyListResponse => ["List environment registration keys", "Lists this universe's registration keys with policy, status, and derived counts. Each key is the group of the environments it admitted."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + METHOD_ENVIRONMENTS_REGISTRATION_KEYS_UPDATE => update_environment_registration_key(EnvironmentRegistrationKeyUpdateParams) -> EnvironmentRegistrationKeyUpdateResponse => + ["Update an environment registration key", "Replaces a key's editable policy: display name, active limit, disconnect grace, offline close-after, and expiry; an omitted field is unset. The identity mode is immutable. The active limit and expiry apply to later admissions; a changed disconnect grace or offline close-after applies at the next sweep to environments whose daemon is away."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), + METHOD_ENVIRONMENTS_REGISTRATION_KEYS_ROTATE => rotate_environment_registration_key(EnvironmentRegistrationKeyRotateParams) -> EnvironmentRegistrationKeyRotateResponse => + ["Rotate an environment registration key", "Replaces the key's secret and display prefix and retires the old secret at once. The key keeps its id, group, policy, and the environments it admitted; daemons already registered are unaffected because they never present the key again. Returns the new secret once. Refused for a revoked key."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_ENVIRONMENTS_REGISTRATION_KEYS_REVOKE => revoke_environment_registration_key(EnvironmentRegistrationKeyRevokeParams) -> EnvironmentRegistrationKeyRevokeResponse => ["Revoke an environment registration key", "Stops the key from admitting new daemon identities; already registered daemons keep reconnecting. With closeEnvironments, also closes every non-closed environment the key admitted. Idempotent."], access: MethodAccess::Universe(UniverseAction::ConfigureResource), METHOD_TRANSCRIPTIONS_START => start_transcription(TranscriptionStartParams) -> TranscriptionResponse => diff --git a/crates/api/src/schema_export.rs b/crates/api/src/schema_export.rs index e3f5e6065..57c4f1c5f 100644 --- a/crates/api/src/schema_export.rs +++ b/crates/api/src/schema_export.rs @@ -205,13 +205,13 @@ mod tests { methods.sort_unstable(); methods.dedup(); assert_eq!(methods.len(), total, "duplicate method in manifest"); - assert_eq!(total, 140); + assert_eq!(total, 141); assert_eq!( manifest .iter() .filter(|spec| spec.scope == crate::MethodScope::Deployment) .count(), - 20 + 19 ); } diff --git a/crates/api/src/service.rs b/crates/api/src/service.rs index 34cb75a32..1d59cb0dc 100644 --- a/crates/api/src/service.rs +++ b/crates/api/src/service.rs @@ -222,6 +222,11 @@ pub trait AgentApiService: Send + Sync { params: EnvironmentReadParams, ) -> Result, AgentApiError>; + async fn update_environment( + &self, + params: EnvironmentUpdateParams, + ) -> Result, AgentApiError>; + async fn list_environments( &self, params: EnvironmentListParams, @@ -247,11 +252,6 @@ pub trait AgentApiService: Send + Sync { params: EnvironmentPowerPutParams, ) -> Result, AgentApiError>; - async fn put_environment_idle_policy( - &self, - params: EnvironmentIdlePolicyPutParams, - ) -> Result, AgentApiError>; - async fn list_environment_provider_bindings( &self, _params: EnvironmentProviderBindingListParams, @@ -333,6 +333,16 @@ pub trait AgentApiService: Send + Sync { params: EnvironmentRegistrationKeyListParams, ) -> Result, AgentApiError>; + async fn update_environment_registration_key( + &self, + params: EnvironmentRegistrationKeyUpdateParams, + ) -> Result, AgentApiError>; + + async fn rotate_environment_registration_key( + &self, + params: EnvironmentRegistrationKeyRotateParams, + ) -> Result, AgentApiError>; + async fn revoke_environment_registration_key( &self, params: EnvironmentRegistrationKeyRevokeParams, diff --git a/crates/api/src/tests.rs b/crates/api/src/tests.rs index 7aad6f5e0..5244c66cc 100644 --- a/crates/api/src/tests.rs +++ b/crates/api/src/tests.rs @@ -171,6 +171,22 @@ fn auth_provider_create_params_redact_credential_in_debug_output() { ); } +#[test] +fn environment_provider_put_params_redact_credential_in_debug_output() { + let params: DeploymentEnvironmentProviderPutParams = serde_json::from_value(json!({ + "providerId": "incus-local", + "controllerEndpoint": "wss://incus.example.com/control", + "credential": "provider-bearer-secret" + })) + .expect("deserialize environment provider put params"); + + let debug = format!("{params:?}"); + + assert!(!debug.contains("provider-bearer-secret"), "{debug}"); + assert!(debug.contains("")); + assert_eq!(params.credential.as_deref(), Some("provider-bearer-secret")); +} + #[test] fn request_ids_accept_number_or_string() { let numeric: JsonRpcRequest = serde_json::from_value(json!({ @@ -894,9 +910,9 @@ async fn dispatch_json_rpc_routes_environments_create() { method: METHOD_ENVIRONMENTS_CREATE.to_owned(), params: Some(json!({ "requestId": "request-1", - "bindingId": "primary", + "providerId": "bridge-local", "templateId": "rust-v1", - "resources": { "cpu": 2, "memoryBytes": 2147483648_u64, "diskBytes": 10737418240_u64 } + "idle": { "pauseAfterMs": 60000 } })), }, ) @@ -910,7 +926,7 @@ async fn dispatch_json_rpc_routes_environments_create() { } #[tokio::test(flavor = "current_thread")] -async fn dispatch_json_rpc_routes_environment_power_and_idle_policy_put() { +async fn dispatch_json_rpc_routes_environment_power_put() { let response = dispatch_json_rpc( &TestService, JsonRpcRequest { @@ -925,30 +941,77 @@ async fn dispatch_json_rpc_routes_environment_power_and_idle_policy_put() { .await; assert!(response.error.is_none(), "{:?}", response.error); let environment = &response.result.expect("result")["result"]["environment"]; - assert_eq!(environment["desiredPower"], json!("paused")); + assert_eq!(environment["status"], json!("paused")); assert_eq!( - environment["incarnation"]["powerStates"], - json!(["running", "paused"]) + environment["source"]["observed"], + json!({"phase": "active", "power": "paused", "observedAtMs": 20}) ); + assert!(environment.get("desiredPower").is_none()); +} +#[tokio::test(flavor = "current_thread")] +async fn environment_create_rejects_unknown_idle_fields() { let response = dispatch_json_rpc( &TestService, JsonRpcRequest { - id: RequestId::Number(2), - method: METHOD_ENVIRONMENTS_IDLE_POLICY_PUT.to_owned(), + id: RequestId::Number(1), + method: METHOD_ENVIRONMENTS_CREATE.to_owned(), + params: Some(json!({ + "requestId": "request-1", + "providerId": "bridge-local", + "templateId": "rust-v1", + "idle": { "hibernateAfterMs": 60000 } + })), + }, + ) + .await; + assert_eq!( + response.error.expect("invalid params").code, + JsonRpcError::invalid_params("").code + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn dispatch_json_rpc_routes_environment_update() { + let response = dispatch_json_rpc( + &TestService, + JsonRpcRequest { + id: RequestId::Number(1), + method: METHOD_ENVIRONMENTS_UPDATE.to_owned(), params: Some(json!({ "environmentId": "evi_test", - "idlePolicy": {"pauseAfterMs": 60000, "closeAfterMs": 3600000} + "displayName": "Renamed", + "metadata": {"team": "build"} })), }, ) .await; assert!(response.error.is_none(), "{:?}", response.error); let environment = &response.result.expect("result")["result"]["environment"]; - assert_eq!( - environment["idlePolicy"], - json!({"pauseAfterMs": 60000, "closeAfterMs": 3600000}) - ); + assert_eq!(environment["displayName"], json!("Renamed")); + assert_eq!(environment["metadata"], json!({"team": "build"})); +} + +#[tokio::test(flavor = "current_thread")] +async fn dispatch_json_rpc_routes_environment_list_pages() { + let response = dispatch_json_rpc( + &TestService, + JsonRpcRequest { + id: RequestId::Number(1), + method: METHOD_ENVIRONMENTS_LIST.to_owned(), + params: Some(json!({ + "state": "open", + "includeClosed": true, + "limit": 1, + "after": "evi_a" + })), + }, + ) + .await; + assert!(response.error.is_none(), "{:?}", response.error); + let result = &response.result.expect("result")["result"]; + assert_eq!(result["environments"][0]["state"], json!("open")); + assert_eq!(result["nextAfter"], json!("evi_test")); } #[tokio::test(flavor = "current_thread")] @@ -968,9 +1031,9 @@ async fn dispatch_json_rpc_routes_environment_ingress_put() { assert!(response.error.is_none()); let environment = &response.result.expect("result")["result"]["environment"]; - assert_eq!(environment["publicIngressEnabled"], json!(true)); + assert!(environment.get("publicIngressEnabled").is_none()); assert_eq!( - environment["publicEndpoint"], + environment["source"]["publicEndpoint"], json!("https://opaque.env.example") ); } @@ -2217,7 +2280,14 @@ impl AgentApiService for TestService { &self, params: EnvironmentCreateParams, ) -> Result, AgentApiError> { - assert_eq!(params.binding_id, "primary"); + assert_eq!(params.provider_id, "bridge-local"); + assert_eq!( + params.idle, + Some(EnvironmentIdlePolicyView { + pause_after_ms: Some(60_000), + ..EnvironmentIdlePolicyView::default() + }) + ); Ok(AgentApiOutcome::new(EnvironmentCreateResponse { environment: test_environment_instance(), })) @@ -2232,12 +2302,29 @@ impl AgentApiService for TestService { })) } + async fn update_environment( + &self, + params: EnvironmentUpdateParams, + ) -> Result, AgentApiError> { + let mut environment = test_environment_instance(); + environment.display_name = params.display_name; + environment.metadata = params.metadata; + Ok(AgentApiOutcome::new(EnvironmentUpdateResponse { + environment, + })) + } + async fn list_environments( &self, - _params: EnvironmentListParams, + params: EnvironmentListParams, ) -> Result, AgentApiError> { + let next_after = params + .limit + .is_some() + .then(|| test_environment_instance().environment_id); Ok(AgentApiOutcome::new(EnvironmentListResponse { environments: vec![test_environment_instance()], + next_after, })) } @@ -2264,10 +2351,14 @@ impl AgentApiService for TestService { params: EnvironmentIngressPutParams, ) -> Result, AgentApiError> { let mut environment = test_environment_instance(); - environment.public_ingress_enabled = params.enabled; - environment.public_endpoint = params - .enabled - .then(|| "https://opaque.env.example".to_owned()); + if let EnvironmentSourceView::Provisioned { + public_endpoint, .. + } = &mut environment.source + { + *public_endpoint = params + .enabled + .then(|| "https://opaque.env.example".to_owned()); + } Ok(AgentApiOutcome::new(EnvironmentIngressPutResponse { environment, })) @@ -2277,24 +2368,22 @@ impl AgentApiService for TestService { &self, params: EnvironmentPowerPutParams, ) -> Result, AgentApiError> { + assert_eq!(params.power, EnvironmentPowerStateView::Paused); let mut environment = test_environment_instance(); - environment.desired_power = params.power; + environment.status = EnvironmentStatusView::Paused; + if let EnvironmentSourceView::Provisioned { observed, .. } = &mut environment.source { + *observed = Some(EnvironmentObservedTargetView { + phase: EnvironmentTargetPhaseView::Active, + power: Some(params.power), + message: None, + observed_at_ms: 20, + }); + } Ok(AgentApiOutcome::new(EnvironmentPowerPutResponse { environment, })) } - async fn put_environment_idle_policy( - &self, - params: EnvironmentIdlePolicyPutParams, - ) -> Result, AgentApiError> { - let mut environment = test_environment_instance(); - environment.idle_policy = params.idle_policy; - Ok(AgentApiOutcome::new(EnvironmentIdlePolicyPutResponse { - environment, - })) - } - async fn activate_session_environment( &self, params: SessionEnvironmentActivateParams, @@ -2423,6 +2512,36 @@ impl AgentApiService for TestService { )) } + async fn update_environment_registration_key( + &self, + params: EnvironmentRegistrationKeyUpdateParams, + ) -> Result, AgentApiError> { + let mut key = test_registration_key(); + key.display_name = params.display_name; + key.max_active_environments = params.max_active_environments; + key.offline_close_after_ms = params.offline_close_after_ms; + Ok(AgentApiOutcome::new( + EnvironmentRegistrationKeyUpdateResponse { + registration_key: key, + }, + )) + } + + async fn rotate_environment_registration_key( + &self, + params: EnvironmentRegistrationKeyRotateParams, + ) -> Result, AgentApiError> { + assert_eq!(params.registration_key_id, "rk_harbor"); + let mut key = test_registration_key(); + key.key_prefix = "lsrk_rotated".to_owned(); + Ok(AgentApiOutcome::new( + EnvironmentRegistrationKeyRotateResponse { + registration_key: key, + secret: EnvironmentRegistrationSecretView("lsrk_rotated-secret".to_owned()), + }, + )) + } + async fn revoke_environment_registration_key( &self, params: EnvironmentRegistrationKeyRevokeParams, @@ -3237,33 +3356,30 @@ fn test_environment_credential( fn test_environment_instance() -> EnvironmentView { EnvironmentView { environment_id: "evi_test".to_owned(), - request_id: "request-1".to_owned(), + request_id: Some("request-1".to_owned()), source: EnvironmentSourceView::Provisioned { provider_id: "bridge-local".to_owned(), - binding_id: "primary".to_owned(), + template_id: "rust-v1".to_owned(), + target_id: Some("local".to_owned()), + idle: None, + observed: None, + public_endpoint: None, }, display_name: Some("Local".to_owned()), - status: EnvironmentLifecycleStatusView::Ready, - desired_power: EnvironmentPowerStateView::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationView { - incarnation_id: "incarnation-1".to_owned(), - provision_request_id: Some("request-1".to_owned()), - provider_target_id: Some("local".to_owned()), - template_id: Some("rust-v1".to_owned()), - power_states: vec![ - EnvironmentPowerStateView::Running, - EnvironmentPowerStateView::Paused, - ], - created_at_ms: 10, - updated_at_ms: 10, - }, - public_ingress_enabled: false, - public_endpoint: None, + state: EnvironmentStateView::Open, + status: EnvironmentStatusView::Ready, + state_message: None, + close_reason: None, + state_changed_at_ms: 10, + daemon: Some(EnvironmentDaemonView { + version: Some("0.1.0".to_owned()), + git_sha: None, + protocol_version: 4, + }), metadata: BTreeMap::new(), - last_seen_at_ms: None, created_at_ms: 10, updated_at_ms: 10, + closed_at_ms: None, } } @@ -3275,16 +3391,57 @@ fn test_registration_key() -> EnvironmentRegistrationKeyView { identity_mode: EnvironmentIdentityModeView::Ephemeral, max_active_environments: Some(8), ephemeral_disconnect_grace_ms: 300_000, + offline_close_after_ms: None, expires_at_ms: None, status: EnvironmentRegistrationKeyStatusView::Active, registered_environment_count: 1, active_environment_count: 1, last_registered_at_ms: Some(10), created_at_ms: 5, + updated_at_ms: 5, revoked_at_ms: None, } } +#[tokio::test(flavor = "current_thread")] +async fn dispatch_json_rpc_routes_registration_key_update_and_rotate() { + let response = dispatch_json_rpc( + &TestService, + JsonRpcRequest { + id: RequestId::Number(1), + method: METHOD_ENVIRONMENTS_REGISTRATION_KEYS_UPDATE.to_owned(), + params: Some(json!({ + "registrationKeyId": "rk_harbor", + "displayName": "harbor-fleet", + "offlineCloseAfterMs": 86400000 + })), + }, + ) + .await; + assert!(response.error.is_none(), "{:?}", response.error); + let key = &response.result.expect("result")["result"]["registrationKey"]; + assert_eq!(key["displayName"], json!("harbor-fleet")); + assert_eq!(key["offlineCloseAfterMs"], json!(86_400_000)); + assert!(key.get("maxActiveEnvironments").is_none()); + + let response = dispatch_json_rpc( + &TestService, + JsonRpcRequest { + id: RequestId::Number(2), + method: METHOD_ENVIRONMENTS_REGISTRATION_KEYS_ROTATE.to_owned(), + params: Some(json!({ "registrationKeyId": "rk_harbor" })), + }, + ) + .await; + assert!(response.error.is_none(), "{:?}", response.error); + let result = &response.result.expect("result")["result"]; + assert_eq!( + result["registrationKey"]["keyPrefix"], + json!("lsrk_rotated") + ); + assert_eq!(result["secret"], json!("lsrk_rotated-secret")); +} + #[test] fn registration_secret_views_redact_debug_output() { let response = EnvironmentRegistrationKeyCreateResponse { @@ -3301,32 +3458,21 @@ fn registration_secret_views_redact_debug_output() { fn test_external_environment() -> EnvironmentView { EnvironmentView { environment_id: "evi_enrolled".to_owned(), - request_id: "request-enrolled".to_owned(), + request_id: Some("request-enrolled".to_owned()), source: EnvironmentSourceView::External { - connection: EnvironmentConnectionView { - endpoint: "ws://envd.test:19091".to_owned(), - transport: EnvironmentConnectionTransportView::WebSocket, - }, + endpoint: "ws://envd.test:19091".to_owned(), }, display_name: Some("External".to_owned()), - status: EnvironmentLifecycleStatusView::Ready, - desired_power: EnvironmentPowerStateView::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationView { - incarnation_id: "incarnation-enrolled".to_owned(), - provision_request_id: None, - provider_target_id: None, - template_id: None, - power_states: Vec::new(), - created_at_ms: 10, - updated_at_ms: 10, - }, - public_ingress_enabled: false, - public_endpoint: None, + state: EnvironmentStateView::Open, + status: EnvironmentStatusView::Ready, + state_message: None, + close_reason: None, + state_changed_at_ms: 10, + daemon: None, metadata: BTreeMap::new(), - last_seen_at_ms: None, created_at_ms: 10, updated_at_ms: 10, + closed_at_ms: None, } } @@ -3403,11 +3549,10 @@ fn test_deployment_environment_provider(provider_id: &str) -> DeploymentEnvironm DeploymentEnvironmentProviderView { provider_id: provider_id.to_owned(), display_name: Some("Local Incus".to_owned()), - controller_connection: DeploymentEnvironmentProviderConnection { - endpoint: "ws://127.0.0.1:19090/control".to_owned(), - transport: DeploymentEnvironmentProviderTransport::WebSocket, - }, - metadata: BTreeMap::new(), + controller_endpoint: "ws://127.0.0.1:19090/control".to_owned(), + data_route_base: None, + effective_data_route_base: Some("ws://127.0.0.1:19090/routes".to_owned()), + has_credential: false, created_at_ms: 10, updated_at_ms: 20, } @@ -3550,45 +3695,87 @@ impl DeploymentApiService for TestDeploymentService { )) } - async fn adopt_environment( + async fn put_environment_provider_binding( &self, - params: DeploymentEnvironmentAdoptParams, - ) -> Result, AgentApiError> { - let mut environment = test_environment_instance(); - environment.request_id = params.request_id; - environment.display_name = params.display_name; - environment.incarnation.template_id = None; - Ok(AgentApiOutcome::new(DeploymentEnvironmentAdoptResponse { - environment, + params: DeploymentProviderBindingPutParams, + ) -> Result, AgentApiError> { + Ok(AgentApiOutcome::new(DeploymentProviderBindingPutResponse { + binding: EnvironmentProviderBindingView { + provider_id: params.provider_id, + enabled: params.enabled, + created_at_ms: 10, + updated_at_ms: 20, + }, })) } + + async fn delete_environment_provider_binding( + &self, + params: DeploymentProviderBindingDeleteParams, + ) -> Result, AgentApiError> { + Ok(AgentApiOutcome::new( + DeploymentProviderBindingDeleteResponse { + binding: EnvironmentProviderBindingView { + provider_id: params.provider_id, + enabled: false, + created_at_ms: 10, + updated_at_ms: 20, + }, + }, + )) + } } #[tokio::test(flavor = "current_thread")] async fn deployment_environment_provider_methods_dispatch() { + let universe_id = "6f3a1a52-58c1-4f0e-9c2d-1a2b3c4d5e6f"; let requests = [ ( METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_PUT, json!({ "providerId": "incus-local", "displayName": "Local Incus", - "controllerConnection": { - "endpoint": "ws://127.0.0.1:19090/control", - "transport": { "type": "webSocket" } - } + "controllerEndpoint": "ws://127.0.0.1:19090/control" }), + "provider", + ), + ( + METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_PUT, + json!({ + "providerId": "incus-local", + "controllerEndpoint": "unix:///run/incus-provider.sock", + "dataRouteBase": "ws://127.0.0.1:19090/routes", + "clearCredential": true + }), + "provider", + ), + ( + METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_LIST, + json!({}), + "providers", ), - (METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_LIST, json!({})), ( METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_READ, json!({ "providerId": "incus-local" }), + "provider", ), ( METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_DELETE, json!({ "providerId": "incus-local" }), + "provider", + ), + ( + METHOD_DEPLOYMENT_PROVIDER_BINDINGS_PUT, + json!({ "universeId": universe_id, "providerId": "incus-local", "enabled": true }), + "binding", + ), + ( + METHOD_DEPLOYMENT_PROVIDER_BINDINGS_DELETE, + json!({ "universeId": universe_id, "providerId": "incus-local" }), + "binding", ), ]; - for (index, (method, params)) in requests.into_iter().enumerate() { + for (index, (method, params, field)) in requests.into_iter().enumerate() { let response = dispatch_deployment_json_rpc( &TestDeploymentService, JsonRpcRequest { @@ -3601,40 +3788,33 @@ async fn deployment_environment_provider_methods_dispatch() { assert!(response.error.is_none(), "{method}: {:?}", response.error); assert!( response.result.expect("result")["result"] - .get(if method.ends_with("/list") { - "providers" - } else { - "provider" - }) + .get(field) .is_some(), "{method}" ); } } -#[tokio::test(flavor = "current_thread")] -async fn deployment_environment_adoption_dispatches_explicit_ownership_transfer() { - let response = dispatch_deployment_json_rpc( - &TestDeploymentService, - JsonRpcRequest { - id: RequestId::Number(1), - method: METHOD_DEPLOYMENT_ENVIRONMENTS_ADOPT.to_owned(), - params: Some(json!({ - "universeId": "6f3a1a52-58c1-4f0e-9c2d-1a2b3c4d5e6f", - "requestId": "adopt-1", - "bindingId": "primary", - "sourceTarget": "legacy/hand-built-vm", - "takeOwnership": true, - "displayName": "Imported VM" - })), - }, - ) - .await; - assert!(response.error.is_none(), "{:?}", response.error); - let environment = &response.result.expect("result")["result"]["environment"]; - assert_eq!(environment["requestId"], "adopt-1"); - assert_eq!(environment["displayName"], "Imported VM"); - assert!(environment["incarnation"].get("templateId").is_none()); +#[test] +fn environment_provider_put_params_redact_credential_and_view_omits_it() { + let params: DeploymentEnvironmentProviderPutParams = serde_json::from_value(json!({ + "providerId": "incus-local", + "controllerEndpoint": "wss://provider.example/control", + "credential": "provider-bearer-secret" + })) + .expect("put params"); + assert!(!params.clear_credential); + assert!(params.data_route_base.is_none()); + assert!(!format!("{params:?}").contains("provider-bearer-secret")); + let view = serde_json::to_value(test_deployment_environment_provider("incus-local")) + .expect("serialize view"); + assert_eq!(view["hasCredential"], json!(false)); + assert!(view.get("credential").is_none()); + assert!(view.get("dataRouteBase").is_none()); + assert_eq!( + view["effectiveDataRouteBase"], + json!("ws://127.0.0.1:19090/routes") + ); } #[tokio::test(flavor = "current_thread")] diff --git a/crates/cli/src/api_client.rs b/crates/cli/src/api_client.rs index 85d34cfb6..e530ca843 100644 --- a/crates/cli/src/api_client.rs +++ b/crates/cli/src/api_client.rs @@ -15,8 +15,7 @@ use api::{ BlobPutParams, BlobPutResponse, BlobReadParams, BlobReadResponse, EnvironmentCloseParams, EnvironmentCloseResponse, EnvironmentCredentialBindParams, EnvironmentCredentialBindResponse, EnvironmentCredentialListParams, EnvironmentCredentialListResponse, - EnvironmentCredentialUnbindParams, EnvironmentCredentialUnbindResponse, - EnvironmentIdlePolicyPutParams, EnvironmentIdlePolicyPutResponse, EnvironmentListParams, + EnvironmentCredentialUnbindParams, EnvironmentCredentialUnbindResponse, EnvironmentListParams, EnvironmentListResponse, EnvironmentPowerPutParams, EnvironmentPowerPutResponse, EnvironmentProviderBindingListParams, EnvironmentProviderBindingListResponse, EnvironmentReadParams, EnvironmentReadResponse, JsonRpcRequest, JsonRpcResponse, @@ -28,7 +27,7 @@ use api::{ METHOD_AUTH_PROVIDERS_LIST, METHOD_AUTH_PROVIDERS_READ, METHOD_BLOBS_HAS, METHOD_BLOBS_PUT, METHOD_BLOBS_READ, METHOD_ENVIRONMENTS_CLOSE, METHOD_ENVIRONMENTS_CREDENTIALS_BIND, METHOD_ENVIRONMENTS_CREDENTIALS_LIST, METHOD_ENVIRONMENTS_CREDENTIALS_UNBIND, - METHOD_ENVIRONMENTS_IDLE_POLICY_PUT, METHOD_ENVIRONMENTS_LIST, METHOD_ENVIRONMENTS_POWER_PUT, + METHOD_ENVIRONMENTS_LIST, METHOD_ENVIRONMENTS_POWER_PUT, METHOD_ENVIRONMENTS_PROVIDER_BINDINGS_LIST, METHOD_ENVIRONMENTS_READ, METHOD_MCP_SERVERS_DELETE, METHOD_MCP_SERVERS_LIST, METHOD_MCP_SERVERS_PUT, METHOD_MCP_SERVERS_READ, METHOD_MODELS_LIST, METHOD_PROFILES_DELETE, METHOD_PROFILES_LIST, @@ -578,12 +577,11 @@ impl HttpAgentApi { self.request(METHOD_ENVIRONMENTS_POWER_PUT, params).await } - pub(crate) async fn put_environment_idle_policy( + pub(crate) async fn update_environment( &self, - params: EnvironmentIdlePolicyPutParams, - ) -> Result, AgentApiError> { - self.request(METHOD_ENVIRONMENTS_IDLE_POLICY_PUT, params) - .await + params: api::EnvironmentUpdateParams, + ) -> Result, AgentApiError> { + self.request(api::METHOD_ENVIRONMENTS_UPDATE, params).await } pub(crate) async fn create_environment_registration_key( @@ -610,6 +608,22 @@ impl HttpAgentApi { .await } + pub(crate) async fn update_environment_registration_key( + &self, + params: api::EnvironmentRegistrationKeyUpdateParams, + ) -> Result, AgentApiError> { + self.request(api::METHOD_ENVIRONMENTS_REGISTRATION_KEYS_UPDATE, params) + .await + } + + pub(crate) async fn rotate_environment_registration_key( + &self, + params: api::EnvironmentRegistrationKeyRotateParams, + ) -> Result, AgentApiError> { + self.request(api::METHOD_ENVIRONMENTS_REGISTRATION_KEYS_ROTATE, params) + .await + } + pub(crate) async fn revoke_environment_registration_key( &self, params: api::EnvironmentRegistrationKeyRevokeParams, diff --git a/crates/cli/src/env_cli.rs b/crates/cli/src/env_cli.rs index e27cbf9fd..c405533ae 100644 --- a/crates/cli/src/env_cli.rs +++ b/crates/cli/src/env_cli.rs @@ -14,14 +14,14 @@ enum EnvCommand { /// Administer deployment-wide environment provider controllers (deployment key). #[command(visible_alias = "providers")] Provider(ProviderArgs), - /// Provision an environment using an immutable template on a bound provider. + /// Provision an environment from a provider template. Create(CreateArgs), /// Register an existing, Lightspeed-reachable envd WebSocket endpoint. Register(RegisterArgs), - /// Inspect the current universe's routing bindings to environment providers. + /// Inspect or set the current universe's bindings to environment providers. #[command(visible_alias = "bindings")] Binding(BindingArgs), - /// Browse immutable templates offered by bound environment providers. + /// Browse templates offered by the universe's enabled providers. #[command(visible_alias = "templates")] Template(TemplateArgs), /// Enable or disable provider-authorized public HTTPS ingress. @@ -30,19 +30,19 @@ enum EnvCommand { List(EnvListArgs), /// Read one universe environment. Read(EnvironmentResourceArgs), + /// Replace the display name and metadata of one universe environment. + Update(EnvUpdateArgs), /// Close one universe environment. Close(EnvironmentResourceArgs), - /// Set the desired power state (running, paused, suspended, stopped) of - /// a provisioned environment; the runtime converges it asynchronously and - /// a powered-down environment wakes on its next use. + /// Move a provisioned environment to a power state (running, paused, + /// suspended, stopped) through its provider; the provider owns every + /// later transition, and a powered-down environment wakes when used. Power(PowerArgs), - /// Replace or clear the staged idle policy of a provisioned environment. - IdlePolicy(IdlePolicyArgs), /// Bind, list, or unbind universe environment credentials. #[command(name = "credential", visible_alias = "credentials")] Credentials(CredentialArgs), - /// Mint, list, read, or revoke registration keys that let outbound - /// `lightspeed-envd` daemons register as environments. + /// Mint, list, read, update, rotate, or revoke registration keys that let + /// outbound `lightspeed-envd` daemons register as environments. #[command(name = "registration-key", visible_alias = "registration-keys")] RegistrationKeys(RegistrationKeyArgs), } @@ -61,6 +61,11 @@ enum RegistrationKeyCommand { List(ResourceArgs), /// Read one registration key. Read(RegistrationKeyResourceArgs), + /// Replace a key's editable policy; unset flags clear their setting. + Update(RegistrationKeyUpdateArgs), + /// Retire the key's live secret and mint a new one, printed exactly + /// once. Daemons it already admitted keep reconnecting. + Rotate(RegistrationKeyResourceArgs), /// Stop a key from admitting new daemons; optionally close its /// environments. Revoke(RegistrationKeyRevokeArgs), @@ -72,24 +77,70 @@ struct RegistrationKeyCreateArgs { api_url: String, #[arg(long)] json: bool, + /// persistent: environments wait for their daemon to reconnect; + /// ephemeral: they close once their daemon has been away longer than the + /// grace. Immutable once the key exists. + #[arg(long, value_enum, default_value_t = IdentityModeArg::Persistent)] + mode: IdentityModeArg, + #[command(flatten)] + policy: RegistrationKeyPolicyArgs, +} + +#[derive(Args, Debug, Clone)] +struct RegistrationKeyPolicyArgs { /// Group name shown for every environment this key admits. #[arg(long)] name: String, - /// persistent: environments stay offline until closed; ephemeral: they - /// close once their daemon has been away longer than the grace. - #[arg(long, value_enum, default_value_t = IdentityModeArg::Persistent)] - mode: IdentityModeArg, /// Non-closed environments the key may have at once. #[arg(long = "max-active")] max_active: Option, /// Ephemeral disconnect grace in minutes (default: 5). #[arg(long = "grace-min")] grace_min: Option, + /// Persistent keys only: close an environment whose daemon has been + /// away longer than this many minutes. + #[arg(long = "offline-close-after-min")] + offline_close_after_min: Option, /// Stop admitting new daemons after this many hours. #[arg(long = "expires-in-hours")] expires_in_hours: Option, } +impl RegistrationKeyPolicyArgs { + fn expires_at_ms(&self) -> Result> { + let Some(hours) = self.expires_in_hours else { + return Ok(None); + }; + let now_ms = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), + )?; + Ok(Some( + now_ms.saturating_add( + i64::try_from(hours) + .unwrap_or(i64::MAX) + .saturating_mul(3_600_000), + ), + )) + } +} + +#[derive(Args, Debug, Clone)] +struct RegistrationKeyUpdateArgs { + #[arg(skip)] + api_url: String, + #[arg(long)] + json: bool, + registration_key_id: String, + #[command(flatten)] + policy: RegistrationKeyPolicyArgs, +} + +fn minutes_ms(minutes: Option) -> Option { + minutes.map(|minutes| minutes.saturating_mul(60_000)) +} + #[derive(clap::ValueEnum, Debug, Clone, Copy)] enum IdentityModeArg { Persistent, @@ -142,27 +193,6 @@ impl From for api::EnvironmentPowerStateView { } } -#[derive(Args, Debug, Clone)] -struct IdlePolicyArgs { - #[command(flatten)] - common: EnvironmentResourceArgs, - /// Pause after this many minutes idle. - #[arg(long = "pause-after-min")] - pause_after_min: Option, - /// Suspend after this many minutes idle (providers that support it). - #[arg(long = "suspend-after-min")] - suspend_after_min: Option, - /// Stop after this many minutes idle. - #[arg(long = "stop-after-min")] - stop_after_min: Option, - /// Close after this many minutes idle. - #[arg(long = "close-after-min")] - close_after_min: Option, - /// Remove the idle policy entirely. - #[arg(long, conflicts_with_all = ["pause_after_min", "suspend_after_min", "stop_after_min", "close_after_min"])] - clear: bool, -} - #[derive(Args, Debug, Clone)] struct ResourceArgs { #[arg(skip)] @@ -176,15 +206,34 @@ struct EnvListArgs { #[arg(long)] provider: Option, #[arg(long)] - binding: Option, - #[arg(long)] registration_key: Option, + /// Also list closed environments. + #[arg(long)] + include_closed: bool, + /// Most environments per page (1-500; default 100). + #[arg(long)] + limit: Option, + /// Start after this environment id: a previous page's `nextAfter`. + #[arg(long)] + after: Option, #[command(flatten)] common: ResourceArgs, #[command(flatten)] metadata: crate::session_cli::MetadataPairs, } +#[derive(Args, Debug, Clone)] +struct EnvUpdateArgs { + #[command(flatten)] + common: EnvironmentResourceArgs, + /// The new display name; omit to clear it. + #[arg(long)] + name: Option, + /// The complete new metadata; omit every pair to clear it. + #[command(flatten)] + metadata: crate::session_cli::MetadataPairs, +} + #[derive(Args, Debug, Clone)] struct EnvironmentResourceArgs { #[arg(skip)] @@ -248,9 +297,9 @@ pub(crate) async fn handle(args: EnvArgs) -> Result<()> { EnvCommand::Ingress(args) => ingress(args).await, EnvCommand::List(args) => list(args).await, EnvCommand::Read(args) => read(args).await, + EnvCommand::Update(args) => update(args).await, EnvCommand::Close(args) => close(args).await, EnvCommand::Power(args) => power(args).await, - EnvCommand::IdlePolicy(args) => idle_policy(args).await, EnvCommand::Credentials(args) => credentials(args).await, EnvCommand::RegistrationKeys(args) => registration_keys(args).await, } @@ -259,45 +308,54 @@ pub(crate) async fn handle(args: EnvArgs) -> Result<()> { async fn registration_keys(args: RegistrationKeyArgs) -> Result<()> { match args.command { RegistrationKeyCommand::Create(args) => { - let now_ms = i64::try_from( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH)? - .as_millis(), - )?; + let expires_at_ms = args.policy.expires_at_ms()?; let response = HttpAgentApi::new(args.api_url) .create_environment_registration_key(api::EnvironmentRegistrationKeyCreateParams { - display_name: args.name, + display_name: args.policy.name, identity_mode: match args.mode { IdentityModeArg::Persistent => api::EnvironmentIdentityModeView::Persistent, IdentityModeArg::Ephemeral => api::EnvironmentIdentityModeView::Ephemeral, }, - max_active_environments: args.max_active, - ephemeral_disconnect_grace_ms: args - .grace_min - .map(|minutes| minutes.saturating_mul(60_000)), - expires_at_ms: args.expires_in_hours.map(|hours| { - now_ms.saturating_add( - i64::try_from(hours) - .unwrap_or(i64::MAX) - .saturating_mul(3_600_000), - ) - }), + max_active_environments: args.policy.max_active, + ephemeral_disconnect_grace_ms: minutes_ms(args.policy.grace_min), + offline_close_after_ms: minutes_ms(args.policy.offline_close_after_min), + expires_at_ms, }) .await .map_err(crate::api_client::api_error)? .result; print_json_or(args.json, &response, || { - print_registration_key(&response.registration_key); - println!(); - println!("Registration key (shown once, store it now):"); - println!(" {}", response.secret.0); - println!(); - println!("Bootstrap a daemon with:"); - println!( - " LIGHTSPEED_ENVD_GATEWAY_URL=wss:///environment-gateway/connect" - ); - println!(" LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE="); - println!(" lightspeed-envd"); + print_registration_secret(&response.registration_key, &response.secret.0) + }) + } + RegistrationKeyCommand::Update(args) => { + let expires_at_ms = args.policy.expires_at_ms()?; + let response = HttpAgentApi::new(args.api_url) + .update_environment_registration_key(api::EnvironmentRegistrationKeyUpdateParams { + registration_key_id: args.registration_key_id, + display_name: args.policy.name, + max_active_environments: args.policy.max_active, + ephemeral_disconnect_grace_ms: minutes_ms(args.policy.grace_min), + offline_close_after_ms: minutes_ms(args.policy.offline_close_after_min), + expires_at_ms, + }) + .await + .map_err(crate::api_client::api_error)? + .result; + print_json_or(args.json, &response, || { + print_registration_key(&response.registration_key) + }) + } + RegistrationKeyCommand::Rotate(args) => { + let response = HttpAgentApi::new(args.api_url) + .rotate_environment_registration_key(api::EnvironmentRegistrationKeyRotateParams { + registration_key_id: args.registration_key_id, + }) + .await + .map_err(crate::api_client::api_error)? + .result; + print_json_or(args.json, &response, || { + print_registration_secret(&response.registration_key, &response.secret.0) }) } RegistrationKeyCommand::List(args) => { @@ -350,6 +408,18 @@ fn print_registration_key(key: &api::EnvironmentRegistrationKeyView) { crate::output::show(false, key).expect("serializable registration key metadata"); } +fn print_registration_secret(key: &api::EnvironmentRegistrationKeyView, secret: &str) { + print_registration_key(key); + println!(); + println!("Registration key (shown once, store it now):"); + println!(" {secret}"); + println!(); + println!("Bootstrap a daemon with:"); + println!(" LIGHTSPEED_ENVD_GATEWAY_URL=wss:///environment-gateway/connect"); + println!(" LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE="); + println!(" lightspeed-envd"); +} + async fn power(args: PowerArgs) -> Result<()> { let response = HttpAgentApi::new(args.common.api_url) .put_environment_power(api::EnvironmentPowerPutParams { @@ -362,46 +432,24 @@ async fn power(args: PowerArgs) -> Result<()> { if args.common.json { crate::output::show(true, &response) } else { - println!("Power change requested; the runtime converges it asynchronously."); crate::output::show( false, - &serde_json::json!({"environmentId":response.environment.environment_id,"desiredPower":response.environment.desired_power,"observedStatus":response.environment.status}), + &serde_json::json!({"environmentId":response.environment.environment_id,"status":response.environment.status}), ) } } -async fn idle_policy(args: IdlePolicyArgs) -> Result<()> { - let minutes = |value: Option| value.map(|minutes| minutes.saturating_mul(60_000)); - let idle_policy = if args.clear { - None - } else { - let policy = api::EnvironmentIdlePolicyView { - pause_after_ms: minutes(args.pause_after_min), - suspend_after_ms: minutes(args.suspend_after_min), - stop_after_ms: minutes(args.stop_after_min), - close_after_ms: minutes(args.close_after_min), - }; - if policy == api::EnvironmentIdlePolicyView::default() { - anyhow::bail!("set at least one --*-after-min stage or pass --clear"); - } - Some(policy) - }; +async fn update(args: EnvUpdateArgs) -> Result<()> { let response = HttpAgentApi::new(args.common.api_url) - .put_environment_idle_policy(api::EnvironmentIdlePolicyPutParams { + .update_environment(api::EnvironmentUpdateParams { environment_id: args.common.environment_id, - idle_policy, + display_name: args.name, + metadata: args.metadata.map(), }) .await .map_err(crate::api_client::api_error)? .result; - if args.common.json { - crate::output::show(true, &response) - } else { - crate::output::show( - false, - &serde_json::json!({"environmentId":response.environment.environment_id,"idlePolicy":response.environment.idle_policy}), - ) - } + crate::output::show(args.common.json, &response.environment) } async fn list(args: EnvListArgs) -> Result<()> { @@ -409,8 +457,10 @@ async fn list(args: EnvListArgs) -> Result<()> { .list_environments(api::EnvironmentListParams { metadata: args.metadata.map(), provider_id: args.provider, - binding_id: args.binding, registration_key_id: args.registration_key, + include_closed: args.include_closed, + limit: args.limit, + after: args.after, ..Default::default() }) .await @@ -425,10 +475,14 @@ async fn list(args: EnvListArgs) -> Result<()> { ("environmentId", "ID"), ("displayName", "NAME"), ("status", "STATUS"), - ("desiredPower", "DESIRED POWER"), + ("state", "STATE"), ], "No environments found.", - ) + )?; + if let Some(after) = &response.next_after { + println!("More environments: pass --after {after}"); + } + Ok(()) } } @@ -451,10 +505,14 @@ async fn close(args: EnvironmentResourceArgs) -> Result<()> { .map_err(crate::api_client::api_error)? .result; print_json_or(args.json, &response, || { - println!( - "Close requested for {}. Use environment read to inspect completion.", - response.environment.environment_id - ) + if response.environment.state == api::EnvironmentStateView::Closed { + println!("Closed {}.", response.environment.environment_id); + } else { + println!( + "Closing {}; the provider removes its target. Use environment read to inspect completion.", + response.environment.environment_id + ); + } }) } @@ -540,8 +598,10 @@ pub(crate) fn print_json_or( struct CreateArgs { #[command(flatten)] common: ResourceArgs, + /// The provider to create the environment with; the universe needs an + /// enabled binding to it. #[arg(long)] - binding: String, + provider: String, #[arg(long)] template: String, /// Supply the same request ID when retrying an uncertain creation. @@ -556,7 +616,7 @@ struct CreateArgs { struct RegisterArgs { #[command(flatten)] common: ResourceArgs, - /// A ws:// or wss:// endpoint reachable from the runtime. + /// A ws://, wss:// or unix:// endpoint of envd reachable from the runtime. endpoint: String, /// Supply the same request ID when retrying registration. #[arg(long)] @@ -573,17 +633,30 @@ struct BindingArgs { } #[derive(Subcommand, Debug, Clone)] enum BindingCommand { - /// Create or replace a universe binding from DeploymentProviderBindingPutParams JSON (deployment key). - Put(DocumentArgs), - /// Delete a binding from the selected universe (deployment key). + /// Bind the selected universe to a provider, enabled or disabled + /// (deployment key). + Put { + provider_id: String, + #[arg( + long, + conflicts_with = "disabled", + required_unless_present = "disabled" + )] + enabled: bool, + #[arg(long)] + disabled: bool, + #[arg(long)] + json: bool, + }, + /// Delete the selected universe's binding to a provider (deployment key). Delete { - binding_id: String, + provider_id: String, #[arg(long)] json: bool, }, List(ResourceArgs), Read { - binding_id: String, + provider_id: String, #[arg(long)] json: bool, }, @@ -596,15 +669,16 @@ struct TemplateArgs { #[derive(Subcommand, Debug, Clone)] enum TemplateCommand { List { + /// Only this provider's templates; omit for every enabled binding. #[arg(long)] - binding: Option, + provider: Option, #[arg(long)] json: bool, }, Read { template_id: String, #[arg(long)] - binding: String, + provider: String, #[arg(long)] json: bool, }, @@ -624,11 +698,11 @@ async fn create(args: CreateArgs) -> Result<()> { api::METHOD_ENVIRONMENTS_CREATE, api::EnvironmentCreateParams { request_id: args.request_id, - binding_id: args.binding, + provider_id: args.provider, template_id: args.template, display_name: args.display_name, metadata: args.metadata.map(), - idle_policy: None, + idle: None, }, ) .await? @@ -639,20 +713,18 @@ async fn create(args: CreateArgs) -> Result<()> { crate::output::show(args.common.json, &response.environment) } async fn register(args: RegisterArgs) -> Result<()> { - let endpoint = reqwest::Url::parse(&args.endpoint)?; anyhow::ensure!( - matches!(endpoint.scheme(), "ws" | "wss"), - "envd endpoint must use ws:// or wss://" + ["ws://", "wss://", "unix://"] + .iter() + .any(|scheme| args.endpoint.starts_with(scheme)), + "envd endpoint must use ws://, wss:// or unix://" ); let response: api::EnvironmentExternalCreateResponse = HttpAgentApi::new(args.common.api_url) .request( api::METHOD_ENVIRONMENTS_EXTERNAL_CREATE, api::EnvironmentExternalCreateParams { request_id: args.request_id, - connection: api::EnvironmentConnectionView { - endpoint: args.endpoint, - transport: api::EnvironmentConnectionTransportView::WebSocket, - }, + endpoint: args.endpoint, display_name: args.display_name, metadata: args.metadata.map(), }, @@ -664,25 +736,32 @@ async fn register(args: RegisterArgs) -> Result<()> { async fn binding(args: BindingArgs) -> Result<()> { let client = HttpAgentApi::new(""); match args.command { - BindingCommand::Put(args) => { - let params: api::DeploymentProviderBindingPutParams = read_document(&args.file)?; + BindingCommand::Put { + provider_id, + enabled, + disabled: _, + json, + } => { let response: api::DeploymentProviderBindingPutResponse = client - .request(api::METHOD_DEPLOYMENT_PROVIDER_BINDINGS_PUT, params) + .request( + api::METHOD_DEPLOYMENT_PROVIDER_BINDINGS_PUT, + api::DeploymentProviderBindingPutParams { + universe_id: selected_universe_id("binding a provider")?, + provider_id, + enabled, + }, + ) .await? .result; - crate::output::show(args.json, &response.binding) + crate::output::show(json, &response.binding) } - BindingCommand::Delete { binding_id, json } => { - let universe_id = crate::connection::ACTIVE - .get() - .and_then(|c| c.active_universe_id()) - .ok_or_else(|| anyhow::anyhow!("select a universe before deleting a binding"))?; + BindingCommand::Delete { provider_id, json } => { let response: api::DeploymentProviderBindingDeleteResponse = client .request( api::METHOD_DEPLOYMENT_PROVIDER_BINDINGS_DELETE, api::DeploymentProviderBindingDeleteParams { - universe_id, - binding_id, + universe_id: selected_universe_id("deleting a binding")?, + provider_id, }, ) .await? @@ -705,21 +784,16 @@ async fn binding(args: BindingArgs) -> Result<()> { } else { crate::output::table( &response.bindings, - &[ - ("bindingId", "ID"), - ("providerId", "PROVIDER"), - ("status", "STATUS"), - ("revision", "REVISION"), - ], + &[("providerId", "PROVIDER"), ("enabled", "ENABLED")], "No environment provider bindings available.", ) } } - BindingCommand::Read { binding_id, json } => { + BindingCommand::Read { provider_id, json } => { let response: api::EnvironmentProviderBindingReadResponse = client .request( api::METHOD_ENVIRONMENTS_PROVIDER_BINDINGS_READ, - api::EnvironmentProviderBindingReadParams { binding_id }, + api::EnvironmentProviderBindingReadParams { provider_id }, ) .await? .result; @@ -730,12 +804,12 @@ async fn binding(args: BindingArgs) -> Result<()> { async fn template(args: TemplateArgs) -> Result<()> { let client = HttpAgentApi::new(""); match args.command { - TemplateCommand::List { binding, json } => { + TemplateCommand::List { provider, json } => { let response: api::EnvironmentTemplateListResponse = client .request( api::METHOD_ENVIRONMENTS_TEMPLATES_LIST, api::EnvironmentTemplateListParams { - binding_id: binding, + provider_id: provider, }, ) .await? @@ -748,7 +822,8 @@ async fn template(args: TemplateArgs) -> Result<()> { &[ ("templateId", "ID"), ("displayName", "NAME"), - ("bindingId", "BINDING"), + ("providerId", "PROVIDER"), + ("powerStates", "POWER"), ("deprecated", "DEPRECATED"), ], "No environment templates available.", @@ -757,14 +832,14 @@ async fn template(args: TemplateArgs) -> Result<()> { } TemplateCommand::Read { template_id, - binding, + provider, json, } => { let response: api::EnvironmentTemplateReadResponse = client .request( api::METHOD_ENVIRONMENTS_TEMPLATES_READ, api::EnvironmentTemplateReadParams { - binding_id: binding, + provider_id: provider, template_id, }, ) @@ -789,19 +864,34 @@ async fn ingress(args: IngressArgs) -> Result<()> { } #[derive(Args, Debug, Clone)] -struct DocumentArgs { - /// Typed API request JSON document. See the public API reference for fields. +struct ProviderPutArgs { + provider_id: String, + /// `ws://`, `wss://` or `unix://` URL of the provider controller. + #[arg(long)] + controller_endpoint: String, + #[arg(long)] + display_name: Option, + /// `ws://` or `wss://` base of the provider's data routes; omit to derive + /// it from the controller endpoint. Required for a `unix://` controller. + #[arg(long)] + data_route_base: Option, + /// File holding the bearer credential the provider expects; omit to keep + /// the stored one. A provider needs none. + #[arg(long, conflicts_with = "clear_credential")] + credential_file: Option, + /// Remove the stored credential. #[arg(long)] - file: std::path::PathBuf, + clear_credential: bool, #[arg(long)] json: bool, } -fn read_document(path: &std::path::Path) -> Result { - use anyhow::Context; - serde_json::from_slice( - &std::fs::read(path).with_context(|| format!("read {}", path.display()))?, - ) - .context("invalid API request document") + +/// The universe a deployment-scoped binding command addresses. +fn selected_universe_id(action: &str) -> Result { + crate::connection::ACTIVE + .get() + .and_then(|c| c.active_universe_id()) + .ok_or_else(|| anyhow::anyhow!("select a universe before {action}")) } #[derive(Args, Debug, Clone)] struct ProviderArgs { @@ -819,8 +909,8 @@ enum ProviderCommand { #[arg(long)] json: bool, }, - /// Create or replace a controller using DeploymentEnvironmentProviderPutParams JSON. - Put(DocumentArgs), + /// Register or replace a provider controller. + Put(ProviderPutArgs), Delete { provider_id: String, #[arg(long)] @@ -846,7 +936,8 @@ async fn provider(args: ProviderArgs) -> Result<()> { &[ ("providerId", "ID"), ("displayName", "NAME"), - ("controllerConnection", "CONTROLLER"), + ("controllerEndpoint", "CONTROLLER"), + ("hasCredential", "CREDENTIAL"), ], "No environment provider controllers configured.", ) @@ -863,9 +954,23 @@ async fn provider(args: ProviderArgs) -> Result<()> { crate::output::show(json, &response.provider) } ProviderCommand::Put(args) => { - let params: api::DeploymentEnvironmentProviderPutParams = read_document(&args.file)?; + let credential = args + .credential_file + .as_deref() + .map(crate::connection::read_secret) + .transpose()?; let response: api::DeploymentEnvironmentProviderPutResponse = client - .request(api::METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_PUT, params) + .request( + api::METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_PUT, + api::DeploymentEnvironmentProviderPutParams { + provider_id: args.provider_id, + display_name: args.display_name, + controller_endpoint: args.controller_endpoint, + data_route_base: args.data_route_base, + credential, + clear_credential: args.clear_credential, + }, + ) .await? .result; crate::output::show(args.json, &response.provider) diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 6bc8870f5..5ed093a80 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -221,7 +221,7 @@ mod tests { vec![ "environment", "create", - "--binding", + "--provider", "local", "--template", "linux-v1", @@ -235,9 +235,16 @@ mod tests { "--request-id", "retry-2", ], - vec!["environment", "provider", "put", "--file", "provider.json"], - vec!["environment", "binding", "put", "--file", "binding.json"], - vec!["environment", "template", "list", "--binding", "local"], + vec![ + "environment", + "provider", + "put", + "local", + "--controller-endpoint", + "wss://provider.example/control", + ], + vec!["environment", "binding", "put", "local", "--enabled"], + vec!["environment", "template", "list", "--provider", "local"], vec!["profile", "--json", "read", "reviewer"], ] { let mut command = vec![ @@ -646,7 +653,7 @@ mod tests { } #[test] - fn env_power_and_idle_policy_parse() { + fn env_power_update_and_paging_parse() { let cli = Cli::try_parse_from([ "lightspeed", "env", @@ -661,16 +668,30 @@ mod tests { let cli = Cli::try_parse_from([ "lightspeed", "env", - "idle-policy", + "update", "--api-url", "http://127.0.0.1:18080/rpc", "local-host", - "--pause-after-min", - "15", - "--close-after-min", - "240", + "--name", + "Build box", + "--metadata", + "team=platform", + ]) + .expect("parse env update"); + assert!(matches!(cli.command, Command::Env(_))); + let cli = Cli::try_parse_from([ + "lightspeed", + "env", + "list", + "--api-url", + "http://127.0.0.1:18080/rpc", + "--include-closed", + "--limit", + "50", + "--after", + "env-1", ]) - .expect("parse env idle-policy"); + .expect("parse env list paging"); assert!(matches!(cli.command, Command::Env(_))); assert!( Cli::try_parse_from([ @@ -680,9 +701,67 @@ mod tests { "--api-url", "http://127.0.0.1:18080/rpc", "local-host", - "--clear", - "--pause-after-min", - "15", + ]) + .is_err() + ); + } + + #[test] + fn env_registration_key_and_binding_parse() { + for command in [ + vec![ + "registration-key", + "create", + "--name", + "Pods", + "--offline-close-after-min", + "60", + ], + vec![ + "registration-key", + "update", + "registration_key_1", + "--name", + "Pods", + "--max-active", + "3", + ], + vec!["registration-key", "rotate", "registration_key_1"], + vec!["binding", "put", "incus", "--disabled"], + vec!["binding", "delete", "incus"], + vec![ + "provider", + "put", + "incus", + "--controller-endpoint", + "unix:///run/provider.sock", + "--data-route-base", + "ws://127.0.0.1:19090/routes", + "--clear-credential", + ], + ] { + let mut args = vec!["lightspeed", "env"]; + args.extend(command); + let cli = + Cli::try_parse_from(&args).unwrap_or_else(|error| panic!("{args:?}: {error}")); + assert!(matches!(cli.command, Command::Env(_))); + } + assert!( + Cli::try_parse_from(["lightspeed", "env", "binding", "put", "incus"]).is_err(), + "a binding put states enabled or disabled" + ); + assert!( + Cli::try_parse_from([ + "lightspeed", + "env", + "provider", + "put", + "incus", + "--controller-endpoint", + "wss://provider.example/control", + "--credential-file", + "token", + "--clear-credential", ]) .is_err() ); diff --git a/crates/cli/src/profile_cli.rs b/crates/cli/src/profile_cli.rs index 306c2b17d..05e92ed05 100644 --- a/crates/cli/src/profile_cli.rs +++ b/crates/cli/src/profile_cli.rs @@ -5,9 +5,9 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, anyhow, bail}; use api::{ - AgentApiErrorKind, AgentProfile, AgentProfileInput, EnvironmentLifecycleStatusView, - EnvironmentProviderBindingListParams, EnvironmentProviderBindingStatusView, - EnvironmentSourceView, InlineAgentProfile, ProfileApplyParams, ProfileDeleteParams, ProfileId, + AgentApiErrorKind, AgentProfile, AgentProfileInput, EnvironmentIdentityModeView, + EnvironmentProviderBindingListParams, EnvironmentSourceView, EnvironmentStateView, + EnvironmentStatusView, InlineAgentProfile, ProfileApplyParams, ProfileDeleteParams, ProfileId, ProfileListParams, ProfilePutParams, ProfileReadParams, ProfileSource, WorkspaceAccess, WorkspaceAttachment, }; @@ -126,6 +126,8 @@ enum ProvisionVfsMode { struct ValidationReport { errors: Vec, warnings: Vec, + /// Facts worth knowing that need no action. + notes: Vec, } impl ValidationReport { @@ -137,25 +139,37 @@ impl ValidationReport { self.warnings.push(message.into()); } + fn note(&mut self, message: impl Into) { + self.notes.push(message.into()); + } + fn extend(&mut self, other: ValidationReport) { self.errors.extend(other.errors); self.warnings.extend(other.warnings); + self.notes.extend(other.notes); + } + + fn print_notices(&self) { + for note in &self.notes { + eprintln!("note: {note}"); + } + for warning in &self.warnings { + eprintln!("warning: {warning}"); + } } fn finish(self, json: bool) -> Result<()> { if json { crate::output::show( true, - &serde_json::json!({"valid":self.errors.is_empty(),"warnings":self.warnings,"errors":self.errors}), + &serde_json::json!({"valid":self.errors.is_empty(),"notes":self.notes,"warnings":self.warnings,"errors":self.errors}), )?; if !self.errors.is_empty() { bail!("profile validation failed"); } return Ok(()); } - for warning in &self.warnings { - eprintln!("warning: {warning}"); - } + self.print_notices(); if self.errors.is_empty() { println!("Profile validation passed."); return Ok(()); @@ -167,9 +181,7 @@ impl ValidationReport { } fn ensure_success(self) -> Result<()> { - for warning in &self.warnings { - eprintln!("warning: {warning}"); - } + self.print_notices(); if self.errors.is_empty() { return Ok(()); } @@ -443,16 +455,11 @@ async fn validate_import_documents( } fn prefix_validation_report(profile_id: &str, mut report: ValidationReport) -> ValidationReport { - report.warnings = report - .warnings - .into_iter() - .map(|message| format!("profile {profile_id}: {message}")) - .collect(); - report.errors = report - .errors - .into_iter() - .map(|message| format!("profile {profile_id}: {message}")) - .collect(); + for messages in [&mut report.notes, &mut report.warnings, &mut report.errors] { + for message in messages.iter_mut() { + *message = format!("profile {profile_id}: {message}"); + } + } report } @@ -624,7 +631,7 @@ async fn validate_environments( }; let bindings = bindings .into_iter() - .map(|binding| (binding.binding_id.clone(), binding)) + .map(|binding| (binding.provider_id.clone(), binding)) .collect::>(); for environment_id in environment_ids { validate_environment_attachment(api, &environment_id, &bindings, report).await; @@ -653,47 +660,80 @@ async fn validate_environment_attachment( return; } }; - match environment.status { - EnvironmentLifecycleStatusView::Ready => {} - EnvironmentLifecycleStatusView::Closing - | EnvironmentLifecycleStatusView::Closed - | EnvironmentLifecycleStatusView::Failed => report.error(format!( + match environment.state { + EnvironmentStateView::Closing + | EnvironmentStateView::Closed + | EnvironmentStateView::Failed => report.error(format!( "profile environment {environment_id} is {:?}; applying the profile will be rejected until it points at an open environment", - environment.status + environment.state )), - status => report.warning(format!( - "profile environment {environment_id} is {status:?}, not ready" + EnvironmentStateView::Provisioning => report.warning(format!( + "profile environment {environment_id} is still provisioning" )), + EnvironmentStateView::Open => match environment.status { + EnvironmentStatusView::Paused + | EnvironmentStatusView::Suspended + | EnvironmentStatusView::Stopped => report.note(format!( + "profile environment {environment_id} is {:?}; it wakes on use", + environment.status + )), + EnvironmentStatusView::Away => report.warning(match &environment.source { + EnvironmentSourceView::Registered { + identity_mode: EnvironmentIdentityModeView::Persistent, + .. + } => format!( + "profile environment {environment_id}: daemon disconnected (persistent key); uses wait up to two minutes for it to reconnect" + ), + EnvironmentSourceView::Registered { + identity_mode: EnvironmentIdentityModeView::Ephemeral, + .. + } => format!( + "profile environment {environment_id}: daemon disconnected (ephemeral key); uses are refused, and it closes unless the daemon reconnects within the key's disconnect grace" + ), + _ => format!("profile environment {environment_id}: daemon disconnected"), + }), + EnvironmentStatusView::Unknown => report.warning(format!( + "profile environment {environment_id} is open, but its provider's latest observation is inconclusive" + )), + EnvironmentStatusView::Ready + | EnvironmentStatusView::Provisioning + | EnvironmentStatusView::Closing + | EnvironmentStatusView::Closed + | EnvironmentStatusView::Failed => {} + }, } - // A long-lived box (a bot's, or one shared by several sessions) sleeps - // only through its own idle policy; nothing on the profile can add one. - match &environment.source { - EnvironmentSourceView::Provisioned { .. } if environment.idle_policy.is_none() => { - report.warning(format!( - "profile environment {environment_id} has no idle policy: it never pauses, suspends, or stops while idle (set one with `env idle-policy` or on the Environments page)" - )); - } - EnvironmentSourceView::External { .. } => { + if let EnvironmentSourceView::External { .. } = &environment.source { + report.warning(format!( + "profile environment {environment_id} is external: no power control, so it can never be paused or stopped while idle" + )); + } + if let EnvironmentSourceView::Provisioned { provider_id, .. } = &environment.source { + warn_unusable_binding( + bindings, + provider_id, + &format!("profile environment {environment_id}"), + report, + ); + } +} + +fn warn_unusable_binding( + bindings: &BTreeMap, + provider_id: &str, + subject: &str, + report: &mut ValidationReport, +) { + match bindings.get(provider_id) { + Some(binding) if !binding.enabled => { report.warning(format!( - "profile environment {environment_id} is external: no power control, so it can never be paused or stopped while idle" + "{subject}: provider binding {provider_id} is disabled" )); } + None => report.warning(format!( + "{subject}: provider binding {provider_id} is missing" + )), _ => {} } - if let EnvironmentSourceView::Provisioned { binding_id, .. } = &environment.source { - match bindings.get(binding_id) { - Some(binding) if binding.status != EnvironmentProviderBindingStatusView::Enabled => { - report.warning(format!( - "profile environment binding {binding_id} is {:?}", - binding.status - )); - } - None => report.warning(format!( - "profile environment binding {binding_id} is missing" - )), - _ => {} - } - } } /// Inserts the attachment or repoints the existing one at the same path, diff --git a/crates/cli/tests/resource_commands.rs b/crates/cli/tests/resource_commands.rs index 182c2f60c..7fdbf85b9 100644 --- a/crates/cli/tests/resource_commands.rs +++ b/crates/cli/tests/resource_commands.rs @@ -546,24 +546,41 @@ fn oauth_no_wait_and_empty_lists_have_useful_output_and_clean_json() { fn provisioning_and_template_commands_send_current_typed_requests() { let runtime = Runtime::start(|method, params| match method { "environments/templates/list" => { - assert_eq!(params["bindingId"], "local"); - json!({"templates":[{"templateId":"linux-v1","providerId":"incus","bindingId":"local","displayName":"Linux","publicIngress":false,"deprecated":false}]}) + assert_eq!(params["providerId"], "incus"); + json!({"templates":[{"templateId":"linux-v1","providerId":"incus","displayName":"Linux","powerStates":["running","paused"],"publicIngress":false,"deprecated":false}]}) } "environments/create" | "environments/external/create" => { - let source = if method == "environments/create" { + let (source, state, status) = if method == "environments/create" { assert_eq!(params["templateId"], "linux-v1"); - assert_eq!(params["bindingId"], "local"); + assert_eq!(params["providerId"], "incus"); assert_eq!(params["metadata"], json!({"project":"audit"})); - json!({"type":"provisioned","providerId":"incus","bindingId":"local"}) + ( + json!({"type":"provisioned","providerId":"incus","templateId":"linux-v1"}), + "provisioning", + "provisioning", + ) } else { - assert_eq!( - params["connection"], - json!({"endpoint":"wss://env.example/ws","transport":"webSocket"}) - ); - json!({"type":"external","connection":params["connection"]}) + assert_eq!(params["endpoint"], "wss://env.example/ws"); + assert!(params.get("connection").is_none(), "{params}"); + ( + json!({"type":"external","endpoint":params["endpoint"]}), + "open", + "ready", + ) }; assert_eq!(params["requestId"], "retry-1"); - json!({"environment":{"environmentId":"env1","requestId":params["requestId"],"source":source,"status":"provisioning","desiredPower":"running","incarnation":{"incarnationId":"i1","createdAtMs":0,"updatedAtMs":0},"publicIngressEnabled":false,"createdAtMs":0,"updatedAtMs":0}}) + json!({"environment":{"environmentId":"env1","requestId":params["requestId"],"source":source,"state":state,"status":status,"stateChangedAtMs":0,"createdAtMs":0,"updatedAtMs":0}}) + } + "environments/list" => { + assert_eq!(params["includeClosed"], true); + assert_eq!(params["after"], "env0"); + json!({"environments":[{"environmentId":"env1","source":{"type":"external","endpoint":"wss://env.example/ws"},"state":"closed","status":"closed","closeReason":"requested","stateChangedAtMs":0,"closedAtMs":0,"createdAtMs":0,"updatedAtMs":0}],"nextAfter":"env1"}) + } + "environments/update" => { + assert_eq!(params["environmentId"], "env1"); + assert_eq!(params["displayName"], "Build box"); + assert_eq!(params["metadata"], json!({"team":"platform"})); + json!({"environment":{"environmentId":"env1","displayName":"Build box","metadata":{"team":"platform"},"source":{"type":"external","endpoint":"wss://env.example/ws"},"state":"open","status":"ready","stateChangedAtMs":0,"createdAtMs":0,"updatedAtMs":1}}) } "deployment/environment-providers/list" => json!({"providers":[]}), "environments/provider-bindings/list" => json!({"bindings":[]}), @@ -573,16 +590,16 @@ fn provisioning_and_template_commands_send_current_typed_requests() { "environments", "templates", "list", - "--binding", - "local", + "--provider", + "incus", "--json", ]); assert_eq!(templates["templates"][0]["templateId"], "linux-v1"); let created = runtime.json(&[ "environment", "create", - "--binding", - "local", + "--provider", + "incus", "--template", "linux-v1", "--request-id", @@ -613,6 +630,20 @@ fn provisioning_and_template_commands_send_current_typed_requests() { .success(&["environment", "binding", "list"]) .contains("No environment provider bindings") ); + let page = runtime.success(&["environment", "list", "--include-closed", "--after", "env0"]); + assert!(page.contains("STATE"), "{page}"); + assert!(page.contains("--after env1"), "{page}"); + let updated = runtime.json(&[ + "environment", + "update", + "env1", + "--name", + "Build box", + "--metadata", + "team=platform", + "--json", + ]); + assert_eq!(updated["displayName"], "Build box"); } #[test] diff --git a/crates/environment-client/Cargo.toml b/crates/environment-client/Cargo.toml index b868be79b..f408d7b22 100644 --- a/crates/environment-client/Cargo.toml +++ b/crates/environment-client/Cargo.toml @@ -3,15 +3,21 @@ name = "environment-client" version = "0.1.0" edition = "2024" +[features] +# `conformance::controller`: a suite that drives any controller endpoint +# through `ControllerClient` and checks the controller-plane contract; +# `conformance::data_route`: checks of a data route's typed refusals. +conformance = [] + [dependencies] -async-trait = "0.1" -futures-util = "0.3" +futures-util = { version = "0.3", features = ["sink"] } environment-protocol = { path = "../environment-protocol" } serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" -tokio = { version = "1", features = ["net"] } -tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] } +tokio = { version = "1", features = ["macros", "net", "rt", "sync", "time"] } +tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] } [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt", "time"] } +environment-client = { path = ".", features = ["conformance"] } +tokio = { version = "1", features = ["io-util", "macros", "rt", "time"] } diff --git a/crates/environment-client/src/conformance/controller.rs b/crates/environment-client/src/conformance/controller.rs new file mode 100644 index 000000000..f550dab2c --- /dev/null +++ b/crates/environment-client/src/conformance/controller.rs @@ -0,0 +1,790 @@ +//! Controller-plane conformance: drives any controller endpoint through +//! [`ControllerClient`] and checks the contract every provider shares. +//! +//! The suite checks that: +//! +//! - `initialize` names the `controller` plane, refuses a wrong +//! `protocolVersion` with `versionMismatch` (leaving the connection open +//! and uninitialized), refuses requests before it with `notInitialized`, +//! and refuses a repeat with `invalidRequest`; +//! - when the suite is given a bearer credential, a missing or wrong one is +//! refused with HTTP 401 before the upgrade (a provider registered +//! without a credential is not checked for one); +//! - `target/get` never creates and answers `notFound` for an unknown +//! owner or id; a foreign owner on a known `targetId` is +//! `failedPrecondition` (`ownerMismatch`); +//! - `target/create` answers promptly and is idempotent per owner; +//! - `target/close` by owner without a `targetId` closes, is idempotent +//! (a repeat answers `closed`), and answers `closed` without `targetId` +//! for an owner with nothing; +//! - optional methods (`target/setPower` without `capabilities.power`, +//! `ingress/*` without `capabilities.ingress`) answer +//! `capabilityUnavailable`; +//! - when the suite is given the provider's data route base, the created +//! target's data route ([`provider_data_route`]) upgrades, refusing at +//! most `waking` or `booting` with `Retry-After` on the way; with +//! `capabilities.power` and a template that lists a powered-down state, +//! a target moved there by `target/setPower` wakes on the next dial +//! (obligation 2); and once the target is closed the route refuses +//! 410 `gone`. +//! +//! Every answer is decoded strictly, so an observation that carries a +//! member this protocol version does not define fails the check that read +//! it. Owners are `{universeId, environmentId}`. +//! +//! The suite creates one target from the configured template and closes it +//! again; every other check uses fresh owners that hold nothing. The data +//! route checks are built on [`data_route`](super::data_route). +//! +//! ```no_run +//! # async fn example(options: environment_client::ConnectOptions, +//! # template_id: environment_protocol::ids::TemplateId, +//! # owner: std::sync::Arc environment_protocol::control::target::TargetOwner + Send + Sync>) { +//! use environment_client::conformance::controller::{ControllerConformance, run}; +//! +//! let suite = ControllerConformance::new("wss://provider.example/control", options, template_id, owner); +//! run(&suite).await.into_result().expect("controller conforms"); +//! # } +//! ``` + +use std::{ + fmt, + sync::Arc, + time::{Duration, Instant}, +}; + +use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, + bytes::SecretString, + control::{ + handshake::{ControllerInitializeParams, ControllerInitializeResponse}, + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + CloseTargetParams, CreateTargetParams, GetTargetParams, PowerState, + SetTargetPowerParams, TargetObservation, TargetOwner, TargetPhase, TargetRef, + }, + template::ListTemplatesParams, + }, + error::{Capability, ErrorData, PreconditionReason, ProtocolErrorKind}, + handshake::Plane, + ids::{ProviderTargetId, TemplateId}, + manifest::{ControllerInitialize, TargetGet}, + routes::{DataRouteBase, DataRouteRefusal, provider_data_route}, +}; +use thiserror::Error; + +use crate::{ + ClientError, ConnectOptions, ControllerClient, + conformance::data_route::{await_upgrade, expect_refusal}, + connection::Connection, + controller::initialize, +}; + +/// What to run the suite against. +#[derive(Clone)] +pub struct ControllerConformance { + /// The controller URL, as registered for the provider. + pub endpoint: String, + /// Connect options carrying the provider credential, when the provider + /// is registered with one, as `bearer_token`. + pub options: ConnectOptions, + /// A template the suite may create one target from. + pub template_id: TemplateId, + /// Returns a fresh owner on every call, in a universe the provider + /// serves and never used before. + pub owner: Arc TargetOwner + Send + Sync>, + /// The provider's data route base, as Lightspeed dials it: configured, + /// or [`DataRouteBase::derived_from_controller`]. When set, the suite + /// also dials the created target's data route with `options`. Default + /// none. + pub data_route_base: Option, + /// Bound on every answer: target methods answer promptly and continue + /// slow work in the background. Default 15 s. + pub prompt_bound: Duration, + /// How long the created target may take to carry a `targetId`, and its + /// close to reach `closed`. Default 10 min. + pub settle_timeout: Duration, + /// Pause between observations while waiting. Default 2 s. + pub poll_interval: Duration, +} + +impl ControllerConformance { + pub fn new( + endpoint: impl Into, + options: ConnectOptions, + template_id: TemplateId, + owner: Arc TargetOwner + Send + Sync>, + ) -> Self { + Self { + endpoint: endpoint.into(), + options, + template_id, + owner, + data_route_base: None, + prompt_bound: Duration::from_secs(15), + settle_timeout: Duration::from_secs(600), + poll_interval: Duration::from_secs(2), + } + } +} + +impl fmt::Debug for ControllerConformance { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ControllerConformance") + .field("endpoint", &self.endpoint) + .field("template_id", &self.template_id) + .field("data_route_base", &self.data_route_base) + .field("prompt_bound", &self.prompt_bound) + .field("settle_timeout", &self.settle_timeout) + .finish_non_exhaustive() + } +} + +/// The outcome of one check. +#[derive(Clone, Debug)] +pub struct CheckResult { + /// Stable check name, such as `target.get.foreignOwner`. + pub name: &'static str, + /// Why the check failed. + pub failure: Option, +} + +/// Every check that ran, in order. +#[derive(Clone, Debug, Default)] +pub struct ConformanceReport { + pub checks: Vec, +} + +impl ConformanceReport { + pub fn passed(&self) -> bool { + self.checks.iter().all(|check| check.failure.is_none()) + } + + pub fn failures(&self) -> impl Iterator { + self.checks.iter().filter(|check| check.failure.is_some()) + } + + /// `Ok` when every check passed. + pub fn into_result(self) -> Result { + if self.passed() { + return Ok(self); + } + Err(ConformanceFailure { + failures: self + .failures() + .map(|check| (check.name, check.failure.clone().unwrap_or_default())) + .collect(), + }) + } + + fn record(&mut self, name: &'static str, outcome: Result<(), String>) -> bool { + let passed = outcome.is_ok(); + self.checks.push(CheckResult { + name, + failure: outcome.err(), + }); + passed + } +} + +/// The failed checks of a run. +#[derive(Clone, Debug, Error)] +#[error("controller conformance failed: {}", describe(failures))] +pub struct ConformanceFailure { + /// Check name and why it failed. + pub failures: Vec<(&'static str, String)>, +} + +fn describe(failures: &[(&'static str, String)]) -> String { + failures + .iter() + .map(|(name, failure)| format!("{name}: {failure}")) + .collect::>() + .join("; ") +} + +type Check = Result<(), String>; + +/// Run every check against `suite.endpoint`. Checks that need the +/// handshake are skipped (and reported) when it fails. +pub async fn run(suite: &ControllerConformance) -> ConformanceReport { + let mut report = ConformanceReport::default(); + let connected = ControllerClient::connect_initialized( + &suite.endpoint, + with_timeout(suite.options.clone(), suite.prompt_bound), + ) + .await; + let (client, initialize) = match connected { + Ok(connected) => connected, + Err(error) => { + report.record( + "initialize", + Err(format!("connect_initialized failed: {error}")), + ); + return report; + } + }; + report.record("initialize", check_initialize(&initialize)); + report.record( + "initialize.repeat", + check_repeat_initialize(suite, &client).await, + ); + report.record("initialize.order", check_initialize_order(suite).await); + if suite.options.bearer_token.is_some() { + report.record("credential", check_credential(suite).await); + } + report.record( + "target.get.unknown", + check_get_unknown(suite, &client).await, + ); + report.record( + "target.close.nothing", + check_close_nothing(suite, &client).await, + ); + report.record( + "capabilities.optional", + check_optional_methods(suite, &client, &initialize).await, + ); + + let owner = (suite.owner)(); + let created = check_create(suite, &client, &owner).await; + let created_target = created.as_ref().ok().cloned().flatten(); + report.record("target.create", created.map(|_| ())); + let route = created_target.as_ref().and_then(|target_id| { + suite + .data_route_base + .as_ref() + .map(|base| provider_data_route(base, &owner, target_id)) + }); + if let Some(target_id) = &created_target { + report.record( + "target.get.foreignOwner", + check_foreign_owner(suite, &client, target_id).await, + ); + } + if let (Some(target_id), Some(route)) = (&created_target, &route) { + let upgraded = report.record("dataRoute.upgrade", check_upgrade(suite, route).await); + if upgraded && initialize.capabilities.power { + match power_down_state(suite, &client, &owner).await { + Ok(Some(power)) => { + report.record( + "dataRoute.wake", + check_wake(suite, &client, &owner, target_id, route, power).await, + ); + } + Ok(None) => {} + Err(error) => { + report.record("dataRoute.wake", Err(error)); + } + } + } + } + let closed = report.record( + "target.close.byOwner", + check_close_by_owner(suite, &client, &owner).await, + ); + if closed && let Some(route) = &route { + report.record("dataRoute.gone", check_gone(suite, route).await); + } + let _ = client.close().await; + report +} + +fn with_timeout(mut options: ConnectOptions, bound: Duration) -> ConnectOptions { + options.request_timeout = Some(bound); + options +} + +fn check_initialize(initialize: &ControllerInitializeResponse) -> Check { + if initialize.plane != Plane::Controller { + return Err(format!("plane is {}", initialize.plane)); + } + if initialize.protocol_version != CONTROLLER_PROTOCOL_VERSION { + return Err(format!( + "protocolVersion is {}", + initialize.protocol_version + )); + } + Ok(()) +} + +async fn check_repeat_initialize( + suite: &ControllerConformance, + client: &ControllerClient, +) -> Check { + let repeated = client + .call::(&ControllerInitializeParams { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + client_info: suite.options.client_info.clone(), + }) + .await; + expect_kind(repeated.map(|_| ()), ProtocolErrorKind::InvalidRequest) +} + +/// `notInitialized` before `initialize`; `versionMismatch` for another +/// version, after which the same connection still initializes. +async fn check_initialize_order(suite: &ControllerConformance) -> Check { + let options = with_timeout(suite.options.clone(), suite.prompt_bound); + let connection = Connection::connect(&suite.endpoint, &options, Plane::Controller) + .await + .map_err(|error| format!("connect: {error}"))?; + let outcome = async { + let early = connection + .call::( + &GetTargetParams { + target: TargetRef { + owner: (suite.owner)(), + target_id: None, + }, + }, + None, + ) + .await; + expect_kind(early.map(|_| ()), ProtocolErrorKind::NotInitialized) + .map_err(|error| format!("target/get before initialize: {error}"))?; + + let wrong = CONTROLLER_PROTOCOL_VERSION + 1; + let params = serde_json::to_value(ControllerInitializeParams { + protocol_version: wrong, + client_info: suite.options.client_info.clone(), + }) + .map_err(|error| error.to_string())?; + let response = connection + .request(ControllerInitialize::NAME, params, Some(suite.prompt_bound)) + .await + .map_err(|error| format!("initialize with version {wrong}: {error}"))?; + match response.result { + Err(error) => match error.data { + ErrorData::VersionMismatch { expected, received } + if expected == CONTROLLER_PROTOCOL_VERSION && received == wrong => {} + data => { + return Err(format!( + "initialize with version {wrong} answered {data:?}, expected \ + versionMismatch {{expected: {CONTROLLER_PROTOCOL_VERSION}, received: {wrong}}}" + )); + } + }, + Ok(_) => return Err(format!("initialize with version {wrong} succeeded")), + } + initialize(&connection, &options) + .await + .map(|_| ()) + .map_err(|error| format!("initialize after a mismatch: {error}")) + } + .await; + let _ = connection.close().await; + outcome +} + +async fn check_credential(suite: &ControllerConformance) -> Check { + let mut missing = suite.options.clone(); + missing.bearer_token = None; + let mut wrong = suite.options.clone(); + wrong.bearer_token = Some(SecretString::new("lightspeed-conformance-wrong-credential")); + for (label, options) in [("missing", missing), ("wrong", wrong)] { + match ControllerClient::connect_initialized(&suite.endpoint, options).await { + Err(ClientError::UpgradeRejected { status: 401, .. }) => {} + Err(error) => { + return Err(format!( + "{label} credential: expected HTTP 401, got {error}" + )); + } + Ok((client, _)) => { + let _ = client.close().await; + return Err(format!("{label} credential was accepted")); + } + } + } + Ok(()) +} + +async fn check_get_unknown(suite: &ControllerConformance, client: &ControllerClient) -> Check { + let owner = (suite.owner)(); + let by_owner = get(client, &owner, None).await; + expect_kind(by_owner.map(|_| ()), ProtocolErrorKind::NotFound) + .map_err(|error| format!("target/get of an owner with nothing: {error}"))?; + let unknown = ProviderTargetId::new("lightspeed-conformance-unknown-target") + .map_err(|error| error.to_string())?; + let by_id = get(client, &owner, Some(unknown)).await; + expect_kind(by_id.map(|_| ()), ProtocolErrorKind::NotFound) + .map_err(|error| format!("target/get of an unknown targetId: {error}"))?; + let again = get(client, &owner, None).await; + expect_kind(again.map(|_| ()), ProtocolErrorKind::NotFound) + .map_err(|error| format!("target/get created a target: {error}")) +} + +async fn check_close_nothing(suite: &ControllerConformance, client: &ControllerClient) -> Check { + let owner = (suite.owner)(); + let observation = close(client, &owner) + .await + .map_err(|error| format!("target/close of an owner with nothing: {error}"))?; + if observation.phase != TargetPhase::Closed || observation.target_id.is_some() { + return Err(format!( + "target/close of an owner with nothing answered phase {:?} with targetId {:?}; \ + expected closed without targetId", + observation.phase, observation.target_id + )); + } + check_owner_echo(&observation, &owner) +} + +async fn check_optional_methods( + suite: &ControllerConformance, + client: &ControllerClient, + initialize: &ControllerInitializeResponse, +) -> Check { + if !initialize.capabilities.power { + let powered = client + .set_target_power(&SetTargetPowerParams { + target: TargetRef { + owner: (suite.owner)(), + target_id: None, + }, + power: PowerState::Running, + }) + .await; + expect_capability(powered.map(|_| ()), Capability::Power) + .map_err(|error| format!("target/setPower without power: {error}"))?; + } + if !initialize.capabilities.ingress { + let target = TargetRef { + owner: (suite.owner)(), + target_id: None, + }; + let ensured = client + .ensure_ingress(&EnsureIngressParams { + target: target.clone(), + }) + .await; + expect_capability(ensured.map(|_| ()), Capability::Ingress) + .map_err(|error| format!("ingress/ensure without ingress: {error}"))?; + let removed = client.remove_ingress(&RemoveIngressParams { target }).await; + expect_capability(removed.map(|_| ()), Capability::Ingress) + .map_err(|error| format!("ingress/remove without ingress: {error}"))?; + } + Ok(()) +} + +/// Create twice for one owner, each answered within the prompt bound, then +/// wait for the target's id. Returns that id. +async fn check_create( + suite: &ControllerConformance, + client: &ControllerClient, + owner: &TargetOwner, +) -> Result, String> { + let params = CreateTargetParams { + owner: owner.clone(), + template_id: suite.template_id.clone(), + display_name: None, + idle: None, + }; + let first = timed(suite, "target/create", client.create_target(¶ms)).await?; + let second = timed( + suite, + "repeated target/create", + client.create_target(¶ms), + ) + .await?; + check_owner_echo(&first.target, owner)?; + check_owner_echo(&second.target, owner)?; + if let (Some(first_id), Some(second_id)) = (&first.target.target_id, &second.target.target_id) + && first_id != second_id + { + return Err(format!( + "target/create is not idempotent per owner: {first_id} then {second_id}" + )); + } + let mut target_id = second.target.target_id.or(first.target.target_id); + let deadline = Instant::now() + suite.settle_timeout; + while target_id.is_none() { + if Instant::now() >= deadline { + return Err(format!( + "the target carried no targetId within {:?}", + suite.settle_timeout + )); + } + tokio::time::sleep(suite.poll_interval).await; + let observation = get(client, owner, None) + .await + .map_err(|error| format!("target/get of the created target: {error}"))?; + target_id = observation.target_id; + } + let repeated = timed( + suite, + "target/create after the id", + client.create_target(¶ms), + ) + .await?; + if repeated.target.target_id != target_id { + return Err(format!( + "target/create after the id answered targetId {:?}, expected {:?}", + repeated.target.target_id, target_id + )); + } + Ok(target_id) +} + +async fn check_foreign_owner( + suite: &ControllerConformance, + client: &ControllerClient, + target_id: &ProviderTargetId, +) -> Check { + let foreign = (suite.owner)(); + match get(client, &foreign, Some(target_id.clone())).await { + Err(ClientError::Protocol(error)) + if matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::OwnerMismatch + } + ) => + { + Ok(()) + } + Err(error) => Err(format!( + "expected failedPrecondition (ownerMismatch), got {error}" + )), + Ok(observation) => Err(format!( + "a foreign owner observed the target (phase {:?})", + observation.phase + )), + } +} + +/// Close by owner without a `targetId` until `closed`, then once more. +async fn check_close_by_owner( + suite: &ControllerConformance, + client: &ControllerClient, + owner: &TargetOwner, +) -> Check { + let deadline = Instant::now() + suite.settle_timeout; + loop { + let observation = timed(suite, "target/close", async { + client + .close_target(&CloseTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: None, + }, + }) + .await + }) + .await? + .target; + check_owner_echo(&observation, owner)?; + match observation.phase { + TargetPhase::Closed => break, + TargetPhase::Closing => {} + phase => return Err(format!("target/close answered phase {phase:?}")), + } + if Instant::now() >= deadline { + return Err(format!( + "the target did not reach closed within {:?}", + suite.settle_timeout + )); + } + tokio::time::sleep(suite.poll_interval).await; + } + let again = close(client, owner) + .await + .map_err(|error| format!("repeated target/close: {error}"))?; + if again.phase != TargetPhase::Closed { + return Err(format!( + "repeated target/close answered phase {:?}", + again.phase + )); + } + Ok(()) +} + +/// The created target's data route upgrades within the settle timeout, +/// refusing at most `waking` or `booting` on the way. +async fn check_upgrade(suite: &ControllerConformance, route: &str) -> Check { + let refusals = await_upgrade(route, suite.options.clone(), suite.settle_timeout).await?; + check_wait_refusals(&refusals) +} + +/// The first powered-down state the configured template lists for +/// `target/setPower`, if any. +async fn power_down_state( + suite: &ControllerConformance, + client: &ControllerClient, + owner: &TargetOwner, +) -> Result, String> { + let templates = timed( + suite, + "template/list", + client.list_templates(&ListTemplatesParams { + universe_id: owner.universe_id.clone(), + }), + ) + .await?; + let template = templates + .templates + .iter() + .find(|template| template.template_id == suite.template_id) + .ok_or_else(|| format!("template/list does not list {}", suite.template_id))?; + Ok([ + PowerState::Paused, + PowerState::Suspended, + PowerState::Stopped, + ] + .into_iter() + .find(|power| template.power_states.contains(power))) +} + +/// Move the target to `power`, wait until it is observed there, then dial +/// its data route: the dial wakes it and upgrades within the settle +/// timeout, refusing at most `waking` or `booting` on the way. +async fn check_wake( + suite: &ControllerConformance, + client: &ControllerClient, + owner: &TargetOwner, + target_id: &ProviderTargetId, + route: &str, + power: PowerState, +) -> Check { + let target = TargetRef { + owner: owner.clone(), + target_id: Some(target_id.clone()), + }; + timed( + suite, + "target/setPower", + client.set_target_power(&SetTargetPowerParams { + target: target.clone(), + power, + }), + ) + .await?; + let deadline = Instant::now() + suite.settle_timeout; + loop { + let observation = get(client, owner, Some(target_id.clone())) + .await + .map_err(|error| format!("target/get after target/setPower {power}: {error}"))?; + if observation.power == Some(power) { + break; + } + if Instant::now() >= deadline { + return Err(format!( + "the target was not observed {power} within {:?} (last {:?})", + suite.settle_timeout, observation.power + )); + } + tokio::time::sleep(suite.poll_interval).await; + } + let refusals = await_upgrade(route, suite.options.clone(), suite.settle_timeout) + .await + .map_err(|error| format!("dialing the {power} target: {error}"))?; + check_wait_refusals(&refusals) +} + +/// Once the target is closed its data route refuses 410 `gone`. +async fn check_gone(suite: &ControllerConformance, route: &str) -> Check { + match expect_refusal(route, suite.options.clone()).await? { + DataRouteRefusal::Gone {} => Ok(()), + refusal => Err(format!( + "the closed target's data route refused {refusal}, expected gone" + )), + } +} + +/// A provider data route waits only with `waking` or `booting`; the other +/// 503 refusals are the gateway's. +fn check_wait_refusals(refusals: &[DataRouteRefusal]) -> Check { + match refusals.iter().find(|refusal| { + !matches!( + refusal, + DataRouteRefusal::Waking {} | DataRouteRefusal::Booting {} + ) + }) { + Some(refusal) => Err(format!( + "the provider data route refused {refusal}; a provider waits only with waking or booting" + )), + None => Ok(()), + } +} + +async fn get( + client: &ControllerClient, + owner: &TargetOwner, + target_id: Option, +) -> Result { + client + .get_target(&GetTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id, + }, + }) + .await + .map(|response| response.target) +} + +async fn close( + client: &ControllerClient, + owner: &TargetOwner, +) -> Result { + client + .close_target(&CloseTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: None, + }, + }) + .await + .map(|response| response.target) +} + +/// Await `call` and require it to answer within the prompt bound. +async fn timed( + suite: &ControllerConformance, + label: &str, + call: impl Future>, +) -> Result { + let started = Instant::now(); + let outcome = call.await.map_err(|error| format!("{label}: {error}"))?; + let elapsed = started.elapsed(); + if elapsed > suite.prompt_bound { + return Err(format!( + "{label} took {elapsed:?}, over the {:?} prompt bound", + suite.prompt_bound + )); + } + Ok(outcome) +} + +fn check_owner_echo(observation: &TargetObservation, owner: &TargetOwner) -> Check { + if &observation.owner != owner { + return Err(format!( + "the observation echoes owner {:?}, expected {:?}", + observation.owner, owner + )); + } + Ok(()) +} + +fn expect_kind(outcome: Result<(), ClientError>, kind: ProtocolErrorKind) -> Check { + match outcome { + Err(ClientError::Protocol(error)) if error.kind() == kind => Ok(()), + Err(error) => Err(format!("expected {kind}, got {error}")), + Ok(()) => Err(format!("expected {kind}, the call succeeded")), + } +} + +fn expect_capability(outcome: Result<(), ClientError>, capability: Capability) -> Check { + match outcome { + Err(ClientError::Protocol(error)) + if error.data == (ErrorData::CapabilityUnavailable { capability }) => + { + Ok(()) + } + Err(error) => Err(format!( + "expected capabilityUnavailable ({capability:?}), got {error}" + )), + Ok(()) => Err(format!( + "expected capabilityUnavailable ({capability:?}), the call succeeded" + )), + } +} diff --git a/crates/environment-client/src/conformance/data_route.rs b/crates/environment-client/src/conformance/data_route.rs new file mode 100644 index 000000000..0b482092e --- /dev/null +++ b/crates/environment-client/src/conformance/data_route.rs @@ -0,0 +1,178 @@ +//! Data-route refusal conformance: a provider data route that does not +//! upgrade answers a typed [`DataRouteRefusal`] body with the status that +//! refusal names, a 503 carries `Retry-After`, and the route answers within +//! [`MAX_DATA_ROUTE_HOLD_MS`]. [`await_upgrade`] dials until the route +//! upgrades, waiting out each conforming 503 at its `Retry-After`; the +//! [`controller`](super::controller) suite runs both against a created +//! target when it is given the provider's data route base. +//! +//! ```no_run +//! # async fn example(options: environment_client::ConnectOptions) { +//! use environment_client::conformance::data_route::expect_refusal; +//! +//! // A data route of a target the provider was just told to pause. +//! let refusal = expect_refusal("wss://provider.example/routes/u1/env_1/ls-3f2a", options) +//! .await +//! .expect("the route refuses with a typed body"); +//! # let _ = refusal; +//! # } +//! ``` + +use std::time::{Duration, Instant}; + +use environment_protocol::{limits::MAX_DATA_ROUTE_HOLD_MS, routes::DataRouteRefusal}; + +use crate::{ClientError, ConnectOptions, DataClient}; + +/// Slack on top of [`MAX_DATA_ROUTE_HOLD_MS`] for the dial itself. +const HOLD_SLACK: Duration = Duration::from_secs(2); + +/// Check one refused upgrade: it is typed as a [`DataRouteRefusal`], its +/// status is the one the refusal names (503, 410 or 409), and a 503 +/// carries `Retry-After` of at least one second. The client types a +/// refusal from its status when the body was not read with the headers +/// ([`ClientError::UpgradeRejected`]), so this check cannot tell a missing +/// body from one the transport split off. +pub fn check_refusal(error: &ClientError) -> Result<&DataRouteRefusal, String> { + let ClientError::UpgradeRejected { + status, + refusal, + retry_after_ms, + } = error + else { + return Err(format!("expected a refused upgrade, got {error}")); + }; + let refusal = refusal + .as_ref() + .ok_or_else(|| format!("HTTP {status} carries no data route refusal body"))?; + if *status != refusal.status() { + return Err(format!( + "refusal {} answered with HTTP {status}, expected {}", + refusal.state(), + refusal.status() + )); + } + if refusal.is_retryable() && !retry_after_ms.is_some_and(|ms| ms >= 1_000) { + return Err(format!( + "refusal {} carries no Retry-After of at least 1 s", + refusal.state() + )); + } + Ok(refusal) +} + +/// Dial `url` as a data route and require a conforming refusal within +/// [`MAX_DATA_ROUTE_HOLD_MS`] (plus dial slack). Returns the refusal. +pub async fn expect_refusal( + url: &str, + options: ConnectOptions, +) -> Result { + let started = Instant::now(); + let outcome = DataClient::connect_initialized(url, options).await; + let elapsed = started.elapsed(); + let error = match outcome { + Ok((client, _)) => { + let _ = client.close().await; + return Err("the data route upgraded instead of refusing".to_owned()); + } + Err(error) => error, + }; + let refusal = check_refusal(&error)?.clone(); + check_hold(elapsed)?; + Ok(refusal) +} + +/// Dial `url` as a data route until it upgrades, within `timeout`. Every +/// refusal on the way must conform ([`check_refusal`]), be a 503 and answer +/// within [`MAX_DATA_ROUTE_HOLD_MS`]; the next dial follows its +/// `Retry-After`. Returns the 503 refusals seen, in order: `waking` for a +/// powered-down target the dial woke, `booting` while envd starts. +pub async fn await_upgrade( + url: &str, + options: ConnectOptions, + timeout: Duration, +) -> Result, String> { + let deadline = Instant::now() + timeout; + let mut refusals = Vec::new(); + loop { + let started = Instant::now(); + let outcome = DataClient::connect_initialized(url, options.clone()).await; + let elapsed = started.elapsed(); + let error = match outcome { + Ok((client, _)) => { + let _ = client.close().await; + return Ok(refusals); + } + Err(error) => error, + }; + let refusal = check_refusal(&error)?.clone(); + check_hold(elapsed)?; + if !refusal.is_retryable() { + return Err(format!( + "the data route refused {refusal} instead of upgrading" + )); + } + let retry_after = Duration::from_millis(error.retry_after_ms().unwrap_or(1_000)); + if Instant::now() + retry_after > deadline { + return Err(format!( + "the data route did not upgrade within {timeout:?}; last refusal {refusal}" + )); + } + refusals.push(refusal); + tokio::time::sleep(retry_after).await; + } +} + +/// A dial that refused must answer within [`MAX_DATA_ROUTE_HOLD_MS`] plus +/// [`HOLD_SLACK`]. +fn check_hold(elapsed: Duration) -> Result<(), String> { + let bound = Duration::from_millis(MAX_DATA_ROUTE_HOLD_MS) + HOLD_SLACK; + if elapsed > bound { + return Err(format!( + "the data route held the dial for {elapsed:?}, over the {bound:?} bound" + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use environment_protocol::metadata::DescriptiveText; + + use super::*; + + fn rejected( + status: u16, + refusal: Option, + retry_after_ms: Option, + ) -> ClientError { + ClientError::UpgradeRejected { + status, + refusal, + retry_after_ms, + } + } + + #[test] + fn refusals_must_match_their_status_and_carry_retry_after() { + assert_eq!( + check_refusal(&rejected( + 503, + Some(DataRouteRefusal::Waking {}), + Some(2_000) + )), + Ok(&DataRouteRefusal::Waking {}) + ); + assert!(check_refusal(&rejected(503, Some(DataRouteRefusal::Waking {}), None)).is_err()); + assert!(check_refusal(&rejected(503, Some(DataRouteRefusal::Gone {}), None)).is_err()); + assert!(check_refusal(&rejected(410, None, None)).is_err()); + assert!(check_refusal(&ClientError::Transport("x".to_owned())).is_err()); + let failed = DataRouteRefusal::Failed { + message: DescriptiveText::lossy("disk lost").unwrap(), + }; + assert_eq!( + check_refusal(&rejected(409, Some(failed.clone()), None)), + Ok(&failed) + ); + } +} diff --git a/crates/environment-client/src/conformance/mod.rs b/crates/environment-client/src/conformance/mod.rs new file mode 100644 index 000000000..77d2dae53 --- /dev/null +++ b/crates/environment-client/src/conformance/mod.rs @@ -0,0 +1,10 @@ +//! Conformance suites that drive a real endpoint through this crate's +//! clients and check the protocol contract. Enabled by the `conformance` +//! feature. +//! +//! - [`controller`]: any provider controller, through +//! [`ControllerClient`](crate::ControllerClient); +//! - [`data_route`]: a provider data route's refusals before the upgrade. + +pub mod controller; +pub mod data_route; diff --git a/crates/environment-client/src/connection.rs b/crates/environment-client/src/connection.rs new file mode 100644 index 000000000..bfed3ce08 --- /dev/null +++ b/crates/environment-client/src/connection.rs @@ -0,0 +1,771 @@ +//! The multiplexed JSON-RPC connection under both clients. +//! +//! One writer task owns the socket's sink and one reader task its stream. +//! A call registers a oneshot under a fresh integer id, queues its frame +//! for the writer and waits; the reader routes each response to its call +//! by id, in whatever order the server answers. A call that is dropped or +//! runs past its deadline removes its entry, so a late response is +//! discarded, and on the data plane an abortable call also sends +//! `$/cancelRequest`. While any call is pending the writer pings every +//! keepalive interval. +//! +//! The server of a data or controller connection only answers. A request +//! or notification from it, a binary frame, a frame over the message +//! limit or an unparseable frame ends the connection: every pending call +//! fails with [`ClientError::InvalidMessage`] and the client closes the +//! socket with close code 1002 (1009 for an oversized frame). + +use std::{ + collections::HashMap, + sync::{ + Arc, Mutex, + atomic::{AtomicI64, Ordering}, + }, + time::Duration, +}; + +use environment_protocol::{ + error::ErrorData, + handshake::Plane, + jsonrpc::{ + self, CancelRequestParams, Id, Message as RpcMessage, Notification, Request, Response, + }, + limits::MAX_MESSAGE_BYTES, + manifest::{self, CancelRequest, RequestMethod}, +}; +use futures_util::{ + SinkExt, StreamExt, + future::BoxFuture, + stream::{SplitSink, SplitStream}, +}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::{mpsc, oneshot}, + time::{Instant, MissedTickBehavior}, +}; +use tokio_tungstenite::{ + WebSocketStream, + tungstenite::{ + self, Message, + protocol::{CloseFrame, frame::coding::CloseCode}, + }, +}; + +use crate::{ + dial::{self, Dialed}, + error::ClientError, + options::ConnectOptions, +}; + +/// Serves one request of an in-process client: the request as the server +/// would receive it, answered with the response it would send. An `Err` +/// fails that one call with the given error, as a test's stand-in for a +/// transport failure; the client stays usable. +pub type InProcessHandler = + Arc BoxFuture<'static, Result> + Send + Sync>; + +/// How long [`Connection::close`] waits for the close frame to be written. +const CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long the reader keeps reading for the server's close reply once +/// the writer has stopped. +const CLOSE_GRACE: Duration = Duration::from_secs(5); + +type Reply = Result; + +/// Calls waiting for a response, and why the connection ended once it +/// has. +#[derive(Default)] +struct Shared { + state: Mutex, +} + +#[derive(Default)] +struct State { + pending: HashMap>, + terminated: Option, +} + +impl Shared { + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn register(&self, id: i64) -> Result, ClientError> { + let mut state = self.state(); + if let Some(error) = &state.terminated { + return Err(error.clone()); + } + let (sender, receiver) = oneshot::channel(); + state.pending.insert(id, sender); + Ok(receiver) + } + + /// Hand a response to its call; a response nobody waits for any more is + /// discarded. + fn resolve(&self, id: i64, response: Response) { + if let Some(sender) = self.state().pending.remove(&id) { + let _ = sender.send(Ok(response)); + } + } + + /// Stop waiting for `id`. True when the call was still unanswered. + fn forget(&self, id: i64) -> bool { + self.state().pending.remove(&id).is_some() + } + + fn has_pending(&self) -> bool { + !self.state().pending.is_empty() + } + + /// End the connection with `error`: every pending call and every later + /// call fails with it. Only the first cause is kept. + fn terminate(&self, error: ClientError) { + let mut state = self.state(); + if state.terminated.is_none() { + state.terminated = Some(error.clone()); + } + for (_, sender) in state.pending.drain() { + let _ = sender.send(Err(error.clone())); + } + } + + fn terminated(&self) -> Option { + self.state().terminated.clone() + } +} + +enum Outgoing { + Text(String), + /// Send `frame` (if any), then stop writing. `ack` learns whether the + /// frame was written. + Close { + frame: Option, + ack: Option>>, + }, +} + +enum Backend { + Socket { + outgoing: mpsc::UnboundedSender, + }, + InProcess { + handler: InProcessHandler, + }, +} + +/// One connection of either plane. +pub(crate) struct Connection { + plane: Plane, + shared: Arc, + backend: Backend, + next_id: AtomicI64, + request_timeout: Option, +} + +impl Connection { + /// Dial `endpoint` (no `initialize` yet). + pub(crate) async fn connect( + endpoint: &str, + options: &ConnectOptions, + plane: Plane, + ) -> Result { + Ok(match dial::dial(endpoint, options).await? { + Dialed::Tcp(socket) => Self::over_socket(*socket, plane, options), + Dialed::Unix(socket) => Self::over_socket(*socket, plane, options), + }) + } + + /// Upgrade an already connected byte stream (no `initialize` yet). + pub(crate) async fn connect_over( + stream: S, + options: &ConnectOptions, + plane: Plane, + ) -> Result + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let socket = dial::upgrade(stream, options).await?; + Ok(Self::over_socket(socket, plane, options)) + } + + fn over_socket(socket: WebSocketStream, plane: Plane, options: &ConnectOptions) -> Self + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let (sink, stream) = socket.split(); + let shared = Arc::new(Shared::default()); + let (outgoing, outgoing_receiver) = mpsc::unbounded_channel(); + let (writer_done, writer_stopped) = oneshot::channel(); + let keepalive = options.keepalive_interval.max(Duration::from_millis(1)); + tokio::spawn(write_loop( + sink, + outgoing_receiver, + shared.clone(), + keepalive, + writer_done, + )); + tokio::spawn(read_loop( + stream, + shared.clone(), + outgoing.downgrade(), + writer_stopped, + )); + Self { + plane, + shared, + backend: Backend::Socket { outgoing }, + next_id: AtomicI64::new(1), + request_timeout: options.request_timeout, + } + } + + pub(crate) fn in_process(plane: Plane, handler: InProcessHandler) -> Self { + Self { + plane, + shared: Arc::new(Shared::default()), + backend: Backend::InProcess { handler }, + next_id: AtomicI64::new(1), + request_timeout: None, + } + } + + pub(crate) fn is_closed(&self) -> bool { + self.shared.terminated().is_some() + } + + /// Close the connection: pending calls fail with `Closed`, a close + /// frame is written (waiting at most a few seconds), and every later + /// call fails. Closing again does nothing. + pub(crate) async fn close(&self) -> Result<(), ClientError> { + self.shared.terminate(ClientError::Closed { + code: Some(u16::from(CloseCode::Normal)), + reason: None, + }); + let Backend::Socket { outgoing } = &self.backend else { + return Ok(()); + }; + let (ack, written) = oneshot::channel(); + let close = Outgoing::Close { + frame: Some(CloseFrame { + code: CloseCode::Normal, + reason: "".into(), + }), + ack: Some(ack), + }; + if outgoing.send(close).is_err() { + return Ok(()); + } + match tokio::time::timeout(CLOSE_TIMEOUT, written).await { + Ok(Ok(Err(message))) => Err(ClientError::Transport(message)), + _ => Ok(()), + } + } + + /// A typed call of this connection's plane, with the connection's + /// default deadline unless `timeout` overrides it. + pub(crate) async fn call( + &self, + params: &M::Params, + timeout: Option, + ) -> Result { + if M::PLANE != self.plane { + return Err(ClientError::Encode(format!( + "{} is a {} method; this client serves the {} plane", + M::NAME, + M::PLANE, + self.plane + ))); + } + let response = self + .request( + M::NAME, + params_value(params)?, + timeout.or(self.request_timeout), + ) + .await?; + decode(response) + } + + /// Run `initialize`: a `versionMismatch` answer, a response naming + /// another plane or another `protocolVersion` fail with the typed + /// handshake errors before the response is decoded strictly. + pub(crate) async fn initialize( + &self, + method: &str, + params: &P, + local_version: u32, + timeout: Duration, + ) -> Result { + let response = self + .request(method, params_value(params)?, Some(timeout)) + .await?; + let id = response.id.clone(); + let value = match response.result { + Ok(value) => value, + Err(error) => { + return Err(match error.data { + ErrorData::VersionMismatch { expected, .. } => ClientError::VersionMismatch { + plane: self.plane, + local: local_version, + remote: expected, + }, + _ => ClientError::Protocol(error), + }); + } + }; + if let Some(actual) = value + .get("plane") + .and_then(|plane| serde_json::from_value::(plane.clone()).ok()) + && actual != self.plane + { + return Err(ClientError::WrongPlane { + expected: self.plane, + actual, + }); + } + if let Some(remote) = value.get("protocolVersion").and_then(Value::as_u64) + && remote != u64::from(local_version) + { + return Err(ClientError::VersionMismatch { + plane: self.plane, + local: local_version, + remote: u32::try_from(remote).unwrap_or(u32::MAX), + }); + } + decode(Response { + id, + result: Ok(value), + }) + } + + /// Send one request and wait for its response, at most `timeout`. + pub(crate) async fn request( + &self, + method: &str, + params: Value, + timeout: Option, + ) -> Reply { + let started = Instant::now(); + if let Some(error) = self.shared.terminated() { + return Err(error); + } + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let request = Request { + id: Id::Number(id), + method: method.to_owned(), + params, + }; + let text = request.to_text(); + check_size(&text)?; + match &self.backend { + Backend::Socket { outgoing } => { + let mut receiver = self.shared.register(id)?; + let _pending = PendingCall { + shared: &self.shared, + outgoing, + id, + cancel: self.plane == Plane::Data + && manifest::find(Plane::Data, method).is_some_and(|spec| spec.abortable), + }; + if outgoing.send(Outgoing::Text(text)).is_err() { + return Err(self.ended()); + } + match timeout { + None => receiver.await.unwrap_or_else(|_| Err(self.ended())), + Some(timeout) => match tokio::time::timeout(timeout, &mut receiver).await { + Ok(reply) => reply.unwrap_or_else(|_| Err(self.ended())), + // A response that raced the deadline still counts. + Err(_) => receiver.try_recv().unwrap_or_else(|_| { + Err(ClientError::DeadlineExceeded { + elapsed_ms: elapsed_ms(started), + }) + }), + }, + } + } + Backend::InProcess { handler } => { + let call = handler(request); + let response = match timeout { + None => call.await?, + Some(timeout) => { + tokio::time::timeout(timeout, call).await.map_err(|_| { + ClientError::DeadlineExceeded { + elapsed_ms: elapsed_ms(started), + } + })?? + } + }; + // The response crosses the same encoding a socket would + // apply, the server's message limit included. + match jsonrpc::parse(&response.to_text()) { + Ok(RpcMessage::Response(response)) if response.id == Some(Id::Number(id)) => { + Ok(response) + } + Ok(_) | Err(_) => Err(ClientError::InvalidMessage(format!( + "the in-process handler did not answer request {id}" + ))), + } + } + } + } + + fn ended(&self) -> ClientError { + self.shared.terminated().unwrap_or(ClientError::Closed { + code: None, + reason: None, + }) + } +} + +/// A registered call. Dropping it (the call finished, ran past its +/// deadline, or its future was dropped) removes the entry; an entry that +/// was still unanswered gets `$/cancelRequest` when the method is +/// abortable. +struct PendingCall<'a> { + shared: &'a Shared, + outgoing: &'a mpsc::UnboundedSender, + id: i64, + cancel: bool, +} + +impl Drop for PendingCall<'_> { + fn drop(&mut self) { + if !self.shared.forget(self.id) || !self.cancel || self.shared.terminated().is_some() { + return; + } + if let Ok(notification) = Notification::new( + CancelRequest::NAME, + &CancelRequestParams { + id: Id::Number(self.id), + }, + ) { + let _ = self.outgoing.send(Outgoing::Text(notification.to_text())); + } + } +} + +fn elapsed_ms(started: Instant) -> u64 { + u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) +} + +fn check_size(text: &str) -> Result<(), ClientError> { + let actual_bytes = text.len() as u64; + if actual_bytes > MAX_MESSAGE_BYTES { + return Err(ClientError::PayloadTooLarge { + limit_bytes: MAX_MESSAGE_BYTES, + actual_bytes, + }); + } + Ok(()) +} + +fn params_value(params: &P) -> Result { + match serde_json::to_value(params) { + Ok(value @ Value::Object(_)) => Ok(value), + Ok(_) => Err(ClientError::Encode( + "params must encode as a JSON object".to_owned(), + )), + Err(error) => Err(ClientError::Encode(error.to_string())), + } +} + +fn decode(response: Response) -> Result { + response.decode().map_err(|error| match error { + jsonrpc::ResponseError::Failed(error) => ClientError::Protocol(error), + jsonrpc::ResponseError::Undecodable(message) => ClientError::Decode(message), + }) +} + +async fn write_loop( + mut sink: SplitSink, Message>, + mut outgoing: mpsc::UnboundedReceiver, + shared: Arc, + keepalive: Duration, + // Dropped when the writer stops, which starts the reader's close grace. + _done: oneshot::Sender<()>, +) where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut ticker = tokio::time::interval_at(Instant::now() + keepalive, keepalive); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + tokio::select! { + command = outgoing.recv() => match command { + Some(Outgoing::Text(text)) => { + if let Err(error) = sink.send(Message::text(text)).await { + shared.terminate(ClientError::Transport(error.to_string())); + return; + } + } + Some(Outgoing::Close { frame, ack }) => { + let written = match frame { + Some(frame) => sink + .send(Message::Close(Some(frame))) + .await + .map_err(|error| error.to_string()), + None => Ok(()), + }; + if let Some(ack) = ack { + let _ = ack.send(written); + } + return; + } + // Every client handle is gone. + None => { + let _ = sink + .send(Message::Close(Some(CloseFrame { + code: CloseCode::Normal, + reason: "".into(), + }))) + .await; + return; + } + }, + _ = ticker.tick() => { + if shared.has_pending() + && let Err(error) = sink.send(Message::Ping(Default::default())).await + { + shared.terminate(ClientError::Transport(error.to_string())); + return; + } + } + } + } +} + +async fn read_loop( + mut stream: SplitStream>, + shared: Arc, + outgoing: mpsc::WeakUnboundedSender, + mut writer_stopped: oneshot::Receiver<()>, +) where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut grace_until: Option = None; + let violation = loop { + let frame = tokio::select! { + frame = stream.next() => frame, + _ = &mut writer_stopped, if grace_until.is_none() => { + grace_until = Some(Instant::now() + CLOSE_GRACE); + continue; + } + _ = tokio::time::sleep_until(grace_until.unwrap_or_else(Instant::now)), + if grace_until.is_some() => break None, + }; + match frame { + None => break None, + Some(Err(tungstenite::Error::ConnectionClosed | tungstenite::Error::AlreadyClosed)) => { + break None; + } + Some(Err(tungstenite::Error::Capacity(error))) => { + break Some(( + CloseCode::Size, + format!("frame over the {MAX_MESSAGE_BYTES}-byte message limit: {error}"), + )); + } + Some(Err(error)) => { + shared.terminate(ClientError::Transport(error.to_string())); + break None; + } + Some(Ok(Message::Text(text))) => match jsonrpc::parse(&text) { + Ok(RpcMessage::Response(response)) => match &response.id { + Some(Id::Number(id)) => shared.resolve(*id, response), + // This client never sends string ids: nothing waits. + Some(Id::String(_)) => {} + None => { + let error = response + .result + .err() + .map(|error| error.to_string()) + .unwrap_or_default(); + break Some(( + CloseCode::Protocol, + format!("the server answered with a null id: {error}"), + )); + } + }, + Ok(RpcMessage::Request(request)) => { + break Some(( + CloseCode::Protocol, + format!("the server sent a request ({})", request.method), + )); + } + Ok(RpcMessage::Notification(notification)) => { + break Some(( + CloseCode::Protocol, + format!("the server sent a notification ({})", notification.method), + )); + } + Err(rejection) => { + break Some(( + CloseCode::Protocol, + format!("unacceptable frame: {}", rejection.error), + )); + } + }, + Some(Ok(Message::Binary(_))) => { + break Some(( + CloseCode::Protocol, + "binary frames are not part of the protocol".to_owned(), + )); + } + Some(Ok(Message::Close(frame))) => { + // Keep reading: the next read flushes the close reply and + // then ends the stream. + shared.terminate(ClientError::Closed { + code: frame.as_ref().map(|frame| u16::from(frame.code)), + reason: frame + .map(|frame| frame.reason.to_string()) + .filter(|reason| !reason.is_empty()), + }); + } + Some(Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_))) => {} + } + }; + let frame = match violation { + Some((code, message)) => { + shared.terminate(ClientError::InvalidMessage(message.clone())); + let mut reason = message; + truncate_reason(&mut reason); + Some(CloseFrame { + code, + reason: reason.into(), + }) + } + None => { + shared.terminate(ClientError::Closed { + code: None, + reason: None, + }); + None + } + }; + if let Some(outgoing) = outgoing.upgrade() { + let _ = outgoing.send(Outgoing::Close { frame, ack: None }); + } +} + +/// A close reason fits in a control frame: at most 123 bytes. +fn truncate_reason(reason: &mut String) { + const MAX: usize = 123; + if reason.len() > MAX { + let mut end = MAX; + while !reason.is_char_boundary(end) { + end -= 1; + } + reason.truncate(end); + } +} + +#[cfg(test)] +mod tests { + use environment_protocol::{ + data::idle::{GetIdleParams, GetIdleResponse}, + error::ProtocolError, + manifest::{EnvGetIdle, TargetGet}, + }; + + use super::*; + + fn idle_handler(response: Value) -> InProcessHandler { + Arc::new(move |request: Request| { + let response = response.clone(); + Box::pin(async move { Ok(Response::success(request.id, &response)) }) + }) + } + + #[tokio::test(flavor = "current_thread")] + async fn in_process_calls_decode_strictly() { + let connection = Connection::in_process( + Plane::Data, + idle_handler(serde_json::json!({"idleForMs": 5})), + ); + let idle = connection + .call::(&GetIdleParams {}, None) + .await + .unwrap(); + assert_eq!( + idle, + GetIdleResponse { + idle_for_ms: 5, + ..GetIdleResponse::default() + } + ); + let broken = Connection::in_process( + Plane::Data, + idle_handler(serde_json::json!({"idleForMs": "soon"})), + ); + assert!(matches!( + broken.call::(&GetIdleParams {}, None).await, + Err(ClientError::Decode(_)) + )); + } + + #[tokio::test(flavor = "current_thread")] + async fn server_errors_keep_their_kind() { + let connection = Connection::in_process( + Plane::Data, + Arc::new(|request: Request| { + Box::pin(async move { + Ok(Response::failure( + Some(request.id), + ProtocolError::not_initialized(), + )) + }) + }), + ); + let error = connection + .call::(&GetIdleParams {}, None) + .await + .unwrap_err(); + assert_eq!( + error.protocol_kind(), + Some(environment_protocol::error::ProtocolErrorKind::NotInitialized) + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn methods_of_another_plane_are_refused_before_sending() { + let connection = Connection::in_process(Plane::Data, idle_handler(serde_json::json!({}))); + let params = serde_json::from_value(serde_json::json!({ + "target": {"owner": {"universeId": "u", "environmentId": "e"}} + })) + .unwrap(); + assert!(matches!( + connection.call::(¶ms, None).await, + Err(ClientError::Encode(_)) + )); + } + + #[tokio::test(flavor = "current_thread")] + async fn closed_connections_refuse_calls() { + let connection = Connection::in_process( + Plane::Data, + idle_handler(serde_json::json!({"idleForMs": 0})), + ); + connection.close().await.unwrap(); + assert!(connection.is_closed()); + assert!(matches!( + connection.call::(&GetIdleParams {}, None).await, + Err(ClientError::Closed { + code: Some(1000), + .. + }) + )); + } + + #[test] + fn oversized_requests_are_refused_with_their_size() { + let text = "x".repeat(MAX_MESSAGE_BYTES as usize + 1); + assert!(matches!( + check_size(&text), + Err(ClientError::PayloadTooLarge { limit_bytes: MAX_MESSAGE_BYTES, actual_bytes }) + if actual_bytes == MAX_MESSAGE_BYTES + 1 + )); + } +} diff --git a/crates/environment-client/src/control.rs b/crates/environment-client/src/control.rs deleted file mode 100644 index 94f39013e..000000000 --- a/crates/environment-client/src/control.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! Typed controller-plane client methods. - -use environment_protocol::control::{ - handshake::{ControllerInitializeParams, ControllerInitializeResponse}, - ingress::{EnsureIngressParams, IngressResponse, RemoveIngressParams}, - methods::{ - ADOPT_TARGET_METHOD, CLOSE_TARGET_METHOD, CREATE_TARGET_METHOD, ENSURE_INGRESS_METHOD, - GET_TARGET_METHOD, INITIALIZE_METHOD, LIST_TARGETS_METHOD, LIST_TEMPLATES_METHOD, - REMOVE_INGRESS_METHOD, SET_TARGET_POWER_METHOD, - }, - targets::{ - AdoptTargetParams, AdoptTargetResponse, CloseTargetParams, CloseTargetResponse, - CreateTargetParams, CreateTargetResponse, GetTargetParams, GetTargetResponse, - ListTargetsParams, ListTargetsResponse, ListTemplatesParams, ListTemplatesResponse, - SetTargetPowerParams, SetTargetPowerResponse, - }, -}; - -use crate::{ - error::EnvironmentClientResult, - rpc::{JsonRpcClient, JsonRpcTransport}, - transport::{WebSocketConnectOptions, WebSocketTransport}, -}; - -pub struct EnvironmentProviderClient { - rpc: JsonRpcClient, -} - -impl EnvironmentProviderClient -where - T: JsonRpcTransport, -{ - pub fn new(transport: T) -> Self { - Self { - rpc: JsonRpcClient::new(transport), - } - } - - pub fn from_rpc(rpc: JsonRpcClient) -> Self { - Self { rpc } - } - - pub fn into_rpc(self) -> JsonRpcClient { - self.rpc - } - - /// Gracefully close the underlying transport. - pub async fn close(&mut self) -> EnvironmentClientResult<()> { - self.rpc.close().await - } - - pub async fn initialize( - &mut self, - params: &ControllerInitializeParams, - ) -> EnvironmentClientResult { - self.rpc.request(INITIALIZE_METHOD, params).await - } - - pub async fn list_targets( - &mut self, - params: &ListTargetsParams, - ) -> EnvironmentClientResult { - self.rpc.request(LIST_TARGETS_METHOD, params).await - } - - pub async fn list_templates( - &mut self, - params: &ListTemplatesParams, - ) -> EnvironmentClientResult { - self.rpc.request(LIST_TEMPLATES_METHOD, params).await - } - - pub async fn create_target( - &mut self, - params: &CreateTargetParams, - ) -> EnvironmentClientResult { - self.rpc.request(CREATE_TARGET_METHOD, params).await - } - - pub async fn adopt_target( - &mut self, - params: &AdoptTargetParams, - ) -> EnvironmentClientResult { - self.rpc.request(ADOPT_TARGET_METHOD, params).await - } - - pub async fn get_target( - &mut self, - params: &GetTargetParams, - ) -> EnvironmentClientResult { - self.rpc.request(GET_TARGET_METHOD, params).await - } - - pub async fn close_target( - &mut self, - params: &CloseTargetParams, - ) -> EnvironmentClientResult { - self.rpc.request(CLOSE_TARGET_METHOD, params).await - } - - pub async fn set_target_power( - &mut self, - params: &SetTargetPowerParams, - ) -> EnvironmentClientResult { - self.rpc.request(SET_TARGET_POWER_METHOD, params).await - } - - pub async fn ensure_ingress( - &mut self, - params: &EnsureIngressParams, - ) -> EnvironmentClientResult { - self.rpc.request(ENSURE_INGRESS_METHOD, params).await - } - - pub async fn remove_ingress( - &mut self, - params: &RemoveIngressParams, - ) -> EnvironmentClientResult { - self.rpc.request(REMOVE_INGRESS_METHOD, params).await - } -} - -impl EnvironmentProviderClient { - pub async fn connect( - endpoint: &str, - options: WebSocketConnectOptions, - ) -> EnvironmentClientResult { - Ok(Self::new( - WebSocketTransport::connect(endpoint, options).await?, - )) - } -} diff --git a/crates/environment-client/src/controller.rs b/crates/environment-client/src/controller.rs new file mode 100644 index 000000000..7a0afbc54 --- /dev/null +++ b/crates/environment-client/src/controller.rs @@ -0,0 +1,166 @@ +//! The controller-plane client: the runtime's connection to one provider +//! controller. + +use std::{sync::Arc, time::Duration}; + +use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, + control::handshake::{ControllerInitializeParams, ControllerInitializeResponse}, + handshake::Plane, + jsonrpc::{Request, Response}, + manifest::{ + ControllerInitialize, IngressEnsure, IngressRemove, RequestMethod, TargetClose, + TargetCreate, TargetGet, TargetSetPower, TemplateList, + }, +}; +use futures_util::future::BoxFuture; + +use crate::{ + connection::{Connection, InProcessHandler}, + data::typed_calls, + error::ClientError, + options::ConnectOptions, +}; + +/// A controller-plane connection after a successful `initialize`. +/// +/// Same shape as [`DataClient`](crate::DataClient): cheap to clone, calls +/// take `&self` and may run concurrently, a dropped call is abandoned. The +/// controller plane has no `$/cancelRequest`; a mutation abandoned by its +/// caller still runs to completion on the provider. +#[derive(Clone)] +pub struct ControllerClient { + inner: Arc, +} + +struct Inner { + connection: Connection, + initialize: ControllerInitializeResponse, +} + +impl ControllerClient { + /// Dial the controller URL and run `initialize` with + /// [`CONTROLLER_PROTOCOL_VERSION`]. Pass the provider credential, when + /// one is configured, as [`ConnectOptions::bearer_token`]; without one + /// no `Authorization` header is sent. Fails with + /// [`ClientError::VersionMismatch`] or [`ClientError::WrongPlane`] + /// before the connection is used. + pub async fn connect_initialized( + endpoint: &str, + options: ConnectOptions, + ) -> Result<(Self, ControllerInitializeResponse), ClientError> { + let connection = Connection::connect(endpoint, &options, Plane::Controller).await?; + let initialized = initialize(&connection, &options).await; + match initialized { + Ok(initialize) => Ok(( + Self { + inner: Arc::new(Inner { + connection, + initialize: initialize.clone(), + }), + }, + initialize, + )), + Err(error) => { + let _ = connection.close().await; + Err(error) + } + } + } + + /// A client whose requests `handler` answers in process, for tests that + /// need no socket (see [`DataClient::in_process`](crate::DataClient::in_process)). + pub fn in_process( + initialize: ControllerInitializeResponse, + handler: impl Fn(Request) -> BoxFuture<'static, Result> + + Send + + Sync + + 'static, + ) -> Self { + let handler: InProcessHandler = Arc::new(handler); + Self { + inner: Arc::new(Inner { + connection: Connection::in_process(Plane::Controller, handler), + initialize, + }), + } + } + + /// The controller's `initialize` response: its capabilities. + pub fn initialize_response(&self) -> &ControllerInitializeResponse { + &self.inner.initialize + } + + /// Call any controller-plane method, with the connection's default + /// deadline ([`ConnectOptions::request_timeout`]). + pub async fn call( + &self, + params: &M::Params, + ) -> Result { + self.inner.connection.call::(params, None).await + } + + /// [`call`](Self::call) with this deadline instead of the default one. + pub async fn call_with_timeout( + &self, + params: &M::Params, + timeout: Duration, + ) -> Result { + self.inner.connection.call::(params, Some(timeout)).await + } + + /// Close the connection for every clone. Idempotent. + pub async fn close(&self) -> Result<(), ClientError> { + self.inner.connection.close().await + } + + /// True once the connection ended, by either side. + pub fn is_closed(&self) -> bool { + self.inner.connection.is_closed() + } + + typed_calls! { + /// `template/list`. + list_templates => TemplateList; + /// `target/create`. + create_target => TargetCreate; + /// `target/get`. + get_target => TargetGet; + /// `target/setPower`. + set_target_power => TargetSetPower; + /// `target/close`. + close_target => TargetClose; + /// `ingress/ensure`. + ensure_ingress => IngressEnsure; + /// `ingress/remove`. + remove_ingress => IngressRemove; + } +} + +impl std::fmt::Debug for ControllerClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ControllerClient") + .field("server_info", &self.inner.initialize.server_info) + .field("closed", &self.is_closed()) + .finish_non_exhaustive() + } +} + +/// Run the controller `initialize` on a dialed connection. +pub(crate) async fn initialize( + connection: &Connection, + options: &ConnectOptions, +) -> Result { + connection + .initialize( + ControllerInitialize::NAME, + &ControllerInitializeParams { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + client_info: options.client_info.clone(), + }, + CONTROLLER_PROTOCOL_VERSION, + options.initialize_timeout, + ) + .await +} diff --git a/crates/environment-client/src/data.rs b/crates/environment-client/src/data.rs index ffdd96a6a..f13d82286 100644 --- a/crates/environment-client/src/data.rs +++ b/crates/environment-client/src/data.rs @@ -1,257 +1,255 @@ -//! Typed data-plane client methods. - -use environment_protocol::data::methods::{FS_CAPTURE_METHOD, FS_MATERIALIZE_METHOD}; -use environment_protocol::data::transfer::*; -use environment_protocol::data::{ - fs::{ - CopyParams, CopyResponse, CreateDirectoryParams, CreateDirectoryResponse, - GetMetadataParams, GetMetadataResponse, GlobFilesParams, GlobFilesResponse, - ReadDirectoryParams, ReadDirectoryResponse, ReadFileParams, ReadFileResponse, RemoveParams, - RemoveResponse, SearchTextParams, SearchTextResponse, WriteFileParams, WriteFileResponse, - }, - handshake::{InitializeParams, InitializeResponse, InitializedParams}, - idle::{IdleParams, IdleResponse}, - jobs::{ - CancelJobsParams, CancelJobsResponse, ListJobsParams, ListJobsResponse, ReadJobsParams, - ReadJobsResponse, StartJobsParams, StartJobsResponse, - }, - methods::{ - ENV_IDLE_METHOD, FS_COPY_METHOD, FS_CREATE_DIRECTORY_METHOD, FS_GET_METADATA_METHOD, - FS_GLOB_FILES_METHOD, FS_READ_DIRECTORY_METHOD, FS_READ_FILE_METHOD, FS_REMOVE_METHOD, - FS_SEARCH_TEXT_METHOD, FS_WRITE_FILE_METHOD, INITIALIZE_METHOD, INITIALIZED_METHOD, - JOB_CANCEL_METHOD, JOB_LIST_METHOD, JOB_READ_METHOD, JOB_START_METHOD, PROCESS_READ_METHOD, - PROCESS_RESIZE_METHOD, PROCESS_START_METHOD, PROCESS_TERMINATE_METHOD, - PROCESS_WRITE_METHOD, - }, - process::{ - ReadProcessParams, ReadProcessResponse, ResizeProcessParams, ResizeProcessResponse, - StartProcessParams, StartProcessResponse, TerminateProcessParams, TerminateProcessResponse, - WriteProcessParams, WriteProcessResponse, +//! The data-plane client: the runtime's connection to one envd. + +use std::{sync::Arc, time::Duration}; + +use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + data::handshake::{InitializeParams, InitializeResponse}, + handshake::Plane, + jsonrpc::{Request, Response}, + manifest::{ + DataInitialize, EnvGetIdle, FsCreateDirectory, FsGetMetadata, FsGlobFiles, FsReadDirectory, + FsReadFile, FsRemove, FsRename, FsScan, FsSearchText, FsWriteFile, JobCancel, JobRead, + JobStart, ProcessRead, ProcessStart, ProcessTerminate, ProcessWrite, RequestMethod, + TransferAbort, TransferAdvance, TransferAppend, TransferBegin, TransferCommit, + TransferGetStatus, TransferListMissing, TransferRead, TransferReadInventory, TransferWrite, }, }; +use futures_util::future::BoxFuture; +use tokio::io::{AsyncRead, AsyncWrite}; use crate::{ - error::EnvironmentClientResult, - rpc::{JsonRpcClient, JsonRpcNotification, JsonRpcTransport}, - transport::{WebSocketConnectOptions, WebSocketTransport}, + connection::{Connection, InProcessHandler}, + error::ClientError, + options::ConnectOptions, }; -pub struct EnvironmentDataClient { - rpc: JsonRpcClient, +/// Typed sugar: one method per manifest entry, each a [`call`] of its +/// marker. +/// +/// [`call`]: DataClient::call +macro_rules! typed_calls { + ($( $(#[$doc:meta])* $name:ident => $marker:ident; )+) => { + $( + $(#[$doc])* + pub async fn $name( + &self, + params: &<$marker as RequestMethod>::Params, + ) -> Result<<$marker as RequestMethod>::Response, ClientError> { + self.call::<$marker>(params).await + } + )+ + }; +} +pub(crate) use typed_calls; + +/// A data-plane connection after a successful `initialize`. +/// +/// Cheap to clone; clones share one socket. Calls take `&self`, may run +/// concurrently and are answered in any order. Dropping a call's future +/// abandons it (an abortable one is sent `$/cancelRequest`). The socket +/// closes when [`close`](Self::close) is called or the last clone is +/// dropped. +#[derive(Clone)] +pub struct DataClient { + inner: Arc, } -impl EnvironmentDataClient -where - T: JsonRpcTransport, -{ - pub fn new(transport: T) -> Self { +struct Inner { + connection: Connection, + initialize: InitializeResponse, +} + +impl DataClient { + /// Dial `endpoint` (`ws://`, `wss://` or `unix:///path`) and run + /// `initialize` with [`ENVD_PROTOCOL_VERSION`]. Fails with + /// [`ClientError::VersionMismatch`] or [`ClientError::WrongPlane`] + /// before the connection is used; sends nothing else. + pub async fn connect_initialized( + endpoint: &str, + options: ConnectOptions, + ) -> Result<(Self, InitializeResponse), ClientError> { + let connection = Connection::connect(endpoint, &options, Plane::Data).await?; + Self::initialize(connection, &options).await + } + + /// [`connect_initialized`](Self::connect_initialized) over a byte + /// stream that is already connected to a data server, such as one end + /// of [`tokio::io::duplex`] or an accepted Unix socket. The upgrade + /// targets `/` with `Host: localhost`. + pub async fn connect_initialized_over( + stream: S, + options: ConnectOptions, + ) -> Result<(Self, InitializeResponse), ClientError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let connection = Connection::connect_over(stream, &options, Plane::Data).await?; + Self::initialize(connection, &options).await + } + + /// A client whose requests `handler` answers in process, for tests that + /// need no socket. `initialize` is what + /// [`initialize_response`](Self::initialize_response) reports; the + /// handler sees every later request, with the id this client chose. + /// Requests and responses pass the same size limit and strict decoding + /// as on a socket; calls have no default deadline. A handler `Err` fails + /// that one call with the error, a stand-in for a lost response. + pub fn in_process( + initialize: InitializeResponse, + handler: impl Fn(Request) -> BoxFuture<'static, Result> + + Send + + Sync + + 'static, + ) -> Self { + let handler: InProcessHandler = Arc::new(handler); Self { - rpc: JsonRpcClient::new(transport), + inner: Arc::new(Inner { + connection: Connection::in_process(Plane::Data, handler), + initialize, + }), } } - pub fn from_rpc(rpc: JsonRpcClient) -> Self { - Self { rpc } - } - - pub fn into_rpc(self) -> JsonRpcClient { - self.rpc - } - - /// Gracefully close the underlying transport. - pub async fn close(&mut self) -> EnvironmentClientResult<()> { - self.rpc.close().await - } - - pub async fn initialize( - &mut self, - params: &InitializeParams, - ) -> EnvironmentClientResult { - self.rpc.request(INITIALIZE_METHOD, params).await - } - - pub async fn initialized(&mut self, params: &InitializedParams) -> EnvironmentClientResult<()> { - self.rpc.notify(INITIALIZED_METHOD, params).await - } - - pub async fn scan( - &mut self, - params: &environment_protocol::data::inventory::ScanParams, - ) -> EnvironmentClientResult { - self.rpc - .request(environment_protocol::data::methods::FS_SCAN_METHOD, params) - .await - } - - pub async fn transfer( - &mut self, - params: &environment_protocol::data::transfer_session::TransferRequest, - ) -> EnvironmentClientResult - { - self.rpc - .request( - environment_protocol::data::methods::FS_TRANSFER_METHOD, - params, + async fn initialize( + connection: Connection, + options: &ConnectOptions, + ) -> Result<(Self, InitializeResponse), ClientError> { + let params = InitializeParams { + protocol_version: ENVD_PROTOCOL_VERSION, + client_info: options.client_info.clone(), + }; + let initialized = connection + .initialize::<_, InitializeResponse>( + DataInitialize::NAME, + ¶ms, + ENVD_PROTOCOL_VERSION, + options.initialize_timeout, ) - .await - } - - pub async fn capture( - &mut self, - params: &CaptureParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_CAPTURE_METHOD, params).await - } - - pub async fn materialize( - &mut self, - params: &MaterializeParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_MATERIALIZE_METHOD, params).await - } - - pub async fn read_file( - &mut self, - params: &ReadFileParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_READ_FILE_METHOD, params).await - } - - pub async fn write_file( - &mut self, - params: &WriteFileParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_WRITE_FILE_METHOD, params).await - } - - pub async fn create_directory( - &mut self, - params: &CreateDirectoryParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_CREATE_DIRECTORY_METHOD, params).await - } - - pub async fn get_metadata( - &mut self, - params: &GetMetadataParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_GET_METADATA_METHOD, params).await - } - - pub async fn read_directory( - &mut self, - params: &ReadDirectoryParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_READ_DIRECTORY_METHOD, params).await - } - - pub async fn remove( - &mut self, - params: &RemoveParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_REMOVE_METHOD, params).await - } - - pub async fn copy(&mut self, params: &CopyParams) -> EnvironmentClientResult { - self.rpc.request(FS_COPY_METHOD, params).await - } - - pub async fn search_text( - &mut self, - params: &SearchTextParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_SEARCH_TEXT_METHOD, params).await - } - - pub async fn glob_files( - &mut self, - params: &GlobFilesParams, - ) -> EnvironmentClientResult { - self.rpc.request(FS_GLOB_FILES_METHOD, params).await - } - - pub async fn start_process( - &mut self, - params: &StartProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_START_METHOD, params).await - } - - pub async fn read_process( - &mut self, - params: &ReadProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_READ_METHOD, params).await - } - - pub async fn write_process( - &mut self, - params: &WriteProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_WRITE_METHOD, params).await - } - - pub async fn terminate_process( - &mut self, - params: &TerminateProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_TERMINATE_METHOD, params).await - } - - pub async fn resize_process( - &mut self, - params: &ResizeProcessParams, - ) -> EnvironmentClientResult { - self.rpc.request(PROCESS_RESIZE_METHOD, params).await - } - - pub async fn start_jobs( - &mut self, - params: &StartJobsParams, - ) -> EnvironmentClientResult { - self.rpc.request(JOB_START_METHOD, params).await - } - - pub async fn list_jobs( - &mut self, - params: &ListJobsParams, - ) -> EnvironmentClientResult { - self.rpc.request(JOB_LIST_METHOD, params).await - } - - pub async fn read_jobs( - &mut self, - params: &ReadJobsParams, - ) -> EnvironmentClientResult { - self.rpc.request(JOB_READ_METHOD, params).await - } - - pub async fn cancel_jobs( - &mut self, - params: &CancelJobsParams, - ) -> EnvironmentClientResult { - self.rpc.request(JOB_CANCEL_METHOD, params).await - } - - /// Daemon activity report for idle power policy. Not itself activity. - pub async fn idle(&mut self, params: &IdleParams) -> EnvironmentClientResult { - self.rpc.request(ENV_IDLE_METHOD, params).await + .await; + match initialized { + Ok(initialize) => Ok(( + Self { + inner: Arc::new(Inner { + connection, + initialize: initialize.clone(), + }), + }, + initialize, + )), + Err(error) => { + let _ = connection.close().await; + Err(error) + } + } } - pub async fn next_notification( - &mut self, - ) -> EnvironmentClientResult> { - self.rpc.next_notification().await + /// The daemon's `initialize` response: instance id, capabilities, + /// ceilings and default cwd. + pub fn initialize_response(&self) -> &InitializeResponse { + &self.inner.initialize + } + + /// Call any data-plane method, with the connection's default deadline + /// ([`ConnectOptions::request_timeout`]). + pub async fn call( + &self, + params: &M::Params, + ) -> Result { + self.inner.connection.call::(params, None).await + } + + /// [`call`](Self::call) with this deadline instead of the default one. + /// When it passes, an abortable method is sent `$/cancelRequest` and the + /// call fails with [`ClientError::DeadlineExceeded`]. + pub async fn call_with_timeout( + &self, + params: &M::Params, + timeout: Duration, + ) -> Result { + self.inner.connection.call::(params, Some(timeout)).await + } + + /// Close the connection for every clone: pending calls fail with + /// [`ClientError::Closed`]. Idempotent. + pub async fn close(&self) -> Result<(), ClientError> { + self.inner.connection.close().await + } + + /// True once the connection ended, by either side. + pub fn is_closed(&self) -> bool { + self.inner.connection.is_closed() + } + + typed_calls! { + /// `env/getIdle`. + get_idle => EnvGetIdle; + /// `fs/readFile`. + read_file => FsReadFile; + /// `fs/writeFile`. + write_file => FsWriteFile; + /// `fs/createDirectory`. + create_directory => FsCreateDirectory; + /// `fs/getMetadata`. + get_metadata => FsGetMetadata; + /// `fs/readDirectory`. + read_directory => FsReadDirectory; + /// `fs/remove`. + remove => FsRemove; + /// `fs/rename`. + rename => FsRename; + /// `fs/searchText`. + search_text => FsSearchText; + /// `fs/globFiles`. + glob_files => FsGlobFiles; + /// `fs/scan`. + scan => FsScan; + /// `transfer/begin`. + transfer_begin => TransferBegin; + /// `transfer/advance`. + transfer_advance => TransferAdvance; + /// `transfer/getStatus`. + transfer_get_status => TransferGetStatus; + /// `transfer/readInventory`. + transfer_read_inventory => TransferReadInventory; + /// `transfer/append`. + transfer_append => TransferAppend; + /// `transfer/listMissing`. + transfer_list_missing => TransferListMissing; + /// `transfer/read`. + transfer_read => TransferRead; + /// `transfer/write`. + transfer_write => TransferWrite; + /// `transfer/commit`. + transfer_commit => TransferCommit; + /// `transfer/abort`. + transfer_abort => TransferAbort; + /// `process/start`. + start_process => ProcessStart; + /// `process/read`. + read_process => ProcessRead; + /// `process/write`. + write_process => ProcessWrite; + /// `process/terminate`. + terminate_process => ProcessTerminate; + /// `job/start`. + start_jobs => JobStart; + /// `job/read`. + read_jobs => JobRead; + /// `job/cancel`. + cancel_jobs => JobCancel; } } -impl EnvironmentDataClient { - pub async fn connect( - endpoint: &str, - options: WebSocketConnectOptions, - ) -> EnvironmentClientResult { - Ok(Self::new( - WebSocketTransport::connect(endpoint, options).await?, - )) +impl std::fmt::Debug for DataClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("DataClient") + .field( + "daemon_instance_id", + &self.inner.initialize.daemon_instance_id, + ) + .field("closed", &self.is_closed()) + .finish_non_exhaustive() } } diff --git a/crates/environment-client/src/dial.rs b/crates/environment-client/src/dial.rs new file mode 100644 index 000000000..e75f94d32 --- /dev/null +++ b/crates/environment-client/src/dial.rs @@ -0,0 +1,360 @@ +//! Endpoint dialing: `ws://`, `wss://` and `unix:///path`, with the +//! protocol's message limits and the caller's headers. + +use std::time::Duration; + +use environment_protocol::{ + limits::MAX_MESSAGE_BYTES, + routes::{DataRouteRefusal, parse_retry_after_ms}, +}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::UnixStream, +}; +use tokio_tungstenite::{ + WebSocketStream, client_async_with_config, connect_async_with_config, + tungstenite::{ + self, + client::IntoClientRequest, + handshake::client::Request, + http::{ + HeaderMap, HeaderName, HeaderValue, + header::{AUTHORIZATION, RETRY_AFTER, USER_AGENT}, + }, + protocol::WebSocketConfig, + }, +}; + +use crate::{error::ClientError, options::ConnectOptions}; + +/// The WebSocket settings of every client socket: one explicit limit for +/// messages and frames, the protocol's [`MAX_MESSAGE_BYTES`]. +pub(crate) fn websocket_config() -> WebSocketConfig { + WebSocketConfig::default() + .max_message_size(Some(MAX_MESSAGE_BYTES as usize)) + .max_frame_size(Some(MAX_MESSAGE_BYTES as usize)) +} + +enum Target<'a> { + /// `ws://` or `wss://`: dialed as given. + Url, + /// `unix:///path`: the socket path; the upgrade targets `/` with + /// `Host: localhost`. + Unix(&'a str), +} + +fn target(endpoint: &str) -> Result, ClientError> { + if endpoint.starts_with("ws://") || endpoint.starts_with("wss://") { + return Ok(Target::Url); + } + if let Some(path) = endpoint.strip_prefix("unix://") { + if path.len() > 1 && path.starts_with('/') && !path.contains(['?', '#']) { + return Ok(Target::Unix(path)); + } + return Err(configuration( + endpoint, + "a unix endpoint names an absolute socket path: unix:///path/to/socket", + )); + } + Err(configuration( + endpoint, + "the endpoint scheme must be ws://, wss:// or unix://", + )) +} + +fn configuration(endpoint: &str, message: impl Into) -> ClientError { + ClientError::Configuration { + endpoint: endpoint.to_owned(), + message: message.into(), + } +} + +fn upgrade_request( + endpoint: &str, + url: &str, + options: &ConnectOptions, +) -> Result { + let mut request = url + .into_client_request() + .map_err(|error| configuration(endpoint, error.to_string()))?; + let headers = request.headers_mut(); + let user_agent = HeaderValue::from_str(&options.effective_user_agent()) + .map_err(|_| configuration(endpoint, "the user agent is not a valid header value"))?; + headers.insert(USER_AGENT, user_agent); + if let Some(token) = &options.bearer_token { + let mut value = HeaderValue::from_str(&format!("Bearer {}", token.expose())) + .map_err(|_| configuration(endpoint, "the bearer token is not a valid header value"))?; + value.set_sensitive(true); + headers.insert(AUTHORIZATION, value); + } + for (name, value) in &options.headers { + let header = HeaderName::from_bytes(name.as_bytes()) + .map_err(|_| configuration(endpoint, format!("invalid header name {name:?}")))?; + let value = HeaderValue::from_str(value).map_err(|_| { + configuration(endpoint, format!("header {name:?} has an invalid value")) + })?; + headers.insert(header, value); + } + Ok(request) +} + +fn dial_error(endpoint: &str, error: tungstenite::Error) -> ClientError { + match error { + tungstenite::Error::Http(response) => { + let status = response.status().as_u16(); + let retry_after_ms = retry_after_ms(response.headers()); + let refusal = DataRouteRefusal::from_refused_upgrade( + status, + response.body().as_deref(), + retry_after_ms, + ); + ClientError::UpgradeRejected { + status, + refusal, + retry_after_ms, + } + } + tungstenite::Error::Url(error) => configuration(endpoint, error.to_string()), + error => ClientError::Connect { + endpoint: endpoint.to_owned(), + message: error.to_string(), + }, + } +} + +fn retry_after_ms(headers: &HeaderMap) -> Option { + parse_retry_after_ms(headers.get(RETRY_AFTER)?.to_str().ok()?) +} + +async fn bounded( + endpoint: &str, + timeout: Duration, + dial: impl Future>, +) -> Result { + tokio::time::timeout(timeout, dial) + .await + .map_err(|_| ClientError::ConnectTimeout { + endpoint: endpoint.to_owned(), + })? +} + +/// A dialed socket of either transport. +pub(crate) enum Dialed { + Tcp(Box>>), + Unix(Box>), +} + +/// Dial `endpoint` and complete the WebSocket upgrade within +/// `options.connect_timeout`. +pub(crate) async fn dial(endpoint: &str, options: &ConnectOptions) -> Result { + match target(endpoint)? { + Target::Url => { + let request = upgrade_request(endpoint, endpoint, options)?; + bounded(endpoint, options.connect_timeout, async { + connect_async_with_config(request, Some(websocket_config()), true) + .await + .map(|(socket, _)| Dialed::Tcp(Box::new(socket))) + .map_err(|error| dial_error(endpoint, error)) + }) + .await + } + Target::Unix(path) => { + let request = upgrade_request(endpoint, "ws://localhost/", options)?; + bounded(endpoint, options.connect_timeout, async { + let stream = + UnixStream::connect(path) + .await + .map_err(|error| ClientError::Connect { + endpoint: endpoint.to_owned(), + message: error.to_string(), + })?; + client_async_with_config(request, stream, Some(websocket_config())) + .await + .map(|(socket, _)| Dialed::Unix(Box::new(socket))) + .map_err(|error| dial_error(endpoint, error)) + }) + .await + } + } +} + +/// Complete the WebSocket upgrade over an already connected byte stream +/// (request target `/`, `Host: localhost`) within +/// `options.connect_timeout`. +pub(crate) async fn upgrade( + stream: S, + options: &ConnectOptions, +) -> Result, ClientError> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + const ENDPOINT: &str = "ws://localhost/"; + let request = upgrade_request(ENDPOINT, ENDPOINT, options)?; + bounded(ENDPOINT, options.connect_timeout, async { + client_async_with_config(request, stream, Some(websocket_config())) + .await + .map(|(socket, _)| socket) + .map_err(|error| dial_error(ENDPOINT, error)) + }) + .await +} + +#[cfg(test)] +mod tests { + use environment_protocol::{handshake::ImplementationInfo, limits::RETRY_AFTER_MAX_SECS}; + + use super::*; + + fn options() -> ConnectOptions { + ConnectOptions::new( + ImplementationInfo::new("lightspeed-test") + .and_then(|info| info.with_version("1.0.0")) + .unwrap(), + ) + } + + #[test] + fn endpoints_select_their_dialer() { + assert!(matches!(target("ws://127.0.0.1:1/x"), Ok(Target::Url))); + assert!(matches!(target("wss://example.test"), Ok(Target::Url))); + assert!(matches!( + target("unix:///run/envd.sock"), + Ok(Target::Unix("/run/envd.sock")) + )); + for bad in [ + "http://example.test", + "unix://relative.sock", + "unix:///", + "unix:///run/envd.sock?x=1", + "stdio://", + ] { + assert!( + matches!(target(bad), Err(ClientError::Configuration { .. })), + "{bad}" + ); + } + } + + #[test] + fn retry_after_is_whole_seconds_clamped_to_the_maximum() { + let header = |value: &str| { + let mut headers = HeaderMap::new(); + headers.insert(RETRY_AFTER, HeaderValue::from_str(value).unwrap()); + retry_after_ms(&headers) + }; + assert_eq!(header("3"), Some(3_000)); + assert_eq!(header("3600"), Some(RETRY_AFTER_MAX_SECS * 1_000)); + assert_eq!(header("Wed, 21 Oct 2026 07:28:00 GMT"), None); + assert_eq!(retry_after_ms(&HeaderMap::new()), None); + } + + #[test] + fn rejected_upgrades_decode_a_refusal_body() { + let response = tungstenite::http::Response::builder() + .status(503) + .header(RETRY_AFTER, "2") + .body(Some(br#"{"state":"booting"}"#.to_vec())) + .unwrap(); + let error = dial_error("ws://h/", tungstenite::Error::Http(Box::new(response))); + assert!( + matches!( + &error, + ClientError::UpgradeRejected { + status: 503, + refusal: Some(DataRouteRefusal::Booting {}), + retry_after_ms: Some(2_000), + } + ), + "{error:?}" + ); + let garbled = tungstenite::http::Response::builder() + .status(503) + .body(Some(b"busy".to_vec())) + .unwrap(); + assert!(matches!( + dial_error("ws://h/", tungstenite::Error::Http(Box::new(garbled))), + ClientError::UpgradeRejected { + status: 503, + refusal: None, + retry_after_ms: None, + } + )); + } + + #[test] + fn a_refusal_whose_body_was_not_read_falls_back_on_its_status() { + let refused = |status: u16, retry_after: Option<&str>, body: Option>| { + let mut response = tungstenite::http::Response::builder().status(status); + if let Some(retry_after) = retry_after { + response = response.header(RETRY_AFTER, retry_after); + } + match dial_error( + "ws://h/", + tungstenite::Error::Http(Box::new(response.body(body).unwrap())), + ) { + ClientError::UpgradeRejected { + refusal, + retry_after_ms, + .. + } => (refusal, retry_after_ms), + error => panic!("expected a refused upgrade, got {error:?}"), + } + }; + // The tail read with the headers is empty, or there is none. + assert_eq!( + refused(410, None, Some(Vec::new())), + (Some(DataRouteRefusal::Gone {}), None) + ); + assert_eq!( + refused(410, None, None), + (Some(DataRouteRefusal::Gone {}), None) + ); + assert!(matches!( + refused(409, None, Some(Vec::new())), + (Some(DataRouteRefusal::Failed { .. }), None) + )); + assert_eq!( + refused(503, Some("4"), Some(Vec::new())), + (Some(DataRouteRefusal::Booting {}), Some(4_000)) + ); + // A partial body does not decode either. + assert_eq!( + refused(503, Some("4"), Some(br#"{"state":"wak"#.to_vec())), + (Some(DataRouteRefusal::Booting {}), Some(4_000)) + ); + // A decoded body wins over the status. + assert_eq!( + refused(503, Some("4"), Some(br#"{"state":"waking"}"#.to_vec())), + (Some(DataRouteRefusal::Waking {}), Some(4_000)) + ); + // Statuses that name no refusal. + assert_eq!(refused(503, None, Some(Vec::new())), (None, None)); + assert_eq!(refused(401, None, None), (None, None)); + assert_eq!(refused(404, Some("4"), None), (None, Some(4_000))); + } + + #[test] + fn upgrade_requests_carry_agent_credential_and_headers() { + let request = upgrade_request( + "ws://h/", + "ws://h/", + &options() + .with_bearer_token("secret") + .with_header("x-lightspeed-test", "1"), + ) + .unwrap(); + let headers = request.headers(); + assert_eq!(headers[USER_AGENT], "lightspeed-test/1.0.0"); + assert_eq!(headers[AUTHORIZATION], "Bearer secret"); + assert!(headers[AUTHORIZATION].is_sensitive()); + assert_eq!(headers["x-lightspeed-test"], "1"); + assert!(matches!( + upgrade_request( + "ws://h/", + "ws://h/", + &options().with_header("bad name", "1") + ), + Err(ClientError::Configuration { .. }) + )); + } +} diff --git a/crates/environment-client/src/error.rs b/crates/environment-client/src/error.rs index 60843fc7d..bdc541d31 100644 --- a/crates/environment-client/src/error.rs +++ b/crates/environment-client/src/error.rs @@ -1,24 +1,285 @@ -//! Client-side environment protocol errors. +//! Client-side errors. -use environment_protocol::error::EnvironmentProtocolError; +use environment_protocol::{ + error::{ProtocolError, ProtocolErrorKind}, + handshake::Plane, + routes::DataRouteRefusal, +}; use thiserror::Error; -pub type EnvironmentClientResult = Result; +/// Why a connect or call failed. +/// +/// A server's error answer is [`ClientError::Protocol`] with the server's +/// error kind intact; everything else is a fact about the connection or a +/// contract breach the client detected itself. The client never +/// synthesizes a protocol error kind. +#[derive(Clone, Debug, Error)] +pub enum ClientError { + /// The connection settings cannot be used: an endpoint that is not + /// `ws://`, `wss://` or `unix:///path`, or a header that is not valid + /// HTTP. Retrying does not help. + #[error("invalid connection settings for {endpoint}: {message}")] + Configuration { endpoint: String, message: String }, -#[derive(Debug, Error)] -pub enum EnvironmentClientError { - #[error("failed to serialize JSON-RPC message: {0}")] - Serialize(#[from] serde_json::Error), + /// TCP or Unix connect, TLS or the WebSocket upgrade failed below HTTP. + #[error("connect to {endpoint}: {message}")] + Connect { endpoint: String, message: String }, - #[error("transport closed")] - TransportClosed, + /// Connect, TLS and upgrade did not finish within + /// [`ConnectOptions::connect_timeout`](crate::ConnectOptions::connect_timeout). + #[error("connect to {endpoint} timed out")] + ConnectTimeout { endpoint: String }, - #[error("transport error: {0}")] - Transport(String), + /// The server answered the upgrade with an HTTP status instead of + /// switching protocols: 401 for a missing or wrong credential, or a + /// data route's 503, 410 or 409. `refusal` is the response body when it + /// decodes as a [`DataRouteRefusal`], else the refusal the status alone + /// names (410 `gone`, 409 `failed`, a 503 with `Retry-After` `booting`), + /// so a body split from the headers by the transport still types the + /// refusal; `retry_after_ms` is the + /// `Retry-After` header in milliseconds, clamped to + /// [`RETRY_AFTER_MAX_SECS`](environment_protocol::limits::RETRY_AFTER_MAX_SECS). + #[error("upgrade refused with HTTP {status}{}", describe_refusal(refusal.as_ref()))] + UpgradeRejected { + status: u16, + refusal: Option, + retry_after_ms: Option, + }, + + /// The server speaks another protocol version on this plane: it + /// answered `initialize` with `versionMismatch`, or with a different + /// `protocolVersion`. `local` is this client's version. + #[error("{plane} protocol mismatch: this client speaks {local}, the server {remote}")] + VersionMismatch { + plane: Plane, + local: u32, + remote: u32, + }, + + /// The server's `initialize` response names another plane: the + /// endpoint serves something else. + #[error("expected the {expected} plane, the server serves {actual}")] + WrongPlane { expected: Plane, actual: Plane }, + + /// The server answered the call with this error. + #[error("{0}")] + Protocol(ProtocolError), + + /// The call's deadline passed before its response arrived. On the data + /// plane an abortable call was also sent `$/cancelRequest`. + #[error("deadline exceeded after {elapsed_ms} ms")] + DeadlineExceeded { elapsed_ms: u64 }, + + /// The connection is closed: by the server (with its close code and + /// reason when it sent a close frame), by the transport ending, or by + /// [`close`](crate::DataClient::close). + #[error("connection closed{}", describe_close(*code, reason.as_deref()))] + Closed { + code: Option, + reason: Option, + }, + + /// The encoded request is larger than the protocol's message limit; it + /// was not sent. + #[error("request of {actual_bytes} bytes exceeds the {limit_bytes}-byte limit")] + PayloadTooLarge { limit_bytes: u64, actual_bytes: u64 }, - #[error("invalid JSON-RPC message: {0}")] + /// The request could not be encoded. + #[error("encode request: {0}")] + Encode(String), + + /// The server's result does not decode as the method's response type: + /// the server broke the contract. + #[error("decode response: {0}")] + Decode(String), + + /// The server sent something no client accepts on this plane: a + /// request or notification, a binary frame, a frame over the message + /// limit, or a frame that is not a valid response. The client closed + /// the connection and failed every pending call with this error. + #[error("invalid message from server: {0}")] InvalidMessage(String), - #[error("environment protocol error: {0:?}")] - Protocol(EnvironmentProtocolError), + /// The socket failed after the connection was established. + #[error("transport: {0}")] + Transport(String), +} + +impl ClientError { + /// The server's error kind: the kind of a [`Protocol`](Self::Protocol) + /// error, and `versionMismatch` for + /// [`VersionMismatch`](Self::VersionMismatch). `None` for failures the + /// client observed itself. + pub fn protocol_kind(&self) -> Option { + match self { + Self::Protocol(error) => Some(error.kind()), + Self::VersionMismatch { .. } => Some(ProtocolErrorKind::VersionMismatch), + _ => None, + } + } + + /// The server's error, for [`Protocol`](Self::Protocol). + pub fn protocol_error(&self) -> Option<&ProtocolError> { + match self { + Self::Protocol(error) => Some(error), + _ => None, + } + } + + /// The data route's typed refusal, for an + /// [`UpgradeRejected`](Self::UpgradeRejected) whose body or status + /// names one. + pub fn data_route_refusal(&self) -> Option<&DataRouteRefusal> { + match self { + Self::UpgradeRejected { refusal, .. } => refusal.as_ref(), + _ => None, + } + } + + /// The refused upgrade's `Retry-After`, in milliseconds. + pub fn retry_after_ms(&self) -> Option { + match self { + Self::UpgradeRejected { retry_after_ms, .. } => *retry_after_ms, + _ => None, + } + } + + /// True when the same call may succeed later without any change: a + /// retryable protocol kind (`busy`, `unavailable`, `aborted`), a + /// connection failure, a closed connection, a deadline, or an upgrade + /// refused with 429 or a 5xx status. Version and plane mismatches, + /// contract breaches and refusals are not. + pub fn is_retryable(&self) -> bool { + match self { + Self::Protocol(error) => error.is_retryable(), + Self::Connect { .. } + | Self::ConnectTimeout { .. } + | Self::Closed { .. } + | Self::Transport(_) + | Self::DeadlineExceeded { .. } => true, + Self::UpgradeRejected { status, .. } => *status == 429 || (500..600).contains(status), + Self::Configuration { .. } + | Self::VersionMismatch { .. } + | Self::WrongPlane { .. } + | Self::PayloadTooLarge { .. } + | Self::Encode(_) + | Self::Decode(_) + | Self::InvalidMessage(_) => false, + } + } +} + +fn describe_refusal(refusal: Option<&DataRouteRefusal>) -> String { + refusal.map_or_else(String::new, |refusal| format!(": {refusal}")) +} + +fn describe_close(code: Option, reason: Option<&str>) -> String { + match (code, reason) { + (Some(code), Some(reason)) if !reason.is_empty() => format!(" ({code}: {reason})"), + (Some(code), _) => format!(" ({code})"), + (None, _) => String::new(), + } +} + +#[cfg(test)] +mod tests { + use environment_protocol::error::{AbortReason, ProtocolError}; + + use super::*; + + #[test] + fn protocol_errors_keep_their_kind_and_retryability() { + let aborted = ClientError::Protocol(ProtocolError::aborted( + Some(AbortReason::Cancelled), + "cancelled", + )); + assert_eq!(aborted.protocol_kind(), Some(ProtocolErrorKind::Aborted)); + assert!(aborted.is_retryable()); + let missing = ClientError::Protocol(ProtocolError::not_found("no such process")); + assert_eq!(missing.protocol_kind(), Some(ProtocolErrorKind::NotFound)); + assert!(!missing.is_retryable()); + assert_eq!(missing.to_string(), "notFound: no such process"); + } + + #[test] + fn connection_failures_are_retryable_and_breaches_are_not() { + assert!( + ClientError::Closed { + code: None, + reason: None + } + .is_retryable() + ); + assert!(ClientError::DeadlineExceeded { elapsed_ms: 10 }.is_retryable()); + assert!(rejected(503, None).is_retryable()); + assert!(rejected(429, None).is_retryable()); + assert!(!rejected(401, None).is_retryable()); + assert!(!rejected(410, Some(DataRouteRefusal::Gone {})).is_retryable()); + assert!(!ClientError::Decode("x".to_owned()).is_retryable()); + assert!( + !ClientError::VersionMismatch { + plane: Plane::Data, + local: 4, + remote: 3 + } + .is_retryable() + ); + assert_eq!( + ClientError::VersionMismatch { + plane: Plane::Controller, + local: 4, + remote: 5 + } + .protocol_kind(), + Some(ProtocolErrorKind::VersionMismatch) + ); + } + + fn rejected(status: u16, refusal: Option) -> ClientError { + ClientError::UpgradeRejected { + status, + refusal, + retry_after_ms: None, + } + } + + #[test] + fn upgrade_refusals_display_their_state() { + assert_eq!( + rejected(401, None).to_string(), + "upgrade refused with HTTP 401" + ); + let waking = ClientError::UpgradeRejected { + status: 503, + refusal: Some(DataRouteRefusal::Waking {}), + retry_after_ms: Some(3_000), + }; + assert_eq!(waking.to_string(), "upgrade refused with HTTP 503: waking"); + assert_eq!( + waking.data_route_refusal(), + Some(&DataRouteRefusal::Waking {}) + ); + assert_eq!(waking.retry_after_ms(), Some(3_000)); + assert_eq!(rejected(401, None).data_route_refusal(), None); + } + + #[test] + fn close_display_names_code_and_reason() { + assert_eq!( + ClientError::Closed { + code: Some(1008), + reason: Some("route closed".to_owned()) + } + .to_string(), + "connection closed (1008: route closed)" + ); + assert_eq!( + ClientError::Closed { + code: None, + reason: None + } + .to_string(), + "connection closed" + ); + } } diff --git a/crates/environment-client/src/lib.rs b/crates/environment-client/src/lib.rs index 4717b0236..8cb949316 100644 --- a/crates/environment-client/src/lib.rs +++ b/crates/environment-client/src/lib.rs @@ -1,16 +1,66 @@ -//! Typed client for environment protocol data-plane and controller-plane calls. +//! Clients of the environment protocol's data and controller planes. //! -//! The crate owns transport and JSON-RPC mechanics while reusing pure protocol -//! records from `environment-protocol`. +//! [`DataClient`] talks to one envd, [`ControllerClient`] to one provider +//! controller. Both are opened with `connect_initialized`, which dials +//! (`ws://`, `wss://` or `unix:///path`), runs `initialize` and fails with a +//! typed [`ClientError::VersionMismatch`] or [`ClientError::WrongPlane`] +//! before anything else is sent. Calls are typed by the protocol's method +//! markers ([`environment_protocol::manifest`]): +//! +//! ```no_run +//! # async fn example() -> Result<(), environment_client::ClientError> { +//! use environment_client::{ConnectOptions, DataClient}; +//! use environment_protocol::{ +//! data::idle::GetIdleParams, handshake::ImplementationInfo, manifest::EnvGetIdle, +//! }; +//! +//! let info = ImplementationInfo::new("lightspeed-runtime").expect("valid name"); +//! let (client, initialize) = +//! DataClient::connect_initialized("ws://127.0.0.1:7070", ConnectOptions::new(info)).await?; +//! let idle = client.call::(&GetIdleParams {}).await?; +//! # let _ = (initialize, idle); +//! # Ok(()) +//! # } +//! ``` +//! +//! # Connection behaviour +//! +//! - **Multiplexed.** Calls take `&self`, may be issued concurrently on +//! one connection and are matched to responses by id. Clients are cheap +//! to clone. +//! - **Cancel-safe.** A call whose future is dropped, or whose deadline +//! passes, is forgotten; its late response is discarded. On the data +//! plane an abandoned abortable call (`process/read`, `job/read`, +//! `fs/searchText`, `fs/globFiles`, `fs/scan`) is also sent +//! `$/cancelRequest`. +//! - **Deadlines.** [`ConnectOptions`] bounds connect and `initialize` +//! (10 s each by default) and can set a default deadline for every call; +//! `call_with_timeout` overrides it per call. +//! - **Keepalive.** While any call is pending the client sends a WebSocket +//! ping every [`ConnectOptions::keepalive_interval`] (20 s by default), +//! so relays never idle-close a long wait. +//! - **Limits.** Messages and frames are limited to +//! [`MAX_MESSAGE_BYTES`](environment_protocol::limits::MAX_MESSAGE_BYTES) +//! in both directions; a larger request fails with +//! [`ClientError::PayloadTooLarge`] without being sent. +//! - **Servers only answer.** A request or notification from the server, a +//! binary frame or an unparseable frame closes the connection and fails +//! every pending call with [`ClientError::InvalidMessage`]. +//! +//! Tests that need no socket build clients with `in_process`. + +mod connection; +mod controller; +mod data; +mod dial; +mod error; +mod options; -pub mod control; -pub mod data; -pub mod error; -pub mod rpc; -pub mod transport; +#[cfg(feature = "conformance")] +pub mod conformance; -pub use control::EnvironmentProviderClient; -pub use data::EnvironmentDataClient; -pub use error::{EnvironmentClientError, EnvironmentClientResult}; -pub use rpc::{JsonRpcClient, JsonRpcNotification, JsonRpcTransport}; -pub use transport::{WebSocketConnectOptions, WebSocketTransport}; +pub use connection::InProcessHandler; +pub use controller::ControllerClient; +pub use data::DataClient; +pub use error::ClientError; +pub use options::{ConnectOptions, DEFAULT_CONNECT_TIMEOUT, DEFAULT_INITIALIZE_TIMEOUT}; diff --git a/crates/environment-client/src/options.rs b/crates/environment-client/src/options.rs new file mode 100644 index 000000000..3c482d9df --- /dev/null +++ b/crates/environment-client/src/options.rs @@ -0,0 +1,112 @@ +//! Connection settings. + +use std::{fmt, time::Duration}; + +use environment_protocol::{ + bytes::SecretString, handshake::ImplementationInfo, limits::KEEPALIVE_INTERVAL_MS, +}; + +/// Default bound of connect, TLS and WebSocket upgrade together. +pub const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +/// Default bound of the `initialize` round trip. +pub const DEFAULT_INITIALIZE_TIMEOUT: Duration = Duration::from_secs(10); + +/// How to dial and greet an endpoint. +#[derive(Clone)] +pub struct ConnectOptions { + /// Sent as `clientInfo` in `initialize`, and as the default + /// `User-Agent` (`name/version`). + pub client_info: ImplementationInfo, + /// Sent as `Authorization: Bearer ` on the upgrade request. + pub bearer_token: Option, + /// Overrides the `User-Agent` derived from `client_info`. + pub user_agent: Option, + /// Extra upgrade request headers. + pub headers: Vec<(String, String)>, + /// Bound of TCP or Unix connect, TLS and the WebSocket upgrade + /// together. Default [`DEFAULT_CONNECT_TIMEOUT`]. + pub connect_timeout: Duration, + /// Bound of the `initialize` round trip. Default + /// [`DEFAULT_INITIALIZE_TIMEOUT`]. + pub initialize_timeout: Duration, + /// Default deadline of every call made through the client; `None` + /// means calls wait as long as the caller does (tool paths rely on + /// their activity deadline). Background paths set one. Default `None`. + pub request_timeout: Option, + /// Ping interval while any call is pending (the waiting peer keeps + /// intermediaries from idle-closing a long wait). Default + /// [`KEEPALIVE_INTERVAL_MS`]. + pub keepalive_interval: Duration, +} + +impl ConnectOptions { + pub fn new(client_info: ImplementationInfo) -> Self { + Self { + client_info, + bearer_token: None, + user_agent: None, + headers: Vec::new(), + connect_timeout: DEFAULT_CONNECT_TIMEOUT, + initialize_timeout: DEFAULT_INITIALIZE_TIMEOUT, + request_timeout: None, + keepalive_interval: Duration::from_millis(KEEPALIVE_INTERVAL_MS), + } + } + + pub fn with_bearer_token(mut self, token: impl Into) -> Self { + self.bearer_token = Some(token.into()); + self + } + + pub fn with_user_agent(mut self, user_agent: impl Into) -> Self { + self.user_agent = Some(user_agent.into()); + self + } + + pub fn with_header(mut self, name: impl Into, value: impl Into) -> Self { + self.headers.push((name.into(), value.into())); + self + } + + pub fn with_request_timeout(mut self, timeout: Duration) -> Self { + self.request_timeout = Some(timeout); + self + } + + /// The `User-Agent` sent on the upgrade request. + pub(crate) fn effective_user_agent(&self) -> String { + if let Some(user_agent) = &self.user_agent { + return user_agent.clone(); + } + match self.client_info.version() { + Some(version) => format!("{}/{version}", self.client_info.name()), + None => self.client_info.name().to_owned(), + } + } +} + +impl fmt::Debug for ConnectOptions { + /// Header values and the bearer token are left out: they may carry + /// credentials. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ConnectOptions") + .field("client_info", &self.client_info) + .field("bearer_token", &self.bearer_token) + .field("user_agent", &self.user_agent) + .field( + "headers", + &self + .headers + .iter() + .map(|(name, _)| name.as_str()) + .collect::>(), + ) + .field("connect_timeout", &self.connect_timeout) + .field("initialize_timeout", &self.initialize_timeout) + .field("request_timeout", &self.request_timeout) + .field("keepalive_interval", &self.keepalive_interval) + .finish() + } +} diff --git a/crates/environment-client/src/rpc.rs b/crates/environment-client/src/rpc.rs deleted file mode 100644 index 1a74572c8..000000000 --- a/crates/environment-client/src/rpc.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! Minimal JSON-RPC 2.0 client machinery. - -use std::collections::VecDeque; - -use async_trait::async_trait; -use environment_protocol::error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}; -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::{Value, json}; - -use crate::error::{EnvironmentClientError, EnvironmentClientResult}; - -#[async_trait] -pub trait JsonRpcTransport: Send { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()>; - async fn recv(&mut self) -> EnvironmentClientResult>; - - /// Gracefully close the transport when it has an application-level - /// shutdown handshake. Stateless/in-process transports need no action. - async fn close(&mut self) -> EnvironmentClientResult<()> { - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct JsonRpcNotification { - pub method: String, - pub params: Value, -} - -pub struct JsonRpcClient { - transport: T, - next_id: u64, - notifications: VecDeque, -} - -impl JsonRpcClient -where - T: JsonRpcTransport, -{ - pub fn new(transport: T) -> Self { - Self { - transport, - next_id: 1, - notifications: VecDeque::new(), - } - } - - pub fn into_inner(self) -> T { - self.transport - } - - pub async fn close(&mut self) -> EnvironmentClientResult<()> { - self.transport.close().await - } - - pub async fn request(&mut self, method: &str, params: &P) -> EnvironmentClientResult - where - P: Serialize + Sync, - R: DeserializeOwned, - { - let id = self.next_request_id(); - self.transport - .send(json!({ - "jsonrpc": "2.0", - "id": id, - "method": method, - "params": serde_json::to_value(params)?, - })) - .await?; - - loop { - let message = self - .transport - .recv() - .await? - .ok_or(EnvironmentClientError::TransportClosed)?; - if let Some(notification) = parse_notification(&message)? { - self.notifications.push_back(notification); - continue; - } - - let Some(response_id) = message.get("id").and_then(Value::as_u64) else { - return Err(EnvironmentClientError::InvalidMessage( - "response missing numeric id".to_owned(), - )); - }; - if response_id != id { - return Err(EnvironmentClientError::InvalidMessage(format!( - "unexpected response id {response_id}, expected {id}" - ))); - } - - if let Some(error) = message.get("error") { - return Err(EnvironmentClientError::Protocol(parse_error(error)?)); - } - - let result = message.get("result").ok_or_else(|| { - EnvironmentClientError::InvalidMessage("response missing result".to_owned()) - })?; - return serde_json::from_value(result.clone()).map_err(Into::into); - } - } - - pub async fn notify

(&mut self, method: &str, params: &P) -> EnvironmentClientResult<()> - where - P: Serialize + Sync, - { - self.transport - .send(json!({ - "jsonrpc": "2.0", - "method": method, - "params": serde_json::to_value(params)?, - })) - .await - } - - pub async fn next_notification( - &mut self, - ) -> EnvironmentClientResult> { - if let Some(notification) = self.notifications.pop_front() { - return Ok(Some(notification)); - } - - loop { - let Some(message) = self.transport.recv().await? else { - return Ok(None); - }; - if let Some(notification) = parse_notification(&message)? { - return Ok(Some(notification)); - } - } - } - - fn next_request_id(&mut self) -> u64 { - let id = self.next_id; - self.next_id += 1; - id - } -} - -fn parse_notification(message: &Value) -> EnvironmentClientResult> { - if message.get("id").is_some() { - return Ok(None); - } - - let Some(method) = message.get("method").and_then(Value::as_str) else { - return Ok(None); - }; - let params = message.get("params").cloned().unwrap_or(Value::Null); - Ok(Some(JsonRpcNotification { - method: method.to_owned(), - params, - })) -} - -fn parse_error(value: &Value) -> EnvironmentClientResult { - if let Ok(error) = serde_json::from_value::(value.clone()) { - return Ok(error); - } - - let message = value - .get("message") - .and_then(Value::as_str) - .unwrap_or("JSON-RPC error"); - Ok(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - message, - )) -} diff --git a/crates/environment-client/src/transport.rs b/crates/environment-client/src/transport.rs deleted file mode 100644 index 80418bca5..000000000 --- a/crates/environment-client/src/transport.rs +++ /dev/null @@ -1,114 +0,0 @@ -//! Transport implementations for environment protocol clients. - -use async_trait::async_trait; -use futures_util::{SinkExt, StreamExt}; -use serde_json::Value; -use tokio::net::TcpStream; -use tokio_tungstenite::{ - MaybeTlsStream, WebSocketStream, connect_async, - tungstenite::{ - Message, - client::IntoClientRequest, - http::{ - HeaderName, HeaderValue, - header::{AUTHORIZATION, USER_AGENT}, - }, - }, -}; - -use crate::{ - error::{EnvironmentClientError, EnvironmentClientResult}, - rpc::JsonRpcTransport, -}; - -pub struct WebSocketTransport { - stream: WebSocketStream>, - closed: bool, -} - -impl WebSocketTransport { - pub async fn connect( - endpoint: &str, - options: WebSocketConnectOptions, - ) -> EnvironmentClientResult { - let mut request = endpoint - .into_client_request() - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - - if let Some(user_agent) = options.user_agent { - let value = HeaderValue::from_str(&user_agent) - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - request.headers_mut().insert(USER_AGENT, value); - } - - if let Some(token) = options.bearer_token { - let value = HeaderValue::from_str(&format!("Bearer {token}")) - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - request.headers_mut().insert(AUTHORIZATION, value); - } - - for (name, value) in options.headers { - let name = HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - let value = HeaderValue::from_str(&value) - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - request.headers_mut().insert(name, value); - } - - let (stream, _) = connect_async(request) - .await - .map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - Ok(Self { - stream, - closed: false, - }) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct WebSocketConnectOptions { - pub bearer_token: Option, - pub user_agent: Option, - pub headers: Vec<(String, String)>, -} - -#[async_trait] -impl JsonRpcTransport for WebSocketTransport { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()> { - self.stream - .send(Message::Text(message.to_string().into())) - .await - .map_err(|error| EnvironmentClientError::Transport(error.to_string())) - } - - async fn recv(&mut self) -> EnvironmentClientResult> { - loop { - let Some(message) = self.stream.next().await else { - return Ok(None); - }; - let message = - message.map_err(|error| EnvironmentClientError::Transport(error.to_string()))?; - match message { - Message::Text(text) => { - return serde_json::from_str(&text).map(Some).map_err(Into::into); - } - Message::Binary(bytes) => { - return serde_json::from_slice(&bytes).map(Some).map_err(Into::into); - } - Message::Close(_) => return Ok(None), - Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => {} - } - } - } - - async fn close(&mut self) -> EnvironmentClientResult<()> { - if self.closed { - return Ok(()); - } - self.closed = true; - self.stream - .close(None) - .await - .map_err(|error| EnvironmentClientError::Transport(error.to_string())) - } -} diff --git a/crates/environment-client/tests/client.rs b/crates/environment-client/tests/client.rs index 0c8d27424..55075f7e9 100644 --- a/crates/environment-client/tests/client.rs +++ b/crates/environment-client/tests/client.rs @@ -1,395 +1,749 @@ -use std::collections::{BTreeMap, VecDeque}; +//! Socket behaviour of the data and controller clients against scripted +//! WebSocket servers. + +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; -use async_trait::async_trait; use environment_client::{ - EnvironmentClientError, EnvironmentClientResult, EnvironmentDataClient, - EnvironmentProviderClient, JsonRpcTransport, WebSocketConnectOptions, + ClientError, ConnectOptions, ControllerClient, DataClient, + conformance::controller::{ControllerConformance, run}, }; use environment_protocol::{ - control::{ - handshake::ControllerInitializeParams, - targets::{AdoptTargetParams, ProviderBindingContext}, - }, + CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION, + bytes::ByteChunk, + control::{handshake::ControllerInitializeResponse, target::TargetOwner}, data::{ - fs::ReadFileParams, - jobs::{JobDependencyPolicy, JobStartSpec, ListJobsParams, StartJobsParams}, - methods::PROCESS_OUTPUT_METHOD, + exec::ReturnOn, + fs::{WriteCondition, WriteFileParams}, + handshake::InitializeResponse, + idle::{GetIdleParams, GetIdleResponse}, + process::ReadProcessParams, }, - error::EnvironmentProtocolErrorCode, - shared::{ByteChunk, CURRENT_PROTOCOL_VERSION, EnvironmentPath, JobId}, + error::{ProtocolError, ProtocolErrorKind}, + handshake::{ImplementationInfo, Plane}, + ids::{EnvironmentId, ProcessId, TemplateId, UniverseId}, + jsonrpc::{self, CancelRequestParams, Id, Message as RpcMessage, Request, Response}, + limits::{MAX_MESSAGE_BYTES, RETRY_AFTER_MAX_SECS}, + manifest::{CancelRequest, EnvGetIdle}, + paths::EnvironmentPath, + routes::DataRouteRefusal, }; +use futures_util::{SinkExt, StreamExt}; use serde_json::{Value, json}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::{TcpListener, UnixListener}, + sync::mpsc, +}; +use tokio_tungstenite::{ + WebSocketStream, accept_async, accept_hdr_async, + tungstenite::{ + Message, + handshake::server::{ + ErrorResponse, Request as UpgradeRequest, Response as UpgradeResponse, + }, + http::StatusCode, + }, +}; -#[tokio::test(flavor = "current_thread")] -async fn websocket_client_close_sends_a_close_frame() { - use futures_util::StreamExt as _; - use tokio::net::TcpListener; - use tokio_tungstenite::{accept_async, tungstenite::Message}; +fn fixture(path: &str) -> Value { + let text = match path { + "data" => include_str!("../../environment-protocol/fixtures/data/initialize.json"), + "controller" => { + include_str!("../../environment-protocol/fixtures/controller/initialize.json") + } + other => panic!("no fixture {other}"), + }; + serde_json::from_str::(text).expect("fixture")["response"].clone() +} + +fn options() -> ConnectOptions { + ConnectOptions::new( + ImplementationInfo::new("lightspeed-client-test") + .and_then(|info| info.with_version("0.0.0")) + .expect("client info"), + ) +} +/// Serve one WebSocket connection on a fresh loopback port with `serve`. +async fn server(serve: F) -> String +where + F: FnOnce(WebSocketStream) -> Fut + Send + 'static, + Fut: Future + Send, +{ let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); - let address = listener.local_addr().expect("listener address"); - let server = tokio::spawn(async move { + let url = format!("ws://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { let (stream, _) = listener.accept().await.expect("accept"); - let mut websocket = accept_async(stream).await.expect("websocket handshake"); - while let Some(message) = websocket.next().await { - if matches!(message.expect("websocket message"), Message::Close(_)) { - return true; - } - } - false + serve(accept_async(stream).await.expect("upgrade")).await; }); + url +} - let mut client = EnvironmentDataClient::connect( - &format!("ws://{address}"), - WebSocketConnectOptions::default(), - ) - .await - .expect("connect client"); - client.close().await.expect("close client"); +/// The next frame that is a request, skipping pings. +async fn next_request(socket: &mut WebSocketStream) -> Option +where + S: AsyncRead + AsyncWrite + Unpin, +{ + loop { + match socket.next().await?.ok()? { + Message::Text(text) => match jsonrpc::parse(&text).expect("valid frame") { + RpcMessage::Request(request) => return Some(request), + RpcMessage::Notification(_) | RpcMessage::Response(_) => {} + }, + Message::Close(_) => return None, + _ => {} + } + } +} - let saw_close = tokio::time::timeout(std::time::Duration::from_secs(2), server) +async fn send(socket: &mut WebSocketStream, response: Response) +where + S: AsyncRead + AsyncWrite + Unpin, +{ + socket + .send(Message::text(response.to_text())) .await - .expect("server observed close before timeout") - .expect("server task"); - assert!(saw_close, "client dropped without a WebSocket close frame"); + .expect("send response"); } -#[derive(Default)] -struct MockTransport { - sent: Vec, - recv: VecDeque, +/// Answer the first request, `initialize`, with `result`. +async fn answer_initialize(socket: &mut WebSocketStream, result: Value) +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let request = next_request(socket).await.expect("initialize"); + assert_eq!(request.method, "initialize"); + assert_eq!( + request.params["protocolVersion"], + json!(ENVD_PROTOCOL_VERSION) + ); + send(socket, Response::success(request.id, &result)).await; } -impl MockTransport { - fn with_recv(messages: impl IntoIterator) -> Self { - Self { - sent: Vec::new(), - recv: messages.into_iter().collect(), - } +fn idle(ms: u64) -> GetIdleResponse { + GetIdleResponse { + idle_for_ms: ms, + ..GetIdleResponse::default() } } -#[async_trait] -impl JsonRpcTransport for MockTransport { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()> { - self.sent.push(message); - Ok(()) - } - - async fn recv(&mut self) -> EnvironmentClientResult> { - Ok(self.recv.pop_front()) +fn read_params() -> ReadProcessParams { + ReadProcessParams { + process_id: ProcessId::new("p1").expect("process id"), + after: None, + tail_bytes: None, + ack: None, + max_bytes: 1024, + wait_ms: 60_000, + return_on: ReturnOn::AnyOutput, } } -#[tokio::test] -async fn data_client_sends_typed_request_and_decodes_response() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "data": "aGk=" +#[tokio::test(flavor = "current_thread")] +async fn connect_initialized_returns_the_handshake_and_close_sends_a_close_frame() { + let (saw_close, mut closed) = mpsc::unbounded_channel(); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + while let Some(Ok(message)) = socket.next().await { + if matches!(message, Message::Close(_)) { + let _ = saw_close.send(()); + return; + } } - })]); - let mut client = EnvironmentDataClient::new(transport); - - let response = client - .read_file(&ReadFileParams { - path: EnvironmentPath::new("README.md").expect("path"), - offset: None, - max_bytes: None, - }) + }) + .await; + let (client, initialize) = DataClient::connect_initialized(&url, options()) .await - .expect("response"); - - assert_eq!(response.data, ByteChunk::from(b"hi".as_slice())); + .expect("connect"); + let expected: InitializeResponse = serde_json::from_value(fixture("data")).expect("fixture"); + assert_eq!(initialize, expected); + assert_eq!(client.initialize_response(), &expected); + client.close().await.expect("close"); + assert!(client.is_closed()); + tokio::time::timeout(Duration::from_secs(5), closed.recv()) + .await + .expect("server saw the close frame in time") + .expect("close frame"); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::Closed { .. }) + )); +} - let transport = client.into_rpc().into_inner(); - assert_eq!( - transport.sent, - vec![json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "fs/readFile", - "params": { - "path": "README.md" +#[tokio::test(flavor = "current_thread")] +async fn a_version_mismatch_answer_is_typed() { + let url = server(|mut socket| async move { + let request = next_request(&mut socket).await.expect("initialize"); + let received = request.params["protocolVersion"].as_u64().unwrap() as u32; + send( + &mut socket, + Response::failure( + Some(request.id), + ProtocolError::version_mismatch(2, received), + ), + ) + .await; + while socket.next().await.is_some() {} + }) + .await; + let error = DataClient::connect_initialized(&url, options()) + .await + .expect_err("mismatch"); + assert!( + matches!( + error, + ClientError::VersionMismatch { + plane: Plane::Data, + local: ENVD_PROTOCOL_VERSION, + remote: 2 } - })] + ), + "{error:?}" ); + assert_eq!( + error.protocol_kind(), + Some(ProtocolErrorKind::VersionMismatch) + ); + assert!(!error.is_retryable()); } -#[tokio::test] -async fn data_client_sends_typed_job_start_request() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "jobs": [ - { - "createdAtMs": 1, - "jobId": "job-1", - "namespace": "session_1", - "status": "queued" - } - ] - } - })]); - let mut client = EnvironmentDataClient::new(transport); - - let response = client - .start_jobs(&StartJobsParams { - namespace: "session_1".to_owned(), - request_id: "request-1".to_owned(), - jobs: vec![JobStartSpec { - job_id: JobId::new("job-1"), - name: None, - argv: vec![ - "/bin/sh".to_owned(), - "-c".to_owned(), - "printf ok".to_owned(), - ], - cwd: None, - env: Default::default(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(1_000), - depends_on: Vec::new(), - dependency_policy: JobDependencyPolicy::AllSucceeded, - queue_key: None, - }], - }) - .await - .expect("response"); +#[tokio::test(flavor = "current_thread")] +async fn a_response_with_another_version_is_a_mismatch() { + let mut response = fixture("data"); + response["protocolVersion"] = json!(ENVD_PROTOCOL_VERSION + 1); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, response).await; + while socket.next().await.is_some() {} + }) + .await; + assert!(matches!( + DataClient::connect_initialized(&url, options()).await, + Err(ClientError::VersionMismatch { remote, .. }) if remote == ENVD_PROTOCOL_VERSION + 1 + )); +} - assert_eq!(response.jobs[0].job_id.as_str(), "job-1"); +#[tokio::test(flavor = "current_thread")] +async fn a_controller_answering_a_data_client_is_the_wrong_plane() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("controller")).await; + while socket.next().await.is_some() {} + }) + .await; + assert!(matches!( + DataClient::connect_initialized(&url, options()).await, + Err(ClientError::WrongPlane { + expected: Plane::Data, + actual: Plane::Controller + }) + )); +} - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "job/start"); - assert_eq!(transport.sent[0]["params"]["jobs"][0]["jobId"], "job-1"); - assert_eq!(transport.sent[0]["params"]["jobs"][0]["argv"][0], "/bin/sh"); +#[tokio::test(flavor = "current_thread")] +async fn controller_clients_check_their_own_plane_and_version() { + let url = server(|mut socket| async move { + let request = next_request(&mut socket).await.expect("initialize"); + assert_eq!( + request.params["protocolVersion"], + json!(CONTROLLER_PROTOCOL_VERSION) + ); + send( + &mut socket, + Response::success(request.id, &fixture("controller")), + ) + .await; + while socket.next().await.is_some() {} + }) + .await; + let (client, initialize) = ControllerClient::connect_initialized(&url, options()) + .await + .expect("connect"); + let expected: ControllerInitializeResponse = + serde_json::from_value(fixture("controller")).expect("fixture"); + assert_eq!(initialize, expected); + assert!(initialize.capabilities.ingress); + client.close().await.expect("close"); } -#[tokio::test] -async fn data_client_sends_typed_job_list_request() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "jobs": [ - { - "createdAtMs": 2, - "jobId": "job-2", - "namespace": "session_1", - "status": "running" - } - ] +#[tokio::test(flavor = "current_thread")] +async fn pipelined_calls_are_matched_by_id() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let first = next_request(&mut socket).await.expect("first"); + let second = next_request(&mut socket).await.expect("second"); + // Answer in reverse order; each result names its own id. + for request in [second, first] { + let Id::Number(id) = request.id else { + panic!("numeric ids") + }; + send(&mut socket, Response::success(request.id, &idle(id as u64))).await; } - })]); - let mut client = EnvironmentDataClient::new(transport); - - let response = client - .list_jobs(&ListJobsParams { - namespace: "session_1".to_owned(), - limit: Some(5), - }) + while socket.next().await.is_some() {} + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) .await - .expect("response"); - - assert_eq!(response.jobs[0].job_id.as_str(), "job-2"); - - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "job/list"); - assert_eq!(transport.sent[0]["params"]["namespace"], "session_1"); - assert_eq!(transport.sent[0]["params"]["limit"], 5); + .expect("connect"); + let (first, second) = tokio::join!( + client.call::(&GetIdleParams {}), + client.get_idle(&GetIdleParams {}), + ); + let (first, second) = (first.expect("first"), second.expect("second")); + assert_ne!(first.idle_for_ms, second.idle_for_ms); + assert!( + first.idle_for_ms < second.idle_for_ms, + "first sent, first id" + ); } -#[tokio::test] -async fn data_client_stashes_notifications_seen_while_waiting_for_response() { - let transport = MockTransport::with_recv([ - json!({ - "jsonrpc": "2.0", - "method": PROCESS_OUTPUT_METHOD, - "params": { - "chunk": "b2sK", - "processId": "proc-1", - "seq": 1, - "stream": "stdout" - } - }), - json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "data": "aGk=" - } - }), - ]); - let mut client = EnvironmentDataClient::new(transport); - - client - .read_file(&ReadFileParams { - path: EnvironmentPath::new("README.md").expect("path"), - offset: None, - max_bytes: None, - }) +#[tokio::test(flavor = "current_thread")] +async fn a_server_notification_fails_pending_calls_and_closes() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let _ = next_request(&mut socket).await.expect("request"); + socket + .send(Message::text( + r#"{"jsonrpc":"2.0","method":"process/output","params":{}}"#, + )) + .await + .expect("send notification"); + while socket.next().await.is_some() {} + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) .await - .expect("response"); + .expect("connect"); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::InvalidMessage(_)) + )); + assert!(client.is_closed()); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::InvalidMessage(_)) + )); +} - let notification = client - .next_notification() +#[tokio::test(flavor = "current_thread")] +async fn a_binary_frame_is_a_protocol_violation() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let _ = next_request(&mut socket).await.expect("request"); + socket + .send(Message::binary(b"{}".to_vec())) + .await + .expect("send binary"); + while socket.next().await.is_some() {} + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) .await - .expect("notification read") - .expect("notification"); - assert_eq!(notification.method, PROCESS_OUTPUT_METHOD); - assert_eq!(notification.params["processId"], "proc-1"); + .expect("connect"); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::InvalidMessage(_)) + )); } -#[tokio::test] -async fn data_client_maps_protocol_error_payloads() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "error": { - "code": "notFound", - "message": "missing" +#[tokio::test(flavor = "current_thread")] +async fn a_deadline_cancels_an_abortable_call() { + let (cancelled, mut cancels) = mpsc::unbounded_channel(); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let read = next_request(&mut socket).await.expect("process/read"); + assert_eq!(read.method, "process/read"); + while let Some(Ok(message)) = socket.next().await { + if let Message::Text(text) = message + && let Ok(RpcMessage::Notification(notification)) = jsonrpc::parse(&text) + { + assert_eq!(notification.method, CancelRequest::NAME); + let params: CancelRequestParams = notification.params().expect("params"); + let _ = cancelled.send((params.id, read.id.clone())); + // The late answer is discarded by the client. + send( + &mut socket, + Response::failure(Some(read.id.clone()), ProtocolError::cancelled()), + ) + .await; + } } - })]); - let mut client = EnvironmentDataClient::new(transport); - + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) + .await + .expect("connect"); let error = client - .read_file(&ReadFileParams { - path: EnvironmentPath::new("missing.txt").expect("path"), - offset: None, - max_bytes: None, - }) + .call_with_timeout::( + &read_params(), + Duration::from_millis(50), + ) .await - .expect_err("environment protocol error"); + .expect_err("deadline"); + assert!( + matches!(error, ClientError::DeadlineExceeded { .. }), + "{error:?}" + ); + let (cancelled_id, read_id) = tokio::time::timeout(Duration::from_secs(5), cancels.recv()) + .await + .expect("cancel in time") + .expect("cancel"); + assert_eq!(cancelled_id, read_id); + assert!( + !client.is_closed(), + "a deadline leaves the connection usable" + ); +} - match error { - EnvironmentClientError::Protocol(error) => { - assert_eq!(error.code, EnvironmentProtocolErrorCode::NotFound) +#[tokio::test(flavor = "current_thread")] +async fn a_dropped_abortable_call_is_cancelled() { + let (cancelled, mut cancels) = mpsc::unbounded_channel(); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let read = next_request(&mut socket).await.expect("process/read"); + while let Some(Ok(message)) = socket.next().await { + if let Message::Text(text) = message + && let Ok(RpcMessage::Notification(notification)) = jsonrpc::parse(&text) + { + let params: CancelRequestParams = notification.params().expect("params"); + let _ = cancelled.send(params.id == read.id); + } } - other => panic!("unexpected error {other:?}"), - } + }) + .await; + let (client, _) = DataClient::connect_initialized(&url, options()) + .await + .expect("connect"); + let params = read_params(); + let read = client.read_process(¶ms); + assert!( + tokio::time::timeout(Duration::from_millis(50), read) + .await + .is_err(), + "the read waits" + ); + let matched = tokio::time::timeout(Duration::from_secs(5), cancels.recv()) + .await + .expect("cancel in time") + .expect("cancel"); + assert!(matched, "the cancel names the dropped request"); } -#[tokio::test] -async fn controller_client_sends_typed_initialize_request() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "capabilities": { - "closeTarget": true, - "createTarget": true, - "getTarget": true, - "listTargets": true - }, - "implementation": { - "name": "test-controller", - "version": "0.1.0" - }, - "protocolVersion": 2 +#[tokio::test(flavor = "current_thread")] +async fn pending_calls_keep_the_connection_alive_with_pings() { + let pings = Arc::new(AtomicUsize::new(0)); + let counted = pings.clone(); + let url = server(move |mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + let request = next_request(&mut socket).await.expect("request"); + let quiet = tokio::time::sleep(Duration::from_millis(300)); + tokio::pin!(quiet); + loop { + tokio::select! { + _ = &mut quiet => break, + message = socket.next() => match message { + Some(Ok(Message::Ping(_))) => { counted.fetch_add(1, Ordering::SeqCst); } + Some(Ok(_)) => {} + _ => return, + }, + } } - })]); - let mut client = EnvironmentProviderClient::new(transport); + send(&mut socket, Response::success(request.id, &idle(0))).await; + while socket.next().await.is_some() {} + }) + .await; + let mut options = options(); + options.keepalive_interval = Duration::from_millis(40); + let (client, _) = DataClient::connect_initialized(&url, options) + .await + .expect("connect"); + client.get_idle(&GetIdleParams {}).await.expect("idle"); + assert!( + pings.load(Ordering::SeqCst) >= 3, + "pinged {} times", + pings.load(Ordering::SeqCst) + ); +} - let response = client - .initialize(&ControllerInitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-test".to_owned(), +#[tokio::test(flavor = "current_thread")] +async fn the_default_request_timeout_applies_to_every_call() { + let url = server(|mut socket| async move { + answer_initialize(&mut socket, fixture("data")).await; + while socket.next().await.is_some() {} + }) + .await; + let (client, _) = DataClient::connect_initialized( + &url, + options().with_request_timeout(Duration::from_millis(50)), + ) + .await + .expect("connect"); + assert!(matches!( + client.get_idle(&GetIdleParams {}).await, + Err(ClientError::DeadlineExceeded { .. }) + )); +} + +#[tokio::test(flavor = "current_thread")] +async fn oversized_requests_fail_without_being_sent() { + let calls = Arc::new(AtomicUsize::new(0)); + let seen = calls.clone(); + let client = DataClient::in_process( + serde_json::from_value(fixture("data")).expect("fixture"), + move |request: Request| { + seen.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { Ok(Response::success(request.id, &json!({}))) }) + }, + ); + let error = client + .write_file(&WriteFileParams { + path: EnvironmentPath::new("/workspace/big").expect("path"), + data: ByteChunk::new(vec![0_u8; MAX_MESSAGE_BYTES as usize]), + create: WriteCondition::default(), + if_match: None, + executable: false, }) .await - .expect("response"); + .expect_err("too large"); + assert!(matches!( + error, + ClientError::PayloadTooLarge { limit_bytes: MAX_MESSAGE_BYTES, actual_bytes } + if actual_bytes > MAX_MESSAGE_BYTES + )); + assert_eq!(calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test(flavor = "current_thread")] +async fn in_process_clients_answer_through_the_handler() { + let client = DataClient::in_process( + serde_json::from_value(fixture("data")).expect("fixture"), + |request: Request| { + Box::pin(async move { + assert_eq!(request.method, "env/getIdle"); + Ok(Response::success(request.id, &idle(7))) + }) + }, + ); + assert_eq!( + client.get_idle(&GetIdleParams {}).await.expect("idle"), + idle(7) + ); +} - assert!(response.capabilities.create_target); - assert_eq!(response.implementation.name, "test-controller"); +#[allow(clippy::result_large_err)] +fn refuse_upgrade( + _: &UpgradeRequest, + _: UpgradeResponse, +) -> Result { + let mut response = ErrorResponse::new(None); + *response.status_mut() = StatusCode::UNAUTHORIZED; + Err(response) +} - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "controller/initialize"); - assert_eq!(transport.sent[0]["params"]["clientName"], "lightspeed-test"); +#[tokio::test(flavor = "current_thread")] +async fn an_upgrade_refusal_reports_its_status() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let url = format!("ws://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let _ = accept_hdr_async(stream, refuse_upgrade).await; + }); + let error = DataClient::connect_initialized(&url, options()) + .await + .expect_err("refused"); + assert!( + matches!( + error, + ClientError::UpgradeRejected { + status: 401, + refusal: None, + retry_after_ms: None, + } + ), + "{error:?}" + ); + assert!(!error.is_retryable()); } -#[tokio::test] -async fn controller_client_sends_typed_adoption_request() { - let transport = MockTransport::with_recv([json!({ - "jsonrpc": "2.0", - "id": 1, - "result": { - "target": { - "targetId": "target-adopted", - "status": "starting", - "scope": { "type": "default" }, - "capabilities": {} +/// Serve one data route refusal: `status`, `Retry-After` when given, and a +/// JSON body. +#[allow(clippy::result_large_err)] +async fn refusing_route( + status: StatusCode, + retry_after: Option<&'static str>, + body: Value, +) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let url = format!("ws://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let _ = accept_hdr_async(stream, move |_: &UpgradeRequest, _: UpgradeResponse| { + let mut response = ErrorResponse::new(Some(body.to_string())); + *response.status_mut() = status; + let headers = response.headers_mut(); + headers.insert("content-type", "application/json".parse().unwrap()); + if let Some(retry_after) = retry_after { + headers.insert("retry-after", retry_after.parse().unwrap()); } - } - })]); - let mut client = EnvironmentProviderClient::new(transport); - - let response = client - .adopt_target(&AdoptTargetParams { - request_id: "adopt-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-1".to_owned(), - binding_id: "primary".to_owned(), - }, - source_target: "legacy/hand-built-vm".to_owned(), + Err(response) }) + .await; + }); + url +} + +#[tokio::test(flavor = "current_thread")] +async fn a_data_route_refusal_is_typed_with_its_retry_after() { + let url = refusing_route( + StatusCode::SERVICE_UNAVAILABLE, + Some("3"), + json!({"state": "waking"}), + ) + .await; + let error = DataClient::connect_initialized(&url, options()) + .await + .expect_err("refused"); + assert!( + matches!( + &error, + ClientError::UpgradeRejected { + status: 503, + refusal: Some(DataRouteRefusal::Waking {}), + retry_after_ms: Some(3_000), + } + ), + "{error:?}" + ); + assert!(error.is_retryable()); + assert_eq!(error.to_string(), "upgrade refused with HTTP 503: waking"); + + let url = refusing_route( + StatusCode::SERVICE_UNAVAILABLE, + Some("86400"), + json!({"state": "booting"}), + ) + .await; + let error = DataClient::connect_initialized(&url, options()) .await - .expect("response"); - assert_eq!(response.target.target_id.as_str(), "target-adopted"); + .expect_err("refused"); + assert_eq!(error.retry_after_ms(), Some(RETRY_AFTER_MAX_SECS * 1_000)); - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "controller/adoptTarget"); + let url = refusing_route(StatusCode::GONE, None, json!({"state": "gone"})).await; + let error = DataClient::connect_initialized(&url, options()) + .await + .expect_err("refused"); + assert_eq!(error.data_route_refusal(), Some(&DataRouteRefusal::Gone {})); + assert!(!error.is_retryable()); assert_eq!( - transport.sent[0]["params"]["sourceTarget"], - "legacy/hand-built-vm" + environment_client::conformance::data_route::check_refusal(&error), + Ok(&DataRouteRefusal::Gone {}) ); } #[tokio::test(flavor = "current_thread")] -async fn data_client_transfer_methods_preserve_binary_payloads() { - use environment_protocol::data::transfer::*; - let entries = vec![TransferEntry { - path: "".into(), - content: TransferContent::File { - data: ByteChunk::from(vec![0, 255, 128]), - executable: true, - }, - }]; - let limits = TransferLimits { - max_entries: 1, - max_depth: 0, - max_file_bytes: 3, - max_total_bytes: 3, - max_duration_ms: 1000, - }; - let transport = MockTransport::with_recv([ - json!({"jsonrpc":"2.0", "id":1, "result": {"destination":"/file", "entries":1, "bytes":3, "retiredDirectory":null}}), - json!({"jsonrpc":"2.0", "id":2, "result": {"source":"/file", "entries":entries, "bytes":3}}), - ]); - let mut client = EnvironmentDataClient::new(transport); - let request = MaterializeParams { - destination: EnvironmentPath::new("/file").unwrap(), - entries: entries.clone(), - limits, - on_existing: TransferOnExisting::Error, - }; - assert_eq!(client.materialize(&request).await.unwrap().bytes, 3); - let captured = client - .capture(&CaptureParams { - source: request.destination.clone(), - limits, - }) +async fn an_undecodable_refusal_body_keeps_the_status() { + let url = refusing_route( + StatusCode::SERVICE_UNAVAILABLE, + None, + json!({"state": "sleeping"}), + ) + .await; + let error = DataClient::connect_initialized(&url, options()) + .await + .expect_err("refused"); + assert!( + matches!( + error, + ClientError::UpgradeRejected { + status: 503, + refusal: None, + retry_after_ms: None, + } + ), + "{error:?}" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_silent_server_hits_the_connect_deadline() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let url = format!("ws://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + let (_stream, _) = listener.accept().await.expect("accept"); + tokio::time::sleep(Duration::from_secs(30)).await; + }); + let mut options = options(); + options.connect_timeout = Duration::from_millis(100); + assert!(matches!( + DataClient::connect_initialized(&url, options).await, + Err(ClientError::ConnectTimeout { .. }) + )); +} + +#[tokio::test(flavor = "current_thread")] +async fn unix_endpoints_upgrade_over_the_socket() { + let path = std::path::PathBuf::from(format!( + "/tmp/lightspeed-client-{}-{}.sock", + std::process::id(), + line!() + )); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).expect("bind unix socket"); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let mut socket = accept_async(stream).await.expect("upgrade"); + answer_initialize(&mut socket, fixture("data")).await; + while socket.next().await.is_some() {} + }); + let endpoint = format!("unix://{}", path.display()); + let (client, _) = DataClient::connect_initialized(&endpoint, options()) .await - .unwrap(); - assert_eq!(captured.entries, entries); - let transport = client.into_rpc().into_inner(); - assert_eq!(transport.sent[0]["method"], "fs/materialize"); + .expect("connect over unix"); + client.close().await.expect("close"); + let _ = std::fs::remove_file(&path); +} + +#[tokio::test(flavor = "current_thread")] +async fn clients_connect_over_an_existing_byte_stream() { + let (client_end, server_end) = tokio::io::duplex(64 * 1024); + tokio::spawn(async move { + let mut socket = accept_async(server_end).await.expect("upgrade"); + answer_initialize(&mut socket, fixture("data")).await; + let request = next_request(&mut socket).await.expect("request"); + send(&mut socket, Response::success(request.id, &idle(3))).await; + while socket.next().await.is_some() {} + }); + let (client, _) = DataClient::connect_initialized_over(client_end, options()) + .await + .expect("connect over duplex"); assert_eq!( - transport.sent[0]["params"], - serde_json::to_value(request).unwrap() + client.get_idle(&GetIdleParams {}).await.expect("idle"), + idle(3) ); - assert_eq!(transport.sent[1]["method"], "fs/capture"); +} + +#[tokio::test(flavor = "current_thread")] +async fn conformance_reports_an_unreachable_controller() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let url = format!("ws://{}", listener.local_addr().expect("address")); + drop(listener); + let owner = Arc::new(|| TargetOwner { + universe_id: UniverseId::new("u1").unwrap(), + environment_id: EnvironmentId::new("e1").unwrap(), + }); + let report = run(&ControllerConformance::new( + url, + options().with_bearer_token("credential"), + TemplateId::new("t1").unwrap(), + owner, + )) + .await; + assert!(!report.passed()); + assert_eq!(report.checks.len(), 1); + assert_eq!(report.checks[0].name, "initialize"); + assert!(report.into_result().is_err()); } diff --git a/crates/environment-daemon/Cargo.toml b/crates/environment-daemon/Cargo.toml index c415affcf..5f35ededb 100644 --- a/crates/environment-daemon/Cargo.toml +++ b/crates/environment-daemon/Cargo.toml @@ -6,13 +6,12 @@ edition = "2024" [dependencies] hex = "0.4" anyhow = "1" -api = { path = "../api" } clap = { version = "4", features = ["derive", "env"] } dotenvy = "0.15" release-info = { path = "../release-info" } futures-util = "0.3" flate2 = "1" -glob = "0.3" +globset = "0.4" grep-regex = "0.1" grep-searcher = "0.1" environment-protocol = { path = "../environment-protocol" } @@ -25,7 +24,8 @@ sha2 = "0.11" tar = { version = "0.4", default-features = false } tempfile = "3" thiserror = "2" -tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "process", "rt", "signal", "sync", "time"] } +tokio = { version = "1", features = ["fs", "io-util", "macros", "net", "process", "rt", "rt-multi-thread", "signal", "sync", "time"] } +tokio-util = "0.7" tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] } ed25519-dalek = { version = "3", features = ["rand_core"] } rand = "0.10" @@ -40,7 +40,6 @@ rustix = { version = "1", features = ["fs", "process"] } libc = "0.2" [dev-dependencies] -async-trait = "0.1" store-fs = { path = "../store-fs" } vfs = { path = "../vfs" } harness = { path = "../harness" } diff --git a/crates/environment-daemon/src/config.rs b/crates/environment-daemon/src/config.rs index cb3fd0ed9..65b596ac8 100644 --- a/crates/environment-daemon/src/config.rs +++ b/crates/environment-daemon/src/config.rs @@ -8,8 +8,12 @@ use std::{ }; use anyhow::{Context, Result, bail}; -use clap::{Parser, Subcommand}; -use environment_protocol::{registration::validate_registration_metadata, shared::SecretString}; +use clap::{Parser, Subcommand, ValueEnum}; +use environment_protocol::{ + bytes::SecretString, + metadata::{DisplayName, Metadata}, + routes::{CanonicalOrigin, canonical_origin, gateway_stream_url}, +}; /// Listener address when the daemon is not registering outbound. pub const DEFAULT_LISTEN: &str = "127.0.0.1:19091"; @@ -20,6 +24,13 @@ pub const ENV_PREFIX: &str = "LIGHTSPEED_ENVD_"; /// Internal one-exec handoff for a registration key that configuration has /// already read and scrubbed. Never documented as operator configuration. pub(crate) const REEXEC_REGISTRATION_KEY_ENV: &str = "LIGHTSPEED_ENVD_REEXEC_REGISTRATION_KEY"; +/// Set on the re-executed replacement after an automatic upgrade: the +/// gateway protocol version the upgrade targeted. A later challenge naming +/// that same version is refused instead of upgrading again. +pub(crate) const AUTO_UPGRADE_ATTEMPTED_ENV: &str = "LIGHTSPEED_ENVD_AUTO_UPGRADE_ATTEMPTED"; +/// Name of the state directory below `$XDG_STATE_HOME` (or +/// `$HOME/.local/state`). +pub const STATE_DIR_NAME: &str = "lightspeed-envd"; #[derive(Parser, Debug)] #[command( @@ -39,23 +50,49 @@ pub struct DaemonArgs { #[arg(long, env = "LIGHTSPEED_ENVD_LISTEN")] pub listen: Option, - /// Public environment-gateway connect URL to dial outbound - /// (`wss://host/environment-gateway/connect`). + /// Environment-gateway connect URL to dial outbound + /// (`wss://host[/prefix]/environment-gateway/connect`). Stream sockets + /// are dialed on the same origin and prefix. #[arg(long, env = "LIGHTSPEED_ENVD_GATEWAY_URL", global = true)] pub gateway_url: Option, /// Discovery document used for manual or automatic upgrades. Defaults to - /// `/.well-known/lightspeed-envd` on the configured gateway host. + /// `/.well-known/lightspeed-envd` on the configured gateway origin. #[arg(long, env = "LIGHTSPEED_ENVD_DISCOVERY_URL", global = true)] pub discovery_url: Option, - /// Upgrade automatically when an outbound gateway advertises a different - /// environment protocol version. + /// Accept a plain `ws://` gateway URL toward a non-loopback host, such + /// as a cluster-internal service inside one trusted network. The + /// registration key and every stream then cross that network in the + /// clear. + #[arg( + long, + env = "LIGHTSPEED_ENVD_ALLOW_INSECURE_GATEWAY", + default_value_t = false + )] + pub allow_insecure_gateway: bool, + + /// What the daemon does once the gateway reports its environment + /// closed: `exit` with status 4, or `park` (stop registering and + /// listening, then wait for `SIGTERM` or `SIGINT` and exit 0) so a + /// supervisor that restarts on every exit does not restart a spent + /// identity. + #[arg( + long, + env = "LIGHTSPEED_ENVD_ON_CLOSED", + value_enum, + default_value_t = OnClosed::Exit + )] + pub on_closed: OnClosed, + + /// Upgrade automatically when an outbound gateway challenges with a + /// different environment protocol version. #[arg(long, env = "LIGHTSPEED_ENVD_AUTO_UPGRADE", default_value_t = false)] pub auto_upgrade: bool, /// Registration key admitting this daemon as a new environment. Read - /// once and removed from the process environment. + /// once, removed from the process environment, and sent only until the + /// first admission. #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_KEY", hide_env_values = true)] pub registration_key: Option, @@ -63,15 +100,16 @@ pub struct DaemonArgs { #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE")] pub registration_key_file: Option, - /// Display-name hint recorded on the environment at first registration. + /// Display name recorded on the environment at first registration. #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_NAME")] pub registration_name: Option, - /// Bounded JSON object of string correlation metadata. + /// Bounded JSON object of string correlation metadata, recorded at + /// first registration. #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_METADATA")] pub registration_metadata: Option, - /// Where to write the registration receipt JSON once accepted. + /// Where to write the registration receipt JSON once admitted. #[arg(long, env = "LIGHTSPEED_ENVD_REGISTRATION_RECEIPT")] pub registration_receipt: Option, @@ -85,6 +123,10 @@ pub struct DaemonArgs { #[arg(long, env = "LIGHTSPEED_ENVD_FS_ROOT")] pub fs_root: Option, + /// Daemon key, job records and transfer journal. Defaults to + /// `$XDG_STATE_HOME/lightspeed-envd`, else + /// `$HOME/.local/state/lightspeed-envd`. The data plane never addresses + /// it, wherever it lives. #[arg(long, env = "LIGHTSPEED_ENVD_STATE_DIR")] pub state_dir: Option, @@ -99,6 +141,17 @@ pub struct DaemonArgs { pub print_build: bool, } +/// What a registered daemon does once its environment is closed. Either +/// way it never reconnects: a closed identity is spent. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] +pub enum OnClosed { + /// End the process with status 4. + #[default] + Exit, + /// Stop registering and listening, and wait for `SIGTERM` or `SIGINT`. + Park, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Subcommand)] pub enum DaemonCommand { /// Install the envd build published by the configured gateway. @@ -155,6 +208,8 @@ pub struct DaemonConfig { pub listen: Option, pub cwd: PathBuf, pub fs_root: PathBuf, + /// Created `0700` when missing. Never addressable through the data + /// plane. pub state_dir: PathBuf, pub read_only_fs: bool, /// Outbound registration, when a gateway URL is configured. @@ -167,19 +222,30 @@ pub struct DaemonConfig { /// it is registration-key policy, and the daemon behaves the same either way. #[derive(Clone)] pub struct RegistrationConfig { + /// The gateway connect URL; it must end with `CONNECT_PATH`. pub gateway_url: String, - /// Present only until the first successful registration consumes it. + /// Sent with `registration/register` until the first admission, then + /// dropped (never handed to an upgraded replacement after that). pub registration_key: Option, - pub display_name: Option, - pub metadata: BTreeMap, + /// Read by the gateway only at first admission. + pub display_name: Option, + /// Read by the gateway only at first admission. + pub metadata: Metadata, pub receipt_path: Option, pub ca_file: Option, /// Absolute discovery URL reported on mismatch and used by automatic /// upgrade. pub discovery_url: String, + /// Accept a plain `ws://` gateway URL toward a non-loopback host. + pub allow_insecure_gateway: bool, + /// What the daemon does once its environment is closed. + pub on_closed: OnClosed, /// Present only when mismatch-triggered replacement and re-exec is opted /// in. The argument vector is captured before the daemon starts serving. pub(crate) auto_upgrade: Option, + /// The gateway protocol an automatic upgrade already targeted in this + /// process lineage. + pub(crate) upgrade_attempted_for: Option, } #[derive(Clone)] @@ -202,14 +268,29 @@ impl std::fmt::Debug for RegistrationConfig { .field("receipt_path", &self.receipt_path) .field("ca_file", &self.ca_file) .field("discovery_url", &self.discovery_url) + .field("allow_insecure_gateway", &self.allow_insecure_gateway) + .field("on_closed", &self.on_closed) .field("auto_upgrade", &self.auto_upgrade.is_some()) + .field("upgrade_attempted_for", &self.upgrade_attempted_for) .finish() } } +/// The URLs one gateway connect URL determines. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GatewayUrls { + /// The connect URL as configured. + pub connect: String, + /// What the registration signature binds. + pub origin: CanonicalOrigin, + /// Where every reverse-dialed stream goes. + pub stream: String, +} + impl RegistrationConfig { /// Registration that never replaces the running binary: no key, name, - /// metadata, receipt, or CA file until the caller sets them. The CLI + /// metadata, receipt, or CA file until the caller sets them; `wss://` + /// (or loopback `ws://`) only; exits once closed. The CLI /// attaches the restart invocation for automatic upgrade itself, so it /// stays crate-private. pub fn new(gateway_url: String, discovery_url: String) -> Self { @@ -217,57 +298,85 @@ impl RegistrationConfig { gateway_url, registration_key: None, display_name: None, - metadata: BTreeMap::new(), + metadata: Metadata::default(), receipt_path: None, ca_file: None, discovery_url, + allow_insecure_gateway: false, + on_closed: OnClosed::Exit, auto_upgrade: None, + upgrade_attempted_for: None, } } - pub fn validate(&self) -> Result<()> { - validate_gateway_url(&self.gateway_url)?; - validate_registration_metadata(self.display_name.as_deref(), &self.metadata) - .map_err(|message| anyhow::anyhow!("registration metadata: {message}"))?; + /// The discovery URL must be HTTPS (or HTTP toward loopback) whenever + /// it can download a build. With an insecure gateway and no automatic + /// upgrade it is only named in mismatch reports, so the plain `http://` + /// URL derived from a `ws://` gateway is accepted there. + pub fn validate(&self) -> Result { + let urls = gateway_urls(&self.gateway_url, self.allow_insecure_gateway)?; if let Some(key) = &self.registration_key && (key.is_empty() || key.expose().chars().any(char::is_whitespace)) { bail!("registration key must be a single non-empty token"); } - crate::upgrade::validate_discovery_url(&self.discovery_url)?; - Ok(()) + if !self.allow_insecure_gateway || self.auto_upgrade.is_some() { + crate::upgrade::validate_discovery_url(&self.discovery_url).with_context(|| { + if self.allow_insecure_gateway { + "automatic upgrade downloads over https: set LIGHTSPEED_ENVD_DISCOVERY_URL" + } else { + "discovery URL" + } + })?; + } + Ok(urls) } } -/// Plain `ws://` is accepted only toward loopback; everything else must be -/// `wss://` so the registration key never crosses a network in the clear. -pub fn validate_gateway_url(url: &str) -> Result<()> { - let rest = if let Some(rest) = url.strip_prefix("wss://") { - rest - } else if let Some(rest) = url.strip_prefix("ws://") { - let host = rest - .split(['/', '?', '#']) - .next() - .unwrap_or_default() - .rsplit_once(':') - .map(|(host, _)| host) - .unwrap_or(rest.split(['/', '?', '#']).next().unwrap_or_default()); - let host = host.trim_start_matches('[').trim_end_matches(']'); - let loopback = host == "localhost" - || host - .parse::() - .is_ok_and(|ip| ip.is_loopback()); - if !loopback { - bail!("gateway URL must use wss:// except toward loopback: {url}"); +/// Validate a gateway connect URL and derive what it determines. Plain +/// `ws://` is accepted toward loopback, and toward any other host only when +/// `allow_insecure` opts in; otherwise the URL must be `wss://`, so neither +/// the registration key nor any stream crosses a network in the clear. The +/// path must end with `CONNECT_PATH` (a prefix before it is kept), with no +/// query or fragment. +pub fn gateway_urls(url: &str, allow_insecure: bool) -> Result { + let scheme = url.split_once("://").map(|(scheme, _)| scheme); + match scheme { + Some("wss") => {} + Some("ws") if allow_insecure => {} + Some("ws") => { + let origin = canonical_origin(url)?; + let host = origin + .as_str() + .trim_start_matches("ws://") + .rsplit_once(':') + .filter(|(_, port)| port.bytes().all(|byte| byte.is_ascii_digit())) + .map_or(origin.as_str().trim_start_matches("ws://"), |(host, _)| { + host + }) + .trim_start_matches('[') + .trim_end_matches(']') + .to_owned(); + let loopback = host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()); + if !loopback { + bail!( + "gateway URL must use wss:// except toward loopback \ + (or set LIGHTSPEED_ENVD_ALLOW_INSECURE_GATEWAY): {url}" + ); + } } - rest - } else { - bail!("gateway URL must start with wss:// (or ws:// toward loopback): {url}"); - }; - if rest.is_empty() || rest.starts_with('/') { - bail!("gateway URL has no host: {url}"); + _ => bail!("gateway URL must start with wss:// (or ws:// toward loopback): {url}"), } - Ok(()) + let origin = canonical_origin(url).with_context(|| format!("gateway URL {url}"))?; + let stream = gateway_stream_url(url).with_context(|| format!("gateway URL {url}"))?; + Ok(GatewayUrls { + connect: url.to_owned(), + origin, + stream, + }) } /// Names of every daemon configuration variable present in the process @@ -293,9 +402,13 @@ impl DaemonArgs { install_path: started_executable().context("find the started envd executable")?, target: release_info::TARGET.to_owned(), expected_protocol: None, + other_than: None, }) } + /// Resolve the configuration. Must run before the async runtime starts + /// any thread: it removes secret-bearing variables from the process + /// environment. pub fn into_config(self) -> Result { let cwd = canonical_dir( self.cwd @@ -313,71 +426,22 @@ impl DaemonArgs { fs_root.display() ); } - let state_dir = match self.state_dir { - Some(path) if path.is_absolute() => path, - Some(path) => cwd.join(path), - None => cwd.join(".lightspeed-envd"), - }; + let state_dir = resolve_state_dir(self.state_dir, &cwd, |name| std::env::var_os(name))?; let scrubbed_env = scrubbed_env_names(); let registration = match self.gateway_url { - Some(gateway_url) => { - let discovery_url = crate::upgrade::resolve_discovery_url( - Some(&gateway_url), - self.discovery_url.as_deref(), - )?; - let auto_upgrade = self - .auto_upgrade - .then(|| -> Result { - Ok(RestartInvocation { - executable: started_executable() - .context("find the started envd executable")?, - arg0: std::env::args_os() - .next() - .ok_or_else(|| anyhow::anyhow!("process has no argv[0]"))?, - args: std::env::args_os().skip(1).collect(), - }) - }) - .transpose()?; - let reexec_registration_key = std::env::var(REEXEC_REGISTRATION_KEY_ENV).ok(); - let registration_key = if let Some(key) = reexec_registration_key { - Some(SecretString::new(key)) - } else { - match (self.registration_key, self.registration_key_file) { - (Some(_), Some(_)) => { - bail!("set either the registration key or the key file, not both") - } - (Some(key), None) => Some(SecretString::new(key.trim().to_owned())), - (None, Some(path)) => Some(SecretString::new( - std::fs::read_to_string(&path) - .with_context(|| { - format!("read registration key file {}", path.display()) - })? - .trim() - .to_owned(), - )), - (None, None) => None, - } - }; - let metadata = match self.registration_metadata.as_deref() { - Some(json) if !json.trim().is_empty() => { - serde_json::from_str::>(json) - .context("registration metadata must be a JSON object of strings")? - } - _ => BTreeMap::new(), - }; - let registration = RegistrationConfig { - gateway_url, - registration_key, - display_name: self.registration_name.filter(|name| !name.is_empty()), - metadata, - receipt_path: self.registration_receipt, - ca_file: self.ca_file, - discovery_url, - auto_upgrade, - }; - registration.validate()?; - Some(registration) - } + Some(gateway_url) => Some(registration_config( + gateway_url, + self.discovery_url, + self.allow_insecure_gateway, + self.on_closed, + self.auto_upgrade, + self.registration_key, + self.registration_key_file, + self.registration_name, + self.registration_metadata, + self.registration_receipt, + self.ca_file, + )?), None => { if self.registration_key.is_some() || self.registration_key_file.is_some() { bail!("a registration key needs LIGHTSPEED_ENVD_GATEWAY_URL to dial"); @@ -385,6 +449,12 @@ impl DaemonArgs { if self.auto_upgrade { bail!("automatic upgrade needs LIGHTSPEED_ENVD_GATEWAY_URL to dial"); } + if self.allow_insecure_gateway { + bail!("an insecure gateway needs LIGHTSPEED_ENVD_GATEWAY_URL to dial"); + } + if self.on_closed == OnClosed::Park { + bail!("parking on close needs LIGHTSPEED_ENVD_GATEWAY_URL to register"); + } None } }; @@ -406,6 +476,151 @@ impl DaemonArgs { } } +#[allow(clippy::too_many_arguments)] +fn registration_config( + gateway_url: String, + discovery_url: Option, + allow_insecure_gateway: bool, + on_closed: OnClosed, + auto_upgrade: bool, + registration_key: Option, + registration_key_file: Option, + registration_name: Option, + registration_metadata: Option, + registration_receipt: Option, + ca_file: Option, +) -> Result { + // An explicit discovery URL is checked here; a derived one by + // `validate`, which knows whether this configuration can download. + let discovery_url = match discovery_url { + Some(url) => { + crate::upgrade::validate_discovery_url(&url)?; + url + } + None => crate::upgrade::derive_discovery_url(&gateway_url)?, + }; + let auto_upgrade = auto_upgrade + .then(|| -> Result { + Ok(RestartInvocation { + executable: started_executable().context("find the started envd executable")?, + arg0: std::env::args_os() + .next() + .ok_or_else(|| anyhow::anyhow!("process has no argv[0]"))?, + args: std::env::args_os().skip(1).collect(), + }) + }) + .transpose()?; + let upgrade_attempted_for = match std::env::var(AUTO_UPGRADE_ATTEMPTED_ENV) { + Ok(value) => Some(value.trim().parse::().with_context(|| { + format!("{AUTO_UPGRADE_ATTEMPTED_ENV} must be a protocol version: {value}") + })?), + Err(_) => None, + }; + let reexec_registration_key = std::env::var(REEXEC_REGISTRATION_KEY_ENV).ok(); + let registration_key = if let Some(key) = reexec_registration_key { + Some(SecretString::new(key)) + } else { + match (registration_key, registration_key_file) { + (Some(_), Some(_)) => { + bail!("set either the registration key or the key file, not both") + } + (Some(key), None) => Some(SecretString::new(key.trim().to_owned())), + (None, Some(path)) => Some(SecretString::new( + std::fs::read_to_string(&path) + .with_context(|| format!("read registration key file {}", path.display()))? + .trim() + .to_owned(), + )), + (None, None) => None, + } + }; + let metadata = match registration_metadata.as_deref() { + Some(json) if !json.trim().is_empty() => Metadata::new( + serde_json::from_str::>(json) + .context("registration metadata must be a JSON object of strings")?, + ) + .context("registration metadata")?, + _ => Metadata::default(), + }; + let display_name = registration_name + .filter(|name| !name.is_empty()) + .map(DisplayName::new) + .transpose() + .context("registration name")?; + let registration = RegistrationConfig { + gateway_url, + registration_key, + display_name, + metadata, + receipt_path: registration_receipt, + ca_file, + discovery_url, + allow_insecure_gateway, + on_closed, + auto_upgrade, + upgrade_attempted_for, + }; + registration.validate()?; + Ok(registration) +} + +/// The state directory, created `0700` and canonical. A defaulted +/// directory must not lie inside `cwd`, the agent's working tree; an +/// explicitly configured one may (confinement refuses data-plane access to +/// it wherever it lives). A relative explicit path is taken against `cwd`. +fn resolve_state_dir( + configured: Option, + cwd: &Path, + env: impl Fn(&str) -> Option, +) -> Result { + let (path, defaulted) = match configured { + Some(path) if path.is_absolute() => (path, false), + Some(path) => (cwd.join(path), false), + None => (default_state_dir(env)?, true), + }; + let state_dir = prepare_state_dir(&path)?; + if defaulted && state_dir.starts_with(cwd) { + bail!( + "the default state directory {} lies inside cwd {}: set LIGHTSPEED_ENVD_STATE_DIR", + state_dir.display(), + cwd.display() + ); + } + Ok(state_dir) +} + +/// `$XDG_STATE_HOME/lightspeed-envd` when that variable is an absolute +/// path, else `$HOME/.local/state/lightspeed-envd`. +fn default_state_dir(env: impl Fn(&str) -> Option) -> Result { + if let Some(state_home) = env("XDG_STATE_HOME").map(PathBuf::from) + && state_home.is_absolute() + { + return Ok(state_home.join(STATE_DIR_NAME)); + } + match env("HOME").map(PathBuf::from) { + Some(home) if home.is_absolute() => { + Ok(home.join(".local").join("state").join(STATE_DIR_NAME)) + } + _ => bail!( + "neither XDG_STATE_HOME nor HOME names an absolute directory: set LIGHTSPEED_ENVD_STATE_DIR" + ), + } +} + +/// Create the state directory when missing (`0700`; parents with default +/// permissions), restrict it, and return its canonical path. +pub fn prepare_state_dir(path: &Path) -> Result { + std::fs::create_dir_all(path) + .with_context(|| format!("create state dir {}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) + .with_context(|| format!("restrict state dir {}", path.display()))?; + } + canonical_dir(path.to_path_buf(), "state dir") +} + /// Resolve argv[0] once, before serving anything, so replacement and re-exec /// use the path the operator or service manager launched rather than a later /// PATH lookup. Preserve a symlink at that path: the atomic replacement is of @@ -434,17 +649,19 @@ fn started_executable() -> Result { std::env::current_exe().context("fall back to the running executable path") } -/// Drop the secret-bearing variables from this process's environment as soon -/// as they have been read. Children additionally get every daemon variable -/// removed at spawn; this narrows what an in-process bug could leak. +/// Drop the secret-bearing and one-exec variables from this process's +/// environment as soon as they have been read. Children additionally get +/// every daemon variable removed at spawn; this narrows what an in-process +/// bug could leak and what a later re-exec could inherit. fn forget_secret_env() { for name in [ "LIGHTSPEED_ENVD_REGISTRATION_KEY", "LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE", REEXEC_REGISTRATION_KEY_ENV, + AUTO_UPGRADE_ATTEMPTED_ENV, ] { - // SAFETY: called from configuration parsing on the main thread before - // the daemon spawns any thread that reads the environment. + // SAFETY: called from configuration parsing on the main thread + // before the async runtime or any other thread exists. unsafe { std::env::remove_var(name) }; } } @@ -476,6 +693,8 @@ mod tests { listen: None, gateway_url: None, discovery_url: None, + allow_insecure_gateway: false, + on_closed: OnClosed::Exit, auto_upgrade: false, registration_key: None, registration_key_file: None, @@ -483,7 +702,7 @@ mod tests { registration_metadata: None, registration_receipt: None, ca_file: None, - cwd: Some(temp.path().to_path_buf()), + cwd: Some(temp.path().join("work")), fs_root: Some(temp.path().to_path_buf()), state_dir: Some(temp.path().join("state")), read_only_fs: false, @@ -491,9 +710,15 @@ mod tests { } } + fn temp() -> tempfile::TempDir { + let temp = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir(temp.path().join("work")).expect("work"); + temp + } + #[test] fn listener_config_needs_no_identity_or_secret() { - let temp = tempfile::tempdir().expect("tempdir"); + let temp = temp(); let config = DaemonArgs { listen: Some("127.0.0.1:0".parse().unwrap()), ..args(&temp) @@ -531,7 +756,7 @@ mod tests { #[test] fn outbound_config_reads_the_key_once_and_disables_the_listener() { - let temp = tempfile::tempdir().expect("tempdir"); + let temp = temp(); let key_file = temp.path().join("key"); std::fs::write(&key_file, "lsrk_secret\n").expect("write key"); let config = DaemonArgs { @@ -552,14 +777,11 @@ mod tests { .map(|key| key.expose()), Some("lsrk_secret") ); - assert_eq!(registration.display_name.as_deref(), Some("worker-1")); assert_eq!( - registration - .metadata - .get("harbor.trialId") - .map(String::as_str), - Some("t-1") + registration.display_name.as_ref().map(DisplayName::as_str), + Some("worker-1") ); + assert_eq!(registration.metadata.get("harbor.trialId"), Some("t-1")); assert!(!format!("{registration:?}").contains("lsrk_secret")); let both = DaemonArgs { @@ -588,14 +810,199 @@ mod tests { } #[test] - fn plain_websocket_gateway_urls_are_loopback_only() { - assert!(validate_gateway_url("wss://gateway.example/environment-gateway/connect").is_ok()); - assert!(validate_gateway_url("ws://127.0.0.1:18080/environment-gateway/connect").is_ok()); - assert!(validate_gateway_url("ws://localhost:18080/x").is_ok()); - assert!(validate_gateway_url("ws://[::1]:18080/x").is_ok()); - assert!(validate_gateway_url("ws://gateway.example/x").is_err()); - assert!(validate_gateway_url("http://gateway.example/x").is_err()); - assert!(validate_gateway_url("wss:///x").is_err()); + fn gateway_urls_are_loopback_only_in_the_clear_and_end_with_the_connect_path() { + let urls = gateway_urls( + "wss://Gateway.example/ls/environment-gateway/connect", + false, + ) + .unwrap(); + assert_eq!(urls.origin.as_str(), "wss://gateway.example"); + assert_eq!( + urls.stream, + "wss://gateway.example/ls/environment-gateway/data" + ); + for good in [ + "ws://127.0.0.1:18080/environment-gateway/connect", + "ws://localhost:18080/environment-gateway/connect", + "ws://[::1]:18080/environment-gateway/connect", + ] { + assert!(gateway_urls(good, false).is_ok(), "{good}"); + } + for bad in [ + "ws://gateway.example/environment-gateway/connect", + "http://gateway.example/environment-gateway/connect", + "wss:///environment-gateway/connect", + "wss://gateway.example/x", + "wss://gateway.example/environment-gateway/connect?x=1", + "wss://user@gateway.example/environment-gateway/connect", + ] { + assert!(gateway_urls(bad, false).is_err(), "{bad}"); + if !bad.starts_with("ws://") { + assert!(gateway_urls(bad, true).is_err(), "{bad} stays refused"); + } + } + } + + #[test] + fn an_opt_in_accepts_a_cluster_internal_gateway_in_the_clear() { + let url = "ws://lightspeed-gateway.lightspeed.svc:8080/environment-gateway/connect"; + let urls = gateway_urls(url, true).expect("opted in"); + assert_eq!( + urls.origin.as_str(), + "ws://lightspeed-gateway.lightspeed.svc:8080" + ); + assert_eq!( + urls.stream, + "ws://lightspeed-gateway.lightspeed.svc:8080/environment-gateway/data" + ); + + let temp = temp(); + let refused = DaemonArgs { + gateway_url: Some(url.to_owned()), + ..args(&temp) + } + .into_config(); + assert!(refused.is_err(), "refused without the opt-in"); + + let config = DaemonArgs { + gateway_url: Some(url.to_owned()), + allow_insecure_gateway: true, + ..args(&temp) + } + .into_config() + .expect("config"); + let registration = config.registration.expect("registration"); + assert!(registration.allow_insecure_gateway); + assert_eq!( + registration.discovery_url, + "http://lightspeed-gateway.lightspeed.svc:8080/.well-known/lightspeed-envd", + "a report-only discovery URL may be plain HTTP" + ); + + // Downloading a build still needs HTTPS discovery. + let downloading = DaemonArgs { + gateway_url: Some(url.to_owned()), + allow_insecure_gateway: true, + auto_upgrade: true, + ..args(&temp) + } + .into_config(); + assert!(downloading.is_err()); + let explicit = DaemonArgs { + gateway_url: Some(url.to_owned()), + allow_insecure_gateway: true, + auto_upgrade: true, + discovery_url: Some("https://downloads.example/envd.json".to_owned()), + ..args(&temp) + } + .into_config() + .expect("https discovery"); + assert!( + explicit + .registration + .expect("registration") + .auto_upgrade + .is_some() + ); + + let without_gateway = DaemonArgs { + allow_insecure_gateway: true, + ..args(&temp) + } + .into_config(); + assert!(without_gateway.is_err()); + } + + #[test] + fn on_closed_defaults_to_exit_and_parking_needs_a_gateway() { + let parsed = DaemonArgs::try_parse_from([ + "lightspeed-envd", + "--gateway-url", + "wss://gateway.example/environment-gateway/connect", + ]) + .expect("args"); + assert_eq!(parsed.on_closed, OnClosed::Exit); + assert!(!parsed.allow_insecure_gateway); + let parked = DaemonArgs::try_parse_from([ + "lightspeed-envd", + "--gateway-url", + "wss://gateway.example/environment-gateway/connect", + "--on-closed", + "park", + "--allow-insecure-gateway", + ]) + .expect("args"); + assert_eq!(parked.on_closed, OnClosed::Park); + assert!(parked.allow_insecure_gateway); + assert!(DaemonArgs::try_parse_from(["lightspeed-envd", "--on-closed", "linger"]).is_err()); + + let temp = temp(); + let config = DaemonArgs { + gateway_url: Some("wss://gateway.example/environment-gateway/connect".to_owned()), + on_closed: OnClosed::Park, + ..args(&temp) + } + .into_config() + .expect("config"); + assert_eq!( + config.registration.expect("registration").on_closed, + OnClosed::Park + ); + let without_gateway = DaemonArgs { + on_closed: OnClosed::Park, + ..args(&temp) + } + .into_config(); + assert!(without_gateway.is_err()); + } + + #[test] + fn the_default_state_dir_follows_xdg_and_stays_out_of_cwd() { + let env = |pairs: &'static [(&'static str, &'static str)]| { + move |name: &str| { + pairs + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| OsString::from(value)) + } + }; + assert_eq!( + default_state_dir(env(&[("XDG_STATE_HOME", "/xdg"), ("HOME", "/home/a")])).unwrap(), + PathBuf::from("/xdg/lightspeed-envd") + ); + assert_eq!( + default_state_dir(env(&[("XDG_STATE_HOME", "relative"), ("HOME", "/home/a")])).unwrap(), + PathBuf::from("/home/a/.local/state/lightspeed-envd") + ); + assert!(default_state_dir(env(&[])).is_err()); + + let temp = temp(); + let cwd = temp.path().join("work").canonicalize().unwrap(); + let home = cwd.clone(); + let inside = resolve_state_dir(None, &cwd, |name| { + (name == "HOME").then(|| home.clone().into_os_string()) + }); + assert!( + inside.is_err(), + "a defaulted state dir inside cwd is refused" + ); + + let explicit = resolve_state_dir(Some(PathBuf::from(".envd")), &cwd, |_| None) + .expect("an explicit state dir inside cwd is allowed"); + assert_eq!(explicit, cwd.join(".envd")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + let mode = std::fs::metadata(&explicit).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o700); + } + + let outside_home = temp.path().canonicalize().unwrap().join("home"); + let outside = resolve_state_dir(None, &cwd, |name| { + (name == "HOME").then(|| outside_home.clone().into_os_string()) + }) + .expect("a default outside cwd"); + assert!(outside.ends_with(".local/state/lightspeed-envd")); } #[test] @@ -622,11 +1029,9 @@ mod tests { #[test] fn outbound_upgrade_configuration_derives_discovery_and_captures_restart() { - let temp = tempfile::tempdir().expect("tempdir"); + let temp = temp(); let config = DaemonArgs { - gateway_url: Some( - "wss://gateway.example/environment-gateway/connect?ignored=1".to_owned(), - ), + gateway_url: Some("wss://gateway.example/ls/environment-gateway/connect".to_owned()), auto_upgrade: true, ..args(&temp) } diff --git a/crates/environment-daemon/src/dispatch.rs b/crates/environment-daemon/src/dispatch.rs new file mode 100644 index 000000000..a1b5d0ecc --- /dev/null +++ b/crates/environment-daemon/src/dispatch.rs @@ -0,0 +1,278 @@ +//! Data-plane request dispatch, driven by the protocol's method manifest. +//! +//! The connection loop ([`crate::server`]) owns the handshake and hands +//! every request after `initialize` to [`handle_request`]. Dispatch looks +//! the method up in [`manifest`], refuses it when its capability is absent, +//! touches the idle clock for `use` methods, decodes the params strictly +//! and calls the handler. Handlers receive a [`RequestContext`] whose +//! cancellation token fires on `$/cancelRequest` and when the connection +//! closes. + +use std::{future::Future, sync::Arc}; + +use environment_protocol::{ + error::{Capability, ProtocolError}, + handshake::Plane, + jsonrpc, + manifest::{ + self, Activity, DataInitialize, Direction, EnvGetIdle, FsCreateDirectory, FsGetMetadata, + FsGlobFiles, FsReadDirectory, FsReadFile, FsRemove, FsRename, FsScan, FsSearchText, + FsWriteFile, JobCancel, JobRead, JobStart, MessageKind, ProcessRead, ProcessStart, + ProcessTerminate, ProcessWrite, RequestMethod, TransferAbort, TransferAdvance, + TransferAppend, TransferBegin, TransferCommit, TransferGetStatus, TransferListMissing, + TransferRead, TransferReadInventory, TransferWrite, + }, +}; +use serde::Serialize; +use serde_json::Value; +use tokio_util::sync::CancellationToken; + +use crate::DaemonRuntime; + +/// One data connection. +pub struct ConnectionInfo { + /// Daemon-unique, increasing; for logs and per-connection ownership. + pub connection_id: u64, + /// Cancelled when the connection's reader loop ends (peer close, + /// transport error, daemon shutdown). + pub closed: CancellationToken, +} + +/// Passed to every handler. +#[derive(Clone)] +pub struct RequestContext { + pub connection: Arc, + /// Child of `connection.closed`; also cancelled by `$/cancelRequest` + /// naming this request. Abortable handlers select on + /// `cancel.cancelled()` and return `Err(ProtocolError::cancelled())` + /// (the reply is dropped anyway when the connection closed). + pub cancel: CancellationToken, +} + +impl RequestContext { + /// A context for one request on `connection`. + pub fn new(connection: Arc) -> Self { + let cancel = connection.closed.child_token(); + Self { connection, cancel } + } + + /// A context on a connection of its own, for in-process callers and + /// tests. + pub fn detached() -> Self { + Self::new(Arc::new(ConnectionInfo { + connection_id: 0, + closed: CancellationToken::new(), + })) + } +} + +/// Runs once, after the response frame was written to the socket. +pub type WrittenHook = Box; + +/// A successful answer plus an optional hook the writer task runs only +/// after the response frame was written to the socket. +pub struct Reply { + pub value: T, + pub on_written: Option, +} + +impl Reply { + pub fn new(value: T) -> Self { + Self { + value, + on_written: None, + } + } + + pub fn with_commit(value: T, commit: impl FnOnce() + Send + 'static) -> Self { + Self { + value, + on_written: Some(Box::new(commit)), + } + } + + pub fn map(self, map: impl FnOnce(T) -> U) -> Reply { + Reply { + value: map(self.value), + on_written: self.on_written, + } + } +} + +impl std::fmt::Debug for Reply +where + T: std::fmt::Debug, +{ + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("Reply") + .field("value", &self.value) + .field("on_written", &self.on_written.is_some()) + .finish() + } +} + +/// Dispatch one already-initialized data request: manifest lookup +/// (methodNotFound), capability check (capabilityUnavailable), activity +/// touch for `Activity::Use`, strict params decode with +/// `jsonrpc::decode_params` (invalidParams naming the field), handler call, +/// result encode. Public for in-process tests. +pub async fn handle_request( + runtime: &DaemonRuntime, + ctx: &RequestContext, + method: &str, + params: Value, +) -> Result, ProtocolError> { + let spec = manifest::find(Plane::Data, method) + .filter(|spec| { + spec.kind == MessageKind::Request && spec.direction == Direction::ClientToServer + }) + .ok_or_else(|| ProtocolError::method_not_found(method))?; + if let Some(capability) = spec.capability { + runtime.require_capability(capability)?; + } + if spec.activity == Some(Activity::Use) { + runtime.touch_activity(); + } + let params = ¶ms; + let filesystem = runtime.filesystem(); + let processes = runtime.processes(); + let jobs = runtime.jobs(); + match spec.name { + DataInitialize::NAME => Err(ProtocolError::invalid_request( + "initialize was already answered on this connection", + )), + EnvGetIdle::NAME => { + answer::(params, |_| async { Ok(runtime.idle_report().await) }).await + } + FsReadFile::NAME => { + answer::(params, |p| filesystem.read_file(ctx, p)).await + } + FsWriteFile::NAME => { + answer::(params, |p| filesystem.write_file(ctx, p)).await + } + FsCreateDirectory::NAME => { + answer::(params, |p| filesystem.create_directory(ctx, p)).await + } + FsGetMetadata::NAME => { + answer::(params, |p| filesystem.get_metadata(ctx, p)).await + } + FsReadDirectory::NAME => { + answer::(params, |p| filesystem.read_directory(ctx, p)).await + } + FsRemove::NAME => answer::(params, |p| filesystem.remove(ctx, p)).await, + FsRename::NAME => answer::(params, |p| filesystem.rename(ctx, p)).await, + FsSearchText::NAME => { + answer::(params, |p| filesystem.search_text(ctx, p)).await + } + FsGlobFiles::NAME => { + answer::(params, |p| filesystem.glob_files(ctx, p)).await + } + FsScan::NAME => answer::(params, |p| filesystem.scan(ctx, p)).await, + TransferBegin::NAME => { + answer::(params, |p| filesystem.transfer_begin(ctx, p)).await + } + TransferAdvance::NAME => { + answer::(params, |p| filesystem.transfer_advance(ctx, p)).await + } + TransferGetStatus::NAME => { + answer::(params, |p| filesystem.transfer_get_status(ctx, p)) + .await + } + TransferReadInventory::NAME => { + answer::(params, |p| { + filesystem.transfer_read_inventory(ctx, p) + }) + .await + } + TransferAppend::NAME => { + answer::(params, |p| filesystem.transfer_append(ctx, p)).await + } + TransferListMissing::NAME => { + answer::(params, |p| { + filesystem.transfer_list_missing(ctx, p) + }) + .await + } + TransferRead::NAME => { + answer::(params, |p| filesystem.transfer_read(ctx, p)).await + } + TransferWrite::NAME => { + answer::(params, |p| filesystem.transfer_write(ctx, p)).await + } + TransferCommit::NAME => { + answer::(params, |p| filesystem.transfer_commit(ctx, p)).await + } + TransferAbort::NAME => { + answer::(params, |p| filesystem.transfer_abort(ctx, p)).await + } + ProcessStart::NAME => { + answer::(params, |p| processes.start(ctx, p)).await + } + ProcessRead::NAME => reply::(params, |p| processes.read(ctx, p)).await, + ProcessWrite::NAME => { + answer::(params, |p| processes.write(ctx, p)).await + } + ProcessTerminate::NAME => { + answer::(params, |p| processes.terminate(ctx, p)).await + } + JobStart::NAME => answer::(params, |p| jobs.start(ctx, p)).await, + JobRead::NAME => answer::(params, |p| jobs.read(ctx, p)).await, + JobCancel::NAME => answer::(params, |p| jobs.cancel(ctx, p)).await, + _ => Err(ProtocolError::method_not_found(method)), + } +} + +/// Decode, call a handler that answers a plain response, encode. +async fn answer(params: &Value, handler: F) -> Result, ProtocolError> +where + M: RequestMethod, + F: FnOnce(M::Params) -> Fut, + Fut: Future>, +{ + let params = jsonrpc::decode_params::(params)?; + let response = handler(params).await?; + Ok(Reply::new(encode(&response)?)) +} + +/// Decode, call a handler that answers a [`Reply`], encode. +async fn reply(params: &Value, handler: F) -> Result, ProtocolError> +where + M: RequestMethod, + F: FnOnce(M::Params) -> Fut, + Fut: Future, ProtocolError>>, +{ + let params = jsonrpc::decode_params::(params)?; + let reply = handler(params).await?; + let value = encode(&reply.value)?; + Ok(Reply { + value, + on_written: reply.on_written, + }) +} + +fn encode(value: &T) -> Result { + serde_json::to_value(value) + .map_err(|error| ProtocolError::internal(format!("encode result: {error}"))) +} + +/// Whether the daemon's capabilities include `capability`. +pub(crate) fn has_capability( + capabilities: &environment_protocol::data::handshake::DataCapabilities, + capability: Capability, +) -> bool { + match capability { + Capability::Filesystem => capabilities.filesystem.is_some(), + Capability::FilesystemWrite => capabilities + .filesystem + .as_ref() + .is_some_and(|filesystem| filesystem.write), + Capability::Process => capabilities.process.is_some(), + Capability::ProcessPty => capabilities + .process + .as_ref() + .is_some_and(|process| process.pty), + Capability::Job => capabilities.job.is_some(), + Capability::Power | Capability::Ingress => false, + } +} diff --git a/crates/environment-daemon/src/exec.rs b/crates/environment-daemon/src/exec.rs new file mode 100644 index 000000000..4c024f29d --- /dev/null +++ b/crates/environment-daemon/src/exec.rs @@ -0,0 +1,323 @@ +//! The execution primitive processes and jobs share. +//! +//! One [`Execution`] is one root process spawned from a +//! [`CommandSpec`]: it runs as the leader of its own process group, its +//! stdout, stderr or PTY output is redacted and captured into a byte-offset +//! [`OutputSink`], its input is fed by a writer of its own, and its exit is +//! observed independently of its pipes. `process/*` and `job/*` differ only +//! in what they do around it: processes keep a daemon cursor and leave +//! leftovers running; jobs persist their output and sweep their group. +//! +//! Start order: validate, spawn, attach the output readers, the exit +//! watcher and the deadline, and only then start the input writer, so no +//! amount of input can block the start or keep a timeout from firing. + +mod execution; +mod input; +pub(crate) mod job_store; +pub(crate) mod log; +pub(crate) mod outcome; +pub(crate) mod spawn_record; + +use std::{collections::BTreeMap, path::PathBuf, process::Stdio}; + +use environment_protocol::{ + bytes::SecretString, + data::exec::{CommandSpec, ExecInput, PtySize}, + error::ProtocolError, +}; +use portable_pty::{CommandBuilder, native_pty_system}; +use tokio::process::Command; + +use crate::{process_group, redaction::redactions_for_secret_env}; + +pub(crate) use execution::{AfterExit, ExecOptions, Execution, OutputSink}; +use input::{Input, InputSink}; + +/// Checks a command before anything runs. `field` is the wire path of the +/// command, such as `command` or `jobs[2].command`. +pub(crate) fn validate_command(spec: &CommandSpec, field: &str) -> Result<(), ProtocolError> { + if spec.argv.is_empty() { + return Err(ProtocolError::invalid_field( + format!("{field}.argv"), + "argv must name a program", + )); + } + if spec.argv[0].is_empty() { + return Err(ProtocolError::invalid_field( + format!("{field}.argv"), + "argv[0] must not be empty", + )); + } + if spec.argv.iter().any(|arg| arg.contains('\0')) { + return Err(ProtocolError::invalid_field( + format!("{field}.argv"), + "arguments must not contain NUL", + )); + } + for (name, value) in &spec.env { + validate_env_name(name, &format!("{field}.env"))?; + if value.contains('\0') { + return Err(ProtocolError::invalid_field( + format!("{field}.env"), + format!("value of {name} contains NUL"), + )); + } + } + for (name, value) in &spec.secret_env { + validate_env_name(name, &format!("{field}.secretEnv"))?; + if value.expose().contains('\0') { + return Err(ProtocolError::invalid_field( + format!("{field}.secretEnv"), + format!("value of {name} contains NUL"), + )); + } + if spec.env.contains_key(name) { + return Err(ProtocolError::invalid_field( + format!("{field}.secretEnv"), + format!("{name} is also set in env"), + )); + } + } + if spec.timeout_ms == Some(0) { + return Err(ProtocolError::invalid_field( + format!("{field}.timeoutMs"), + "timeoutMs must be positive; omit it for no deadline", + )); + } + Ok(()) +} + +fn validate_env_name(name: &str, field: &str) -> Result<(), ProtocolError> { + if name.is_empty() || name.contains('=') || name.contains('\0') { + return Err(ProtocolError::invalid_field( + field, + format!("invalid variable name {name:?}"), + )); + } + Ok(()) +} + +/// A validated command with its working directory resolved, ready to +/// spawn. +pub(crate) struct PreparedCommand { + pub argv: Vec, + pub cwd: PathBuf, + pub env: BTreeMap, + pub secret_env: BTreeMap, + /// Daemon variables no child inherits. + pub scrubbed_env: Vec, + pub input: ExecInput, + pub pty: Option, +} + +impl PreparedCommand { + pub fn new( + spec: CommandSpec, + cwd: PathBuf, + scrubbed_env: &[String], + pty: Option, + ) -> Self { + Self { + argv: spec.argv, + cwd, + env: spec.env, + secret_env: spec.secret_env, + scrubbed_env: scrubbed_env.to_vec(), + input: spec.input, + pty, + } + } + + /// Spawns the root and starts capture, exit watching and the deadline. + /// The input writer starts with [`Execution::start_input`]. + pub fn spawn( + self, + sink: S, + options: ExecOptions, + input_queue_bytes: u64, + ) -> Result, ProtocolError> { + let redactions = redactions_for_secret_env(&self.secret_env); + if let Some(size) = self.pty { + self.spawn_pty(size, sink, options, redactions, input_queue_bytes) + } else { + self.spawn_pipes(sink, options, redactions, input_queue_bytes) + } + } + + fn spawn_pipes( + self, + sink: S, + options: ExecOptions, + redactions: Vec>, + input_queue_bytes: u64, + ) -> Result, ProtocolError> { + let mut command = Command::new(&self.argv[0]); + command + .args(&self.argv[1..]) + .current_dir(&self.cwd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .stdin(match self.input { + ExecInput::None {} => Stdio::null(), + ExecInput::Bytes { .. } | ExecInput::Pipe {} => Stdio::piped(), + }); + for name in &self.scrubbed_env { + command.env_remove(name); + } + command.envs(&self.env); + for (name, value) in &self.secret_env { + command.env(name, value.expose()); + } + process_group::spawn_in_own_group(&mut command); + let mut child = command.spawn().map_err(|error| { + ProtocolError::process_failed(format!("spawn {:?}: {error}", self.argv[0])) + })?; + let pid = child + .id() + .ok_or_else(|| ProtocolError::process_failed("spawned process has no pid"))?; + let input = match (self.input, child.stdin.take()) { + (ExecInput::Bytes { data }, Some(stdin)) => Input::new( + InputSink::Pipe(stdin), + Some(data.into_inner()), + true, + false, + 0, + ), + (ExecInput::Pipe {}, Some(stdin)) => { + Input::new(InputSink::Pipe(stdin), None, false, true, input_queue_bytes) + } + _ => Input::none(), + }; + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + Ok(Execution::start_pipes( + pid, child, stdout, stderr, input, sink, redactions, options, + )) + } + + fn spawn_pty( + self, + size: PtySize, + sink: S, + options: ExecOptions, + redactions: Vec>, + input_queue_bytes: u64, + ) -> Result, ProtocolError> { + let pair = native_pty_system() + .openpty(portable_pty::PtySize { + rows: u16::try_from(size.rows).unwrap_or(u16::MAX), + cols: u16::try_from(size.cols).unwrap_or(u16::MAX), + pixel_width: 0, + pixel_height: 0, + }) + .map_err(|error| ProtocolError::process_failed(format!("open PTY: {error}")))?; + let mut command = CommandBuilder::new(&self.argv[0]); + command.args(&self.argv[1..]); + command.cwd(&self.cwd); + for name in &self.scrubbed_env { + command.env_remove(name); + } + for (name, value) in &self.env { + command.env(name, value); + } + for (name, value) in &self.secret_env { + command.env(name, value.expose()); + } + // Every fallible step on the master comes before the spawn, so a + // started child is never left unsupervised by a failed start. + let master = pair.master; + let reader = master + .try_clone_reader() + .map_err(|error| ProtocolError::process_failed(format!("PTY reader: {error}")))?; + let writer = master + .take_writer() + .map_err(|error| ProtocolError::process_failed(format!("PTY writer: {error}")))?; + let mut child = pair.slave.spawn_command(command).map_err(|error| { + ProtocolError::process_failed(format!("spawn {:?} on a PTY: {error}", self.argv[0])) + })?; + // The child holds the slave now; the reader sees end of file only + // once every holder closed it. + drop(pair.slave); + // The PTY child is a session leader, so its pid is its group id. + let Some(pid) = child.process_id() else { + let _ = child.kill(); + let _ = child.wait(); + return Err(ProtocolError::process_failed("spawned process has no pid")); + }; + let initial = match self.input { + ExecInput::Bytes { data } => Some(data.into_inner()), + ExecInput::None {} | ExecInput::Pipe {} => None, + }; + let input = Input::new( + InputSink::Pty(writer), + initial, + false, + true, + input_queue_bytes, + ); + Ok(Execution::start_pty( + pid, child, master, reader, input, sink, redactions, options, + )) + } +} + +#[cfg(test)] +mod tests { + use environment_protocol::{ + bytes::ByteChunk, + error::{ErrorData, ProtocolErrorKind}, + }; + + use super::*; + + fn spec(argv: &[&str]) -> CommandSpec { + CommandSpec { + argv: argv.iter().map(|arg| (*arg).to_owned()).collect(), + cwd: None, + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input: ExecInput::Bytes { + data: ByteChunk::from(b"x".as_slice()), + }, + timeout_ms: None, + } + } + + fn field(error: &ProtocolError) -> Option<&str> { + match &error.data { + ErrorData::InvalidParams { field, .. } => field.as_deref(), + _ => None, + } + } + + #[test] + fn commands_are_validated_with_wire_field_paths() { + assert!(validate_command(&spec(&["true"]), "command").is_ok()); + let error = validate_command(&spec(&[]), "jobs[1].command").unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + assert_eq!(field(&error), Some("jobs[1].command.argv")); + + let mut collides = spec(&["true"]); + collides.env.insert("TOKEN".to_owned(), "plain".to_owned()); + collides + .secret_env + .insert("TOKEN".to_owned(), SecretString::new("secret")); + let error = validate_command(&collides, "command").unwrap_err(); + assert_eq!(field(&error), Some("command.secretEnv")); + + let mut bad_name = spec(&["true"]); + bad_name.env.insert("A=B".to_owned(), "x".to_owned()); + assert_eq!( + field(&validate_command(&bad_name, "command").unwrap_err()), + Some("command.env") + ); + + let mut zero = spec(&["true"]); + zero.timeout_ms = Some(0); + assert_eq!( + field(&validate_command(&zero, "command").unwrap_err()), + Some("command.timeoutMs") + ); + } +} diff --git a/crates/environment-daemon/src/exec/execution.rs b/crates/environment-daemon/src/exec/execution.rs new file mode 100644 index 000000000..164790833 --- /dev/null +++ b/crates/environment-daemon/src/exec/execution.rs @@ -0,0 +1,551 @@ +//! One running root process: output capture, exit, deadline and signals. + +use std::{ + io::Read as _, + sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; + +use environment_protocol::{ + data::exec::{ExitStatus, LeftoverDisposition, LeftoverReport, OutputStream, StoppedBy}, + error::ProtocolError, +}; +use portable_pty::MasterPty; +use tokio::{ + io::{AsyncRead, AsyncReadExt as _}, + process::{Child, ChildStderr, ChildStdout}, + sync::watch, + task::JoinHandle, +}; + +use super::{input::Input, outcome}; +use crate::{ + process_group::{self, Group, GroupSignal, OUTPUT_DRAIN_GRACE}, + redaction::redact_bytes, +}; + +const READ_BUFFER_BYTES: usize = 8192; + +/// Where captured, redacted output goes. +pub(crate) trait OutputSink: Send + 'static { + fn append(&mut self, stream: OutputStream, data: &[u8]); +} + +/// What happens to the output readers once the root exits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AfterExit { + /// Processes: readers drain for a grace period, then stay on the pipes + /// discarding, so leftovers holding them never see `EPIPE`. + Keep, + /// Jobs: the owner sweeps the group first, then calls + /// [`Execution::settle`]. + Manual, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ExecOptions { + /// `timeoutMs`: stop with `SIGTERM`, the grace, then `SIGKILL`. + pub timeout: Option, + /// Grace between `SIGTERM` and `SIGKILL` for terminate and timeout. + pub terminate_grace: Duration, + pub after_exit: AfterExit, +} + +/// A spawned root process. Cheap to clone. +pub(crate) struct Execution { + inner: Arc>, +} + +impl Clone for Execution { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +struct Inner { + pid: u32, + /// The root's process group, which it leads. + group: Group, + /// Latched once the group was seen empty (or its id reused) after the + /// root exited: nothing of it is signalled or sampled again. + group_gone: AtomicBool, + state: Mutex>, + /// Bumped on every change: output, exit, settlement, stop. + version: watch::Sender, + /// Pipe reader tasks, aborted by a settle that abandons them. + readers: Mutex>>, + /// The PTY master, for the terminal's foreground group. + pty: Option>>, + input: Input, + terminate_grace: Duration, +} + +/// What an execution has observed so far. +pub(crate) struct State { + pub sink: S, + redactions: Vec>, + readers_open: u32, + discarding: bool, + exited: bool, + exit: Option, + error: Option, + stopped_by: Option, + settled_at: Option, +} + +impl State { + /// The root process exited. + pub fn exited(&self) -> bool { + self.exited + } + + /// The root exited and its output settled: every reader reached end of + /// file or was discarded, so the output's end is final. + pub fn settled(&self) -> bool { + self.settled_at.is_some() + } + + pub fn settled_at(&self) -> Option { + self.settled_at + } + + pub fn exit(&self) -> Option<&ExitStatus> { + self.exit.as_ref() + } + + /// Why output capture or the wait failed, if either did. + pub fn error(&self) -> Option<&ProtocolError> { + self.error.as_ref() + } + + pub fn stopped_by(&self) -> Option { + self.stopped_by + } +} + +impl Execution { + #[allow(clippy::too_many_arguments)] + pub(super) fn start_pipes( + pid: u32, + child: Child, + stdout: Option, + stderr: Option, + input: Input, + sink: S, + redactions: Vec>, + options: ExecOptions, + ) -> Self { + let readers_open = u32::from(stdout.is_some()) + u32::from(stderr.is_some()); + let execution = Self::new(pid, None, input, sink, redactions, readers_open, options); + let inner = &execution.inner; + let mut readers = Vec::new(); + if let Some(stdout) = stdout { + readers.push(tokio::spawn(read_pipe( + inner.clone(), + stdout, + OutputStream::Stdout, + ))); + } + if let Some(stderr) = stderr { + readers.push(tokio::spawn(read_pipe( + inner.clone(), + stderr, + OutputStream::Stderr, + ))); + } + *lock(&inner.readers) = readers; + let watched = execution.clone(); + tokio::spawn(async move { + let status = { + let mut child = child; + child.wait().await + }; + watched.observe_exit(status, options.after_exit).await; + }); + execution.start_deadline(options.timeout); + execution + } + + #[allow(clippy::too_many_arguments)] + pub(super) fn start_pty( + pid: u32, + child: Box, + master: Box, + reader: Box, + input: Input, + sink: S, + redactions: Vec>, + options: ExecOptions, + ) -> Self { + let execution = Self::new(pid, Some(master), input, sink, redactions, 1, options); + // PTY reads and waits block. They run on threads of their own, not + // the runtime's blocking pool, so a PTY that leftovers keep open can + // never hold up the daemon's shutdown. + let inner = execution.inner.clone(); + if let Err(error) = std::thread::Builder::new() + .name("envd-pty-reader".to_owned()) + .spawn(move || read_pty(inner, reader)) + { + execution.inner.capture_failed(OutputStream::Pty, error); + execution.inner.reader_closed(); + } + // portable-pty's own exit status reports every signal death as + // code 1; the std child underneath keeps the signal. + let (exited, status) = tokio::sync::oneshot::channel(); + let waiter = match child.into_any().downcast::() { + Ok(mut child) => std::thread::Builder::new() + .name("envd-pty-wait".to_owned()) + .spawn(move || { + let _ = exited.send(child.wait()); + }) + .map(drop), + Err(_) => { + let _ = exited.send(Err(std::io::Error::other( + "the PTY child cannot be waited on", + ))); + Ok(()) + } + }; + let watched = execution.clone(); + tokio::spawn(async move { + let status = match waiter { + Ok(()) => status + .await + .unwrap_or_else(|_| Err(std::io::Error::other("the PTY wait thread ended"))), + Err(error) => Err(error), + }; + watched.observe_exit(status, options.after_exit).await; + }); + execution.start_deadline(options.timeout); + execution + } + + fn new( + pid: u32, + pty: Option>, + input: Input, + sink: S, + redactions: Vec>, + readers_open: u32, + options: ExecOptions, + ) -> Self { + Self { + inner: Arc::new(Inner { + pid, + group: Group::capture(pid), + group_gone: AtomicBool::new(false), + state: Mutex::new(State { + sink, + redactions, + readers_open, + discarding: false, + exited: false, + exit: None, + error: None, + stopped_by: None, + settled_at: None, + }), + version: watch::Sender::new(0), + readers: Mutex::new(Vec::new()), + pty: pty.map(Mutex::new), + input, + terminate_grace: options.terminate_grace, + }), + } + } + + async fn observe_exit( + &self, + status: std::io::Result, + after_exit: AfterExit, + ) { + { + let mut state = self.state(); + state.exited = true; + match status { + Ok(status) => state.exit = Some(outcome::exit_status(status)), + Err(error) => { + state + .error + .get_or_insert(ProtocolError::process_failed(format!( + "wait for process {}: {error}", + self.inner.pid + ))); + } + } + } + self.inner.input.shut(); + self.bump(); + if after_exit == AfterExit::Keep { + self.settle(false).await; + } + } + + fn start_deadline(&self, timeout: Option) { + let Some(timeout) = timeout else { + return; + }; + let execution = Arc::downgrade(&self.inner); + tokio::spawn(async move { + tokio::time::sleep(timeout).await; + if let Some(inner) = execution.upgrade() { + let execution = Execution { inner }; + if !execution.state().exited { + execution.stop(StoppedBy::Timeout); + } + } + }); + } +} + +impl Execution { + /// OS pid of the root, which is also its process group id. + pub fn pid(&self) -> u32 { + self.inner.pid + } + + /// The root's process group. + pub fn group(&self) -> Group { + self.inner.group + } + + /// The root runs, or its group still has members of its own. After the + /// root exited, an emptied group latches as gone, so a later group that + /// reuses the id is never mistaken for this one. + pub fn group_alive(&self) -> bool { + if !self.state().exited { + return true; + } + if self.inner.group_gone.load(Ordering::Acquire) { + return false; + } + let alive = self.inner.group.alive(); + if !alive { + self.inner.group_gone.store(true, Ordering::Release); + } + alive + } + + /// What is alive in the group now, as a report with `disposition`. + /// Absent when nothing is left or the platform cannot enumerate groups. + pub fn sample_leftovers(&self, disposition: LeftoverDisposition) -> Option { + if !self.group_alive() { + return None; + } + outcome::leftover_report(process_group::group_members(self.inner.pid), disposition) + } + + /// Runs `f` on the current state under the execution's lock. + pub fn with_state(&self, f: impl FnOnce(&mut State) -> R) -> R { + f(&mut self.state()) + } + + /// Notified on every change. + pub fn subscribe(&self) -> watch::Receiver { + self.inner.version.subscribe() + } + + /// Starts the input writer: the last step of a start. + pub fn start_input(&self) { + self.inner.input.start(); + } + + /// Queues input; see [`Input::offer`]. + pub fn offer_input(&self, data: &[u8], close: bool) -> Result { + self.inner.input.offer(data, close) + } + + /// Waits until `done` holds. + pub async fn wait_until(&self, done: impl Fn(&State) -> bool) { + let mut changes = self.subscribe(); + loop { + changes.borrow_and_update(); + if done(&self.state()) { + return; + } + if changes.changed().await.is_err() { + return; + } + } + } + + /// Settles the output after the root exited: waits up to the drain + /// grace for the readers to reach end of file, then discards what the + /// rest still read, and with `abort_readers` also abandons the pipes. + pub async fn settle(&self, abort_readers: bool) { + let mut changes = self.subscribe(); + let deadline = tokio::time::Instant::now() + OUTPUT_DRAIN_GRACE; + loop { + changes.borrow_and_update(); + if self.state().readers_open == 0 { + break; + } + tokio::select! { + changed = changes.changed() => if changed.is_err() { break }, + _ = tokio::time::sleep_until(deadline) => break, + } + } + if abort_readers { + for reader in std::mem::take(&mut *lock(&self.inner.readers)) { + reader.abort(); + } + } + { + let mut state = self.state(); + if state.readers_open > 0 { + state.discarding = true; + } + // Nothing is captured any more, so the secret values go. + state.redactions = Vec::new(); + state.settled_at.get_or_insert_with(Instant::now); + } + self.bump(); + } + + /// Stops the group: `kill` at once, `terminate` and `timeout` with + /// `SIGTERM`, the grace, then `SIGKILL`. Records `stoppedBy` only while + /// the root still runs; after its exit the signal still reaches its + /// leftovers. Returns whether the root was running. + pub fn stop(&self, reason: StoppedBy) -> bool { + let running = { + let mut state = self.state(); + let running = !state.exited; + if running && (state.stopped_by.is_none() || reason == StoppedBy::Kill) { + state.stopped_by = Some(reason); + } + running + }; + if running || self.group_alive() { + let group = self.inner.group; + match reason { + StoppedBy::Kill => { + group.signal(GroupSignal::Kill); + } + StoppedBy::Terminate | StoppedBy::Timeout => { + if group.signal(GroupSignal::Terminate) { + tokio::spawn(group.kill_after_grace(self.inner.terminate_grace)); + } + } + } + } + self.bump(); + running + } + + /// `SIGINT` to the foreground group: the terminal's under a PTY, + /// otherwise the process group. Returns whether the root was running. + pub fn interrupt(&self) -> bool { + let running = !self.state().exited; + if !self.group_alive() { + return false; + } + let foreground = self.inner.pty.as_ref().and_then(|master| { + lock(master) + .process_group_leader() + .and_then(|leader| u32::try_from(leader).ok()) + }); + match foreground { + Some(foreground) => process_group::interrupt_foreground(foreground), + None => self.inner.group.signal(GroupSignal::Interrupt), + }; + running + } + + fn state(&self) -> MutexGuard<'_, State> { + lock(&self.inner.state) + } + + fn bump(&self) { + self.inner + .version + .send_modify(|version| *version = version.wrapping_add(1)); + } +} + +impl Inner { + fn push(&self, stream: OutputStream, data: &[u8]) { + { + let mut state = lock(&self.state); + if state.discarding { + return; + } + if state.redactions.is_empty() { + state.sink.append(stream, data); + } else { + let redacted = redact_bytes(data, &state.redactions); + state.sink.append(stream, &redacted); + } + } + self.bump(); + } + + fn capture_failed(&self, stream: OutputStream, error: std::io::Error) { + let mut state = lock(&self.state); + if !state.exited { + state + .error + .get_or_insert(ProtocolError::process_failed(format!( + "capture {stream:?} output: {error}" + ))); + } + } + + fn reader_closed(&self) { + { + let mut state = lock(&self.state); + state.readers_open = state.readers_open.saturating_sub(1); + } + self.bump(); + } + + fn bump(&self) { + self.version + .send_modify(|version| *version = version.wrapping_add(1)); + } +} + +async fn read_pipe( + inner: Arc>, + mut reader: R, + stream: OutputStream, +) { + let mut buffer = vec![0; READ_BUFFER_BYTES]; + loop { + match reader.read(&mut buffer).await { + Ok(0) => break, + Ok(read) => inner.push(stream, &buffer[..read]), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} + Err(error) => { + inner.capture_failed(stream, error); + break; + } + } + } + inner.reader_closed(); +} + +fn read_pty(inner: Arc>, mut reader: Box) { + let mut buffer = vec![0; READ_BUFFER_BYTES]; + loop { + match reader.read(&mut buffer) { + Ok(0) => break, + Ok(read) => inner.push(OutputStream::Pty, &buffer[..read]), + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} + // Linux answers EIO once every slave descriptor is closed: the + // PTY's end of file. + Err(_) => break, + } + } + inner.reader_closed(); +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} diff --git a/crates/environment-daemon/src/exec/input.rs b/crates/environment-daemon/src/exec/input.rs new file mode 100644 index 000000000..6c9f44bb2 --- /dev/null +++ b/crates/environment-daemon/src/exec/input.rs @@ -0,0 +1,310 @@ +//! A command's input: a bounded queue drained by its own writer. +//! +//! The writer runs off the runtime thread: a Tokio task for a +//! non-blocking pipe, a thread of its own for a PTY master, whose writes +//! block when the child stops reading. Callers only enqueue, so a +//! child that never reads can fill the queue but never stall the daemon, +//! and signals still reach it while a write is pending. + +use std::{ + io::Write as _, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, +}; + +use environment_protocol::error::{PreconditionReason, ProtocolError}; +use tokio::{io::AsyncWriteExt as _, process::ChildStdin, sync::mpsc}; + +/// `VEOF` in the default terminal settings (Ctrl-D): end of input for a +/// canonical-mode reader at the start of a line. +const PTY_END_OF_INPUT: u8 = 0x04; + +/// Where queued input goes. +pub(crate) enum InputSink { + Pipe(ChildStdin), + Pty(Box), +} + +enum Message { + Data(Vec), + Close, +} + +#[derive(Default)] +struct Shared { + queued: AtomicU64, + /// The input takes no more bytes: closed by the caller, by the child + /// (broken pipe), or because the process exited. + closed: AtomicBool, +} + +/// The input of one execution. +pub(crate) struct Input { + /// `process/write` is served: a pipe or a PTY. + writable: bool, + capacity: u64, + shared: Arc, + sender: Mutex>>, + /// The sink and the queue's receiver until the writer starts. + pending: Mutex)>>, +} + +impl Input { + /// No input at all: `/dev/null`. + pub fn none() -> Self { + let shared = Arc::new(Shared::default()); + shared.closed.store(true, Ordering::Release); + Self { + writable: false, + capacity: 0, + shared, + sender: Mutex::new(None), + pending: Mutex::new(None), + } + } + + /// Input to `sink`. `initial` is queued first whatever the capacity; + /// `close_after_initial` then ends the input. `writable` inputs accept + /// `process/write` up to `capacity` queued bytes. + pub fn new( + sink: InputSink, + initial: Option>, + close_after_initial: bool, + writable: bool, + capacity: u64, + ) -> Self { + let (sender, receiver) = mpsc::unbounded_channel(); + let shared = Arc::new(Shared::default()); + if let Some(initial) = initial.filter(|initial| !initial.is_empty()) { + shared + .queued + .fetch_add(initial.len() as u64, Ordering::AcqRel); + let _ = sender.send(Message::Data(initial)); + } + let sender = if close_after_initial { + let _ = sender.send(Message::Close); + shared.closed.store(true, Ordering::Release); + None + } else { + Some(sender) + }; + Self { + writable, + capacity, + shared, + sender: Mutex::new(sender), + pending: Mutex::new(Some((sink, receiver))), + } + } + + /// Starts the writer. Called once the output readers, the exit watcher + /// and the deadline are in place, so input can never block them. + pub fn start(&self) { + let Some((sink, receiver)) = self.pending.lock().expect("input poisoned").take() else { + return; + }; + let shared = self.shared.clone(); + match sink { + InputSink::Pipe(stdin) => { + tokio::spawn(write_pipe(stdin, receiver, shared)); + } + InputSink::Pty(writer) => { + // A thread of its own, not the blocking pool: a write the + // child never drains must not hold up the daemon's shutdown. + let closed = shared.clone(); + if std::thread::Builder::new() + .name("envd-pty-writer".to_owned()) + .spawn(move || write_pty(writer, receiver, shared)) + .is_err() + { + closed.closed.store(true, Ordering::Release); + } + } + } + } + + /// Queues a prefix of `data` and answers how many bytes were accepted; + /// `close` ends the input once every byte was accepted. + pub fn offer(&self, data: &[u8], close: bool) -> Result { + let mut sender = self.sender.lock().expect("input poisoned"); + let open = sender + .as_ref() + .filter(|_| self.writable && !self.shared.closed.load(Ordering::Acquire)); + let Some(open) = open else { + return Err(ProtocolError::failed_precondition( + PreconditionReason::InputClosed, + if self.writable { + "the process input is closed" + } else { + "the process was started without writable input" + }, + )); + }; + let queued = self.shared.queued.load(Ordering::Acquire); + let free = self.capacity.saturating_sub(queued); + let accepted = (data.len() as u64).min(free); + if accepted > 0 { + self.shared.queued.fetch_add(accepted, Ordering::AcqRel); + if open + .send(Message::Data(data[..accepted as usize].to_vec())) + .is_err() + { + self.shared.queued.fetch_sub(accepted, Ordering::AcqRel); + self.shared.closed.store(true, Ordering::Release); + return Err(ProtocolError::failed_precondition( + PreconditionReason::InputClosed, + "the process input is closed", + )); + } + } + if close && accepted == data.len() as u64 { + let _ = open.send(Message::Close); + self.shared.closed.store(true, Ordering::Release); + *sender = None; + } + Ok(accepted) + } + + /// Ends the input without a close message, because the process exited. + /// A writer stuck on a full PTY stays on its blocking thread until the + /// write fails; nothing waits for it. + pub fn shut(&self) { + self.shared.closed.store(true, Ordering::Release); + self.sender.lock().expect("input poisoned").take(); + self.pending.lock().expect("input poisoned").take(); + } + + /// Bytes queued and not yet written. + #[cfg(test)] + pub fn queued_bytes(&self) -> u64 { + self.shared.queued.load(Ordering::Acquire) + } +} + +async fn write_pipe( + stdin: ChildStdin, + mut receiver: mpsc::UnboundedReceiver, + shared: Arc, +) { + let mut stdin = Some(stdin); + while let Some(message) = receiver.recv().await { + match message { + Message::Data(data) => { + if let Some(writer) = stdin.as_mut() + && writer.write_all(&data).await.is_err() + { + // Broken pipe: the child closed its input, which is its + // right, not a failure of the command. + stdin = None; + shared.closed.store(true, Ordering::Release); + } + shared.queued.fetch_sub(data.len() as u64, Ordering::AcqRel); + } + Message::Close => { + stdin = None; + shared.closed.store(true, Ordering::Release); + } + } + } +} + +fn write_pty( + writer: Box, + mut receiver: mpsc::UnboundedReceiver, + shared: Arc, +) { + let mut writer = Some(writer); + while let Some(message) = receiver.blocking_recv() { + let (data, close) = match message { + Message::Data(data) => (data, false), + Message::Close => (vec![PTY_END_OF_INPUT], true), + }; + if let Some(pty) = writer.as_mut() + && pty.write_all(&data).and_then(|()| pty.flush()).is_err() + { + writer = None; + shared.closed.store(true, Ordering::Release); + } + if close { + shared.closed.store(true, Ordering::Release); + } else { + shared.queued.fetch_sub(data.len() as u64, Ordering::AcqRel); + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use environment_protocol::error::{ErrorData, ProtocolErrorKind}; + + use super::*; + + fn is_input_closed(error: &ProtocolError) -> bool { + matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::InputClosed + } + ) + } + + /// A writer that never makes progress, like a PTY whose child stopped + /// reading. + struct Stuck(std::sync::mpsc::Receiver<()>); + + impl std::io::Write for Stuck { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + let _ = self.0.recv(); + Err(std::io::ErrorKind::BrokenPipe.into()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[tokio::test(flavor = "current_thread")] + async fn a_stuck_writer_fills_the_queue_without_blocking_the_caller() { + let (release, wait) = std::sync::mpsc::channel(); + let input = Input::new( + InputSink::Pty(Box::new(Stuck(wait))), + None, + false, + true, + 1024, + ); + input.start(); + let started = std::time::Instant::now(); + assert_eq!(input.offer(&[b'x'; 4096], false).unwrap(), 1024); + assert_eq!(input.offer(b"more", false).unwrap(), 0, "queue full"); + assert!(started.elapsed() < Duration::from_secs(1)); + // The runtime thread is free: a timer still fires. + tokio::time::sleep(Duration::from_millis(10)).await; + drop(release); + for _ in 0..200 { + if input.offer(b"x", false).is_err() { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + let error = input.offer(b"x", false).unwrap_err(); + assert!(is_input_closed(&error), "a broken pipe closes the input"); + } + + #[tokio::test(flavor = "current_thread")] + async fn close_applies_only_when_every_byte_was_accepted() { + let (_release, wait) = std::sync::mpsc::channel(); + let input = Input::new(InputSink::Pty(Box::new(Stuck(wait))), None, false, true, 4); + assert_eq!(input.offer(b"abcdef", true).unwrap(), 4); + assert_eq!(input.queued_bytes(), 4); + assert_eq!(input.offer(b"", false).unwrap(), 0, "still open"); + let none = Input::none(); + let error = none.offer(b"x", false).unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::FailedPrecondition); + assert!(is_input_closed(&error)); + } +} diff --git a/crates/environment-daemon/src/exec/job_store.rs b/crates/environment-daemon/src/exec/job_store.rs new file mode 100644 index 000000000..1e52aabb8 --- /dev/null +++ b/crates/environment-daemon/src/exec/job_store.rs @@ -0,0 +1,613 @@ +//! Job records and output on disk, written by one thread of their own. +//! +//! Each job has a directory `//` with `record.json`, its +//! summary (rewritten only on state transitions), and its output as +//! append-only framed logs: `output.head` holds the bytes below the head +//! cap, `output.tail` the bytes after it. The tail log is compacted to the +//! newest tail-cap bytes once it holds twice that, so a verbose job writes +//! O(output) bytes in total and never more than the cap plus slack stays on +//! disk. Writes are buffered and flushed on an interval and when the job +//! finishes; nothing here runs on the runtime thread. +//! +//! Output waits for the store thread in a per-job pending buffer, not in +//! the command queue: of the bytes past the head cap only the newest +//! tail-cap bytes are kept there, so a job that produces faster than the +//! disk takes it holds at most its retained output in memory, and the +//! bytes dropped are ones the tail compaction would drop anyway. + +use std::{ + collections::BTreeMap, + fs::{File, OpenOptions}, + io::{BufWriter, Read as _, Write as _}, + path::{Path, PathBuf}, + sync::{Arc, Mutex, MutexGuard, mpsc}, + time::{Duration, Instant}, +}; + +use environment_protocol::{data::exec::OutputStream, ids::JobId}; +use tokio::sync::oneshot; + +use super::log::OutputLog; + +/// How often buffered output reaches the disk while a job runs. +const FLUSH_INTERVAL: Duration = Duration::from_millis(250); + +const RECORD_FILE: &str = "record.json"; +const HEAD_FILE: &str = "output.head"; +const TAIL_FILE: &str = "output.tail"; + +/// Frame header: offset (u64), stream (u8), length (u32), little endian. +const FRAME_HEADER_BYTES: usize = 13; + +enum Command { + WriteRecord { + job_id: JobId, + record: Vec, + done: Option>, + }, + /// The job has pending output. + Drain { + job_id: JobId, + }, + /// Flushes and closes the job's output, then writes its final record. + Finish { + job_id: JobId, + record: Vec, + done: oneshot::Sender<()>, + }, + Remove { + job_id: JobId, + }, +} + +/// A frame of output: offset, stream, bytes. +type Frame = (u64, OutputStream, Vec); + +/// Output of one job not yet taken by the store thread. +#[derive(Default)] +struct Pending { + /// In offset order; a frame lies wholly below or wholly past the head + /// cap. + frames: Vec, + /// Bytes of `frames` past the head cap. + tail_bytes: u64, + /// A drain command for the job is queued. + queued: bool, +} + +type PendingOutput = Arc>>; + +/// A handle to the store thread. Cheap to clone. +#[derive(Clone)] +pub(crate) struct JobStore { + root: PathBuf, + head_cap: u64, + tail_cap: u64, + pending: PendingOutput, + sender: mpsc::Sender, +} + +impl JobStore { + /// Starts the store thread for job directories under `root`. Output at + /// offsets below `head_cap` goes to the head log, the rest to the tail + /// log, which keeps at least the newest `tail_cap` bytes. + pub fn start(root: PathBuf, head_cap: u64, tail_cap: u64) -> std::io::Result { + let (sender, receiver) = mpsc::channel(); + let pending = PendingOutput::default(); + let writer = Writer { + root: root.clone(), + head_cap, + tail_cap, + pending: pending.clone(), + open: BTreeMap::new(), + }; + std::thread::Builder::new() + .name("envd-job-store".to_owned()) + .spawn(move || writer.run(receiver))?; + Ok(Self { + root, + head_cap, + tail_cap, + pending, + sender, + }) + } + + pub fn root(&self) -> &Path { + &self.root + } + + /// Queues a record write; the receiver answers once it is on disk. + pub fn write_record(&self, job_id: &JobId, record: Vec) -> oneshot::Receiver<()> { + let (done, written) = oneshot::channel(); + self.send(Command::WriteRecord { + job_id: job_id.clone(), + record, + done: Some(done), + }); + written + } + + /// Queues a record write without waiting. + pub fn queue_record(&self, job_id: &JobId, record: Vec) { + self.send(Command::WriteRecord { + job_id: job_id.clone(), + record, + done: None, + }); + } + + /// Adds output at `offset` to the job's pending buffer, keeping its + /// head bytes and its newest tail-cap bytes past the head cap. + pub fn append(&self, job_id: &JobId, offset: u64, stream: OutputStream, data: &[u8]) { + let head_len = self.head_cap.saturating_sub(offset).min(data.len() as u64) as usize; + let (head, tail) = data.split_at(head_len); + let wake = { + let mut pending = lock(&self.pending); + let entry = pending.entry(job_id.clone()).or_default(); + entry.push(offset, stream, head, self.head_cap); + entry.push(offset + head_len as u64, stream, tail, self.head_cap); + entry.tail_bytes += tail.len() as u64; + entry.trim_tail(self.head_cap, self.tail_cap); + !std::mem::replace(&mut entry.queued, true) + }; + if wake { + self.send(Command::Drain { + job_id: job_id.clone(), + }); + } + } + + /// Flushes the job's output and writes its final record; answers once + /// both are on disk. + pub async fn finish(&self, job_id: &JobId, record: Vec) { + let (done, written) = oneshot::channel(); + self.send(Command::Finish { + job_id: job_id.clone(), + record, + done, + }); + let _ = written.await; + } + + pub fn remove(&self, job_id: &JobId) { + self.send(Command::Remove { + job_id: job_id.clone(), + }); + } + + fn send(&self, command: Command) { + if self.sender.send(command).is_err() { + eprintln!("lightspeed-envd: the job store thread has stopped"); + } + } +} + +/// Reads a job's retained output back. `end` is the end recorded with the +/// job's final status; a larger end found in the logs wins (a job that was +/// interrupted before its final record). +pub(crate) fn load_output(dir: &Path, end: u64, cap: u64) -> std::io::Result { + let mut records = read_frames(&dir.join(HEAD_FILE))?; + records.extend(read_frames(&dir.join(TAIL_FILE))?); + records.sort_by_key(|(offset, _, _)| *offset); + let mut log = OutputLog::from_records(records, end); + log.enforce_cap(cap, 0); + Ok(log) +} + +struct OpenOutput { + head: Option>, + tail: Option>, + tail_bytes: u64, +} + +impl Pending { + fn push(&mut self, offset: u64, stream: OutputStream, data: &[u8], head_cap: u64) { + if data.is_empty() { + return; + } + // Contiguous output of one stream on one side of the head cap + // becomes one frame. + if let Some((last_offset, last_stream, last_data)) = self.frames.last_mut() + && *last_stream == stream + && *last_offset + last_data.len() as u64 == offset + && (*last_offset < head_cap) == (offset < head_cap) + { + last_data.extend_from_slice(data); + return; + } + self.frames.push((offset, stream, data.to_vec())); + } + + /// Drops the oldest bytes past the head cap beyond the newest + /// `tail_cap`. + fn trim_tail(&mut self, head_cap: u64, tail_cap: u64) { + while self.tail_bytes > tail_cap { + let Some(first) = self + .frames + .iter() + .position(|(offset, _, _)| *offset >= head_cap) + else { + return; + }; + let excess = self.tail_bytes - tail_cap; + let (offset, _, data) = &mut self.frames[first]; + if data.len() as u64 <= excess { + self.tail_bytes -= data.len() as u64; + self.frames.remove(first); + } else { + data.drain(..excess as usize); + *offset += excess; + self.tail_bytes -= excess; + } + } + } +} + +struct Writer { + root: PathBuf, + head_cap: u64, + tail_cap: u64, + pending: PendingOutput, + open: BTreeMap, +} + +impl Writer { + fn run(mut self, receiver: mpsc::Receiver) { + let mut flushed = Instant::now(); + loop { + // Flushes on the interval even while commands keep arriving. + let wait = FLUSH_INTERVAL.saturating_sub(flushed.elapsed()); + match receiver.recv_timeout(wait) { + Ok(command) => self.handle(command), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + let jobs: Vec = lock(&self.pending).keys().cloned().collect(); + for job_id in jobs { + self.drain(&job_id); + } + self.flush_all(); + return; + } + } + if flushed.elapsed() >= FLUSH_INTERVAL { + self.flush_all(); + flushed = Instant::now(); + } + } + } + + /// Writes the job's pending output. + fn drain(&mut self, job_id: &JobId) { + let frames = { + let mut pending = lock(&self.pending); + let Some(entry) = pending.get_mut(job_id) else { + return; + }; + entry.queued = false; + entry.tail_bytes = 0; + std::mem::take(&mut entry.frames) + }; + for (offset, stream, data) in frames { + if let Err(error) = self.append(job_id, offset, stream, &data) { + eprintln!("lightspeed-envd: could not store output of job {job_id}: {error}"); + return; + } + } + } + + fn handle(&mut self, command: Command) { + match command { + Command::WriteRecord { + job_id, + record, + done, + } => { + self.write_record(&job_id, &record); + if let Some(done) = done { + let _ = done.send(()); + } + } + Command::Drain { job_id } => self.drain(&job_id), + Command::Finish { + job_id, + record, + done, + } => { + self.drain(&job_id); + lock(&self.pending).remove(&job_id); + if let Some(mut output) = self.open.remove(&job_id) { + for writer in [output.head.as_mut(), output.tail.as_mut()] + .into_iter() + .flatten() + { + if let Err(error) = writer.flush() { + eprintln!( + "lightspeed-envd: could not flush output of job {job_id}: {error}" + ); + } + } + } + self.write_record(&job_id, &record); + let _ = done.send(()); + } + Command::Remove { job_id } => { + lock(&self.pending).remove(&job_id); + self.open.remove(&job_id); + let dir = self.root.join(job_id.as_str()); + if let Err(error) = std::fs::remove_dir_all(&dir) + && error.kind() != std::io::ErrorKind::NotFound + { + eprintln!("lightspeed-envd: could not remove job {job_id}: {error}"); + } + } + } + } + + fn write_record(&self, job_id: &JobId, record: &[u8]) { + let dir = self.root.join(job_id.as_str()); + let result = std::fs::create_dir_all(&dir).and_then(|()| { + let path = dir.join(RECORD_FILE); + let temp = dir.join(format!("{RECORD_FILE}.tmp")); + std::fs::write(&temp, record)?; + std::fs::rename(&temp, &path) + }); + if let Err(error) = result { + eprintln!("lightspeed-envd: could not persist job {job_id}: {error}"); + } + } + + fn append( + &mut self, + job_id: &JobId, + offset: u64, + stream: OutputStream, + data: &[u8], + ) -> std::io::Result<()> { + let head_cap = self.head_cap; + let head_len = head_cap.saturating_sub(offset).min(data.len() as u64) as usize; + let (head, tail) = data.split_at(head_len); + let dir = self.root.join(job_id.as_str()); + if !self.open.contains_key(job_id) { + std::fs::create_dir_all(&dir)?; + self.open.insert( + job_id.clone(), + OpenOutput { + head: None, + tail: None, + tail_bytes: file_len(&dir.join(TAIL_FILE)), + }, + ); + } + let output = self.open.get_mut(job_id).expect("opened above"); + if !head.is_empty() { + if output.head.is_none() { + output.head = Some(BufWriter::new(append_to(&dir.join(HEAD_FILE))?)); + } + let writer = output.head.as_mut().expect("opened above"); + write_frame(writer, offset, stream, head)?; + } + if !tail.is_empty() { + if output.tail.is_none() { + output.tail = Some(BufWriter::new(append_to(&dir.join(TAIL_FILE))?)); + } + let writer = output.tail.as_mut().expect("opened above"); + write_frame(writer, offset + head_len as u64, stream, tail)?; + output.tail_bytes += (FRAME_HEADER_BYTES + tail.len()) as u64; + if output.tail_bytes > 2 * self.tail_cap + 64 * 1024 { + let tail_cap = self.tail_cap; + compact_tail(&dir, output, tail_cap)?; + } + } + Ok(()) + } + + fn flush_all(&mut self) { + for (job_id, output) in &mut self.open { + for writer in [output.head.as_mut(), output.tail.as_mut()] + .into_iter() + .flatten() + { + if let Err(error) = writer.flush() { + eprintln!("lightspeed-envd: could not flush output of job {job_id}: {error}"); + } + } + } + } +} + +/// Rewrites the tail log with only its newest `tail_cap` bytes. +fn compact_tail(dir: &Path, output: &mut OpenOutput, tail_cap: u64) -> std::io::Result<()> { + if let Some(mut writer) = output.tail.take() { + writer.flush()?; + } + let path = dir.join(TAIL_FILE); + let mut records = read_frames(&path)?; + let mut kept = 0u64; + let mut first = records.len(); + while first > 0 && kept < tail_cap { + first -= 1; + kept += records[first].2.len() as u64; + } + let mut records = records.split_off(first); + if let Some((offset, _, data)) = records.first_mut() + && kept > tail_cap + { + let excess = (kept - tail_cap) as usize; + data.drain(..excess); + *offset += excess as u64; + } + let temp = dir.join(format!("{TAIL_FILE}.tmp")); + { + let mut writer = BufWriter::new(File::create(&temp)?); + for (offset, stream, data) in &records { + write_frame(&mut writer, *offset, *stream, data)?; + } + writer.flush()?; + } + std::fs::rename(&temp, &path)?; + output.tail_bytes = file_len(&path); + output.tail = Some(BufWriter::new(append_to(&path)?)); + Ok(()) +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn append_to(path: &Path) -> std::io::Result { + OpenOptions::new().create(true).append(true).open(path) +} + +fn file_len(path: &Path) -> u64 { + std::fs::metadata(path).map_or(0, |metadata| metadata.len()) +} + +fn write_frame( + writer: &mut impl std::io::Write, + offset: u64, + stream: OutputStream, + data: &[u8], +) -> std::io::Result<()> { + let stream = match stream { + OutputStream::Stdout => 0u8, + OutputStream::Stderr => 1, + OutputStream::Pty => 2, + }; + let len = u32::try_from(data.len()).map_err(std::io::Error::other)?; + writer.write_all(&offset.to_le_bytes())?; + writer.write_all(&[stream])?; + writer.write_all(&len.to_le_bytes())?; + writer.write_all(data) +} + +/// Every whole frame of a log; a frame cut short by a crash ends it. +fn read_frames(path: &Path) -> std::io::Result)>> { + let mut bytes = Vec::new(); + match File::open(path) { + Ok(mut file) => { + file.read_to_end(&mut bytes)?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + } + let mut frames = Vec::new(); + let mut rest = bytes.as_slice(); + while rest.len() >= FRAME_HEADER_BYTES { + let offset = u64::from_le_bytes(rest[..8].try_into().expect("8 bytes")); + let stream = match rest[8] { + 0 => OutputStream::Stdout, + 1 => OutputStream::Stderr, + 2 => OutputStream::Pty, + _ => break, + }; + let len = u32::from_le_bytes(rest[9..13].try_into().expect("4 bytes")) as usize; + let Some(data) = rest.get(FRAME_HEADER_BYTES..FRAME_HEADER_BYTES + len) else { + break; + }; + frames.push((offset, stream, data.to_vec())); + rest = &rest[FRAME_HEADER_BYTES + len..]; + } + Ok(frames) +} + +#[cfg(test)] +mod tests { + use environment_protocol::data::exec::GapReason; + + use super::*; + + #[tokio::test(flavor = "current_thread")] + async fn output_round_trips_with_head_tail_and_bounded_disk_use() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = JobStore::start(temp.path().to_owned(), 1000, 1000).expect("store"); + let job_id = JobId::new("verbose").unwrap(); + // 200 KiB through 8 KiB appends against a 1000-byte tail. + let mut offset = 0u64; + for i in 0..25u8 { + let data = vec![i; 8192]; + store.append(&job_id, offset, OutputStream::Stdout, &data); + offset += data.len() as u64; + } + store.finish(&job_id, b"{}".to_vec()).await; + let dir = temp.path().join("verbose"); + assert!( + file_len(&dir.join(TAIL_FILE)) + <= 2 * 1000 + 64 * 1024 + 8192 + 2 * FRAME_HEADER_BYTES as u64, + "the tail log is compacted" + ); + assert_eq!( + file_len(&dir.join(HEAD_FILE)), + 1000 + FRAME_HEADER_BYTES as u64 + ); + assert_eq!(std::fs::read(dir.join(RECORD_FILE)).unwrap(), b"{}"); + + let log = load_output(&dir, offset, 2000).expect("load"); + assert_eq!(log.end(), offset); + let page = log.page(0, 10_000, 4096); + assert_eq!(page.gaps.len(), 1); + assert_eq!(page.gaps[0].reason, GapReason::Overflow); + assert_eq!(page.gaps[0].offset, 1000); + assert_eq!(page.gaps[0].omitted_bytes, offset - 2000); + assert_eq!(page.bytes, 2000); + assert_eq!( + page.chunks.last().unwrap().data.as_slice().last(), + Some(&24) + ); + + store.remove(&job_id); + let _ = store + .write_record(&JobId::new("other").unwrap(), b"{}".to_vec()) + .await; + assert!(!dir.exists(), "commands run in order"); + } + + #[test] + fn pending_output_keeps_the_head_and_the_newest_tail_only() { + let mut pending = Pending::default(); + let (head_cap, tail_cap) = (100u64, 300u64); + let mut offset = 0u64; + for i in 0..50u8 { + let data = [i; 64]; + let head_len = head_cap.saturating_sub(offset).min(64) as usize; + let (head, tail) = data.split_at(head_len); + pending.push(offset, OutputStream::Stdout, head, head_cap); + pending.push( + offset + head_len as u64, + OutputStream::Stdout, + tail, + head_cap, + ); + pending.tail_bytes += tail.len() as u64; + pending.trim_tail(head_cap, tail_cap); + offset += 64; + let held: u64 = pending + .frames + .iter() + .map(|(_, _, data)| data.len() as u64) + .sum(); + assert!(held <= head_cap + tail_cap, "{held} bytes pending"); + } + let frames = &pending.frames; + assert_eq!(frames.len(), 2, "contiguous output coalesces per side"); + assert_eq!((frames[0].0, frames[0].2.len()), (0, 100)); + assert_eq!(frames[1].0, offset - tail_cap); + assert_eq!(frames[1].2.len() as u64, tail_cap); + assert_eq!(frames[1].2.last(), Some(&49)); + } + + #[test] + fn a_frame_cut_short_ends_the_log() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("log"); + let mut bytes = Vec::new(); + write_frame(&mut bytes, 0, OutputStream::Stderr, b"whole").unwrap(); + write_frame(&mut bytes, 5, OutputStream::Stdout, b"cut").unwrap(); + bytes.truncate(bytes.len() - 1); + std::fs::write(&path, bytes).unwrap(); + let frames = read_frames(&path).unwrap(); + assert_eq!(frames, vec![(0, OutputStream::Stderr, b"whole".to_vec())]); + } +} diff --git a/crates/environment-daemon/src/exec/log.rs b/crates/environment-daemon/src/exec/log.rs new file mode 100644 index 000000000..8e711c446 --- /dev/null +++ b/crates/environment-daemon/src/exec/log.rs @@ -0,0 +1,422 @@ +//! Byte-offset output logs. +//! +//! A log holds a handle's combined output as segments addressed by byte +//! offset. `end` counts every byte ever appended; a retention cap drops +//! bytes but never renumbers them, so a page cut at any byte loses nothing +//! and every range the log no longer holds is reported as a typed gap. + +use std::collections::VecDeque; + +use environment_protocol::{ + bytes::ByteChunk, + data::exec::{GapReason, OutputChunk, OutputGap, OutputStream}, +}; + +/// Consecutive appends from one stream merge into one segment up to this +/// size, which keeps the chunk count of a page low. +const MERGE_LIMIT_BYTES: usize = 64 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Segment { + offset: u64, + stream: OutputStream, + data: Vec, +} + +impl Segment { + fn end(&self) -> u64 { + self.offset + self.data.len() as u64 + } +} + +/// One page cut from a log. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Page { + pub chunks: Vec, + pub gaps: Vec, + /// Offset after the last returned byte or gap; the start when the page + /// is empty. + pub next: u64, + /// Bytes of `data` returned. + pub bytes: u64, +} + +impl Page { + pub fn items(&self) -> u32 { + u32::try_from(self.chunks.len() + self.gaps.len()).unwrap_or(u32::MAX) + } +} + +/// Retained output of one handle. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct OutputLog { + /// Sorted by offset, disjoint. + segments: VecDeque, + end: u64, + retained: u64, + /// Bytes below this offset were delivered and released. + released_until: u64, + /// Ranges `[start, end)` dropped from the middle, sorted, disjoint and + /// never adjacent. Ranges below `released_until` are forgotten and then + /// read as released. + overflow: Vec<(u64, u64)>, +} + +impl OutputLog { + /// A log rebuilt from persisted records, in offset order. Holes between + /// records, and a missing tail up to `end`, read as overflow. + pub fn from_records( + records: impl IntoIterator)>, + end: u64, + ) -> Self { + let mut log = Self::default(); + for (offset, stream, data) in records { + if data.is_empty() { + continue; + } + // Overlapping records (a tail that was compacted after a crash + // left a stale copy) keep the first copy of every byte. + let record_end = offset + data.len() as u64; + if record_end <= log.end { + continue; + } + let skip = log.end.saturating_sub(offset) as usize; + let offset = offset.max(log.end); + if offset > log.end { + log.add_overflow(log.end, offset); + log.end = offset; + } + log.append(stream, &data[skip..]); + } + if end > log.end { + log.add_overflow(log.end, end); + log.end = end; + } + log + } + + /// Bytes produced so far. + pub fn end(&self) -> u64 { + self.end + } + + /// Bytes held. + pub fn retained_bytes(&self) -> u64 { + self.retained + } + + pub fn append(&mut self, stream: OutputStream, data: &[u8]) { + if data.is_empty() { + return; + } + let offset = self.end; + match self.segments.back_mut() { + Some(last) + if last.stream == stream + && last.end() == offset + && last.data.len() + data.len() <= MERGE_LIMIT_BYTES => + { + last.data.extend_from_slice(data); + } + _ => self.segments.push_back(Segment { + offset, + stream, + data: data.to_vec(), + }), + } + self.end += data.len() as u64; + self.retained += data.len() as u64; + } + + /// Keeps at most `cap` bytes. First releases delivered bytes (below + /// `cursor`), oldest first; if still over, drops from the middle of the + /// undelivered region, keeping its head (the first half of `cap` from + /// `cursor`) and the newest bytes. Every dropped range becomes an + /// overflow gap; consecutive drops extend one gap. + pub fn enforce_cap(&mut self, cap: u64, cursor: u64) { + while self.retained > cap { + let Some(front) = self.segments.front_mut() else { + break; + }; + if front.offset >= cursor { + break; + } + let releasable = cursor.min(front.end()) - front.offset; + let release = releasable.min(self.retained - cap) as usize; + front.data.drain(..release); + front.offset += release as u64; + self.retained -= release as u64; + self.released_until = self.released_until.max(front.offset); + if front.data.is_empty() { + self.segments.pop_front(); + } + } + let released_until = self.released_until; + self.overflow.retain(|(_, end)| *end > released_until); + + if self.retained <= cap { + return; + } + let head_end = cursor.max(self.released_until).saturating_add(cap / 2); + let mut excess = self.retained - cap; + let mut index = self + .segments + .partition_point(|segment| segment.end() <= head_end); + while excess > 0 && index < self.segments.len() { + let segment = &mut self.segments[index]; + if segment.offset < head_end { + // The segment straddles the head: keep its head part. + let keep = (head_end - segment.offset) as usize; + let rest = segment.data.split_off(keep); + let tail = Segment { + offset: head_end, + stream: segment.stream, + data: rest, + }; + self.segments.insert(index + 1, tail); + index += 1; + continue; + } + let drop = excess.min(segment.data.len() as u64); + let start = segment.offset; + segment.data.drain(..drop as usize); + segment.offset += drop; + let emptied = segment.data.is_empty(); + self.retained -= drop; + excess -= drop; + self.add_overflow(start, start + drop); + if emptied { + self.segments.remove(index); + } + } + } + + /// A page from `start`: at most `max_bytes` of data, split at exactly + /// that byte, and at most `max_items` chunks and gaps together. An empty + /// budget returns an empty page at `start`. + pub fn page(&self, start: u64, max_bytes: u64, max_items: u32) -> Page { + let mut page = Page { + next: start, + ..Page::default() + }; + let mut pos = start; + let mut budget = max_bytes; + while pos < self.end && budget > 0 && page.items() < max_items { + if let Some(&(_, end)) = self + .overflow + .iter() + .find(|(range_start, end)| *range_start <= pos && pos < *end) + { + push_gap(&mut page, pos, end, GapReason::Overflow, max_items); + pos = end; + } else if pos < self.released_until { + let end = self + .overflow + .iter() + .map(|(range_start, _)| *range_start) + .find(|range_start| *range_start > pos) + .map_or(self.released_until, |range_start| { + range_start.min(self.released_until) + }); + push_gap(&mut page, pos, end, GapReason::Expired, max_items); + pos = end; + } else { + let index = self + .segments + .partition_point(|segment| segment.end() <= pos); + match self.segments.get(index) { + Some(segment) if segment.offset <= pos => { + let from = (pos - segment.offset) as usize; + let take = (segment.data.len() - from).min(budget as usize); + page.chunks.push(OutputChunk { + offset: pos, + stream: segment.stream, + data: ByteChunk::from(&segment.data[from..from + take]), + }); + pos += take as u64; + budget -= take as u64; + page.bytes += take as u64; + } + next => { + // A hole no rule explains reads as overflow. + let end = next.map_or(self.end, |segment| segment.offset); + push_gap(&mut page, pos, end, GapReason::Overflow, max_items); + pos = end; + } + } + } + page.next = pos; + } + page + } + + fn add_overflow(&mut self, start: u64, end: u64) { + if start >= end { + return; + } + let index = self + .overflow + .partition_point(|(_, range_end)| *range_end < start); + let mut merged = (start, end); + while let Some(&(range_start, range_end)) = self.overflow.get(index) { + if range_start > merged.1 { + break; + } + merged = (merged.0.min(range_start), merged.1.max(range_end)); + self.overflow.remove(index); + } + self.overflow.insert(index, merged); + } +} + +fn push_gap(page: &mut Page, start: u64, end: u64, reason: GapReason, max_items: u32) { + if let Some(last) = page.gaps.last_mut() + && last.reason == reason + && last.offset + last.omitted_bytes == start + && page + .chunks + .last() + .is_none_or(|chunk| chunk.offset < last.offset) + { + last.omitted_bytes += end - start; + return; + } + debug_assert!(page.items() < max_items); + page.gaps.push(OutputGap { + offset: start, + omitted_bytes: end - start, + reason, + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bytes(page: &Page) -> Vec { + page.chunks + .iter() + .flat_map(|chunk| chunk.data.as_slice().to_vec()) + .collect() + } + + fn log_of(parts: &[&[u8]]) -> OutputLog { + let mut log = OutputLog::default(); + for (index, part) in parts.iter().enumerate() { + let stream = if index % 2 == 0 { + OutputStream::Stdout + } else { + OutputStream::Stderr + }; + log.append(stream, part); + } + log + } + + #[test] + fn pages_cut_at_max_bytes_and_continue_without_loss() { + // Job paging used to skip the remainder of a cut chunk: every byte + // of 20 000 must arrive through 100-byte pages. + let data: Vec = (0..20_000u32).map(|i| (i % 251) as u8).collect(); + let mut log = OutputLog::default(); + for chunk in data.chunks(8192) { + log.append(OutputStream::Stdout, chunk); + } + let mut collected = Vec::new(); + let mut next = 0; + while next < log.end() { + let page = log.page(next, 100, 4096); + assert!(page.bytes <= 100); + assert!(page.gaps.is_empty()); + assert_eq!(page.chunks[0].offset, next); + collected.extend(bytes(&page)); + next = page.next; + } + assert_eq!(collected, data); + } + + #[test] + fn zero_budget_leaves_the_position_in_place() { + let log = log_of(&[b"abc"]); + let page = log.page(1, 0, 4096); + assert!(page.chunks.is_empty() && page.gaps.is_empty()); + assert_eq!(page.next, 1); + } + + #[test] + fn chunks_keep_stream_and_offset() { + let log = log_of(&[b"out", b"err"]); + let page = log.page(2, 100, 4096); + assert_eq!(page.chunks.len(), 2); + assert_eq!(page.chunks[0].offset, 2); + assert_eq!(page.chunks[0].stream, OutputStream::Stdout); + assert_eq!(page.chunks[1].offset, 3); + assert_eq!(page.chunks[1].stream, OutputStream::Stderr); + assert_eq!(bytes(&page), b"terr"); + assert_eq!(page.next, 6); + } + + #[test] + fn cap_drops_one_middle_range_and_keeps_head_and_newest() { + let mut log = OutputLog::default(); + for i in 0..64u8 { + log.append(OutputStream::Stdout, &[i; 100]); + log.enforce_cap(1000, 0); + } + assert_eq!(log.end(), 6400); + assert_eq!(log.retained_bytes(), 1000); + let page = log.page(0, 10_000, 4096); + assert_eq!(page.gaps.len(), 1, "one contiguous gap: {:?}", page.gaps); + assert_eq!(page.gaps[0].reason, GapReason::Overflow); + assert_eq!(page.gaps[0].offset, 500); + assert_eq!(page.gaps[0].omitted_bytes, 6400 - 1000); + let data = bytes(&page); + assert_eq!( + &data[..500], + &[[0u8; 100], [1; 100], [2; 100], [3; 100], [4; 100]].concat()[..] + ); + assert_eq!(data[999], 63, "the newest byte survives"); + assert_eq!(page.next, 6400); + } + + #[test] + fn delivered_bytes_are_released_first_and_read_as_expired() { + let mut log = log_of(&[b"0123456789"]); + // Everything up to 8 was delivered through the cursor. + log.append(OutputStream::Stdout, b"abcdef"); + log.enforce_cap(10, 8); + assert_eq!(log.retained_bytes(), 10); + let page = log.page(0, 100, 4096); + assert_eq!(page.gaps[0].reason, GapReason::Expired); + assert_eq!(page.gaps[0].offset, 0); + assert_eq!(page.gaps[0].omitted_bytes, 6); + assert_eq!(bytes(&page), b"6789abcdef"); + } + + #[test] + fn page_item_limit_ends_the_page_early_without_loss() { + let log = log_of(&[b"a", b"b", b"c", b"d"]); + let page = log.page(0, 100, 2); + assert_eq!(bytes(&page), b"ab"); + assert_eq!(page.next, 2); + let rest = log.page(page.next, 100, 2); + assert_eq!(bytes(&rest), b"cd"); + } + + #[test] + fn records_rebuild_head_tail_and_the_gap_between() { + let log = OutputLog::from_records( + [ + (0, OutputStream::Stdout, b"head".to_vec()), + (100, OutputStream::Stderr, b"tail".to_vec()), + ], + 110, + ); + assert_eq!(log.end(), 110); + let page = log.page(0, 100, 4096); + assert_eq!(bytes(&page), b"headtail"); + assert_eq!(page.gaps.len(), 2); + assert_eq!((page.gaps[0].offset, page.gaps[0].omitted_bytes), (4, 96)); + assert_eq!((page.gaps[1].offset, page.gaps[1].omitted_bytes), (104, 6)); + } +} diff --git a/crates/environment-daemon/src/exec/outcome.rs b/crates/environment-daemon/src/exec/outcome.rs new file mode 100644 index 000000000..05d1d01a0 --- /dev/null +++ b/crates/environment-daemon/src/exec/outcome.rs @@ -0,0 +1,179 @@ +//! How a root process ended, and what its group left behind. + +use environment_protocol::{ + data::exec::{ExitStatus, LeftoverDisposition, LeftoverProcess, LeftoverReport, SignalName}, + limits::{MAX_LEFTOVER_COMMAND_BYTES, MAX_LEFTOVER_PROCESSES}, +}; + +/// The wire exit of a reaped child: its status code, or the signal that +/// killed it by name. +pub(crate) fn exit_status(status: std::process::ExitStatus) -> ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + if let Some(signal) = status.signal() { + return ExitStatus::Signal { + signal: signal_name(signal), + core_dumped: status.core_dumped(), + }; + } + } + ExitStatus::Code { + code: status.code().unwrap_or(-1), + } +} + +/// `exit 3` or `SIGSEGV (core dumped)`, for error messages. +pub(crate) fn describe_exit(exit: &ExitStatus) -> String { + match exit { + ExitStatus::Code { code } => format!("exited with status {code}"), + ExitStatus::Signal { + signal, + core_dumped: true, + } => format!("terminated by signal {signal} (core dumped)"), + ExitStatus::Signal { signal, .. } => format!("terminated by signal {signal}"), + } +} + +/// The POSIX name of signal number `raw` on this platform. A number +/// without a portable name is spelled `SIG`. +pub(crate) fn signal_name(raw: i32) -> SignalName { + #[cfg(unix)] + { + use rustix::process::Signal; + let known: &[(Signal, &str)] = &[ + (Signal::HUP, "SIGHUP"), + (Signal::INT, "SIGINT"), + (Signal::QUIT, "SIGQUIT"), + (Signal::ILL, "SIGILL"), + (Signal::TRAP, "SIGTRAP"), + (Signal::ABORT, "SIGABRT"), + (Signal::BUS, "SIGBUS"), + (Signal::FPE, "SIGFPE"), + (Signal::KILL, "SIGKILL"), + (Signal::USR1, "SIGUSR1"), + (Signal::SEGV, "SIGSEGV"), + (Signal::USR2, "SIGUSR2"), + (Signal::PIPE, "SIGPIPE"), + (Signal::ALARM, "SIGALRM"), + (Signal::TERM, "SIGTERM"), + (Signal::CHILD, "SIGCHLD"), + (Signal::CONT, "SIGCONT"), + (Signal::STOP, "SIGSTOP"), + (Signal::TSTP, "SIGTSTP"), + (Signal::TTIN, "SIGTTIN"), + (Signal::TTOU, "SIGTTOU"), + (Signal::URG, "SIGURG"), + (Signal::XCPU, "SIGXCPU"), + (Signal::XFSZ, "SIGXFSZ"), + (Signal::VTALARM, "SIGVTALRM"), + (Signal::PROF, "SIGPROF"), + (Signal::WINCH, "SIGWINCH"), + (Signal::IO, "SIGIO"), + (Signal::SYS, "SIGSYS"), + #[cfg(target_os = "linux")] + (Signal::STKFLT, "SIGSTKFLT"), + #[cfg(target_os = "linux")] + (Signal::POWER, "SIGPWR"), + #[cfg(target_os = "macos")] + (Signal::EMT, "SIGEMT"), + #[cfg(target_os = "macos")] + (Signal::INFO, "SIGINFO"), + ]; + if let Some((_, name)) = known.iter().find(|(signal, _)| signal.as_raw() == raw) { + return SignalName::new(*name).expect("known signal names are valid"); + } + } + SignalName::new(format!("SIG{}", raw.unsigned_abs())) + .expect("SIG followed by at most 10 digits is a valid signal name") +} + +/// A report from enumerated members, lowest pid first: at most +/// `MAX_LEFTOVER_PROCESSES` listed, commands cut to +/// `MAX_LEFTOVER_COMMAND_BYTES` on a character boundary. +pub(crate) fn leftover_report( + mut members: Vec<(u32, Option)>, + disposition: LeftoverDisposition, +) -> Option { + if members.is_empty() { + return None; + } + members.sort_by_key(|(pid, _)| *pid); + let listed = MAX_LEFTOVER_PROCESSES as usize; + let omitted_process_count = + u32::try_from(members.len().saturating_sub(listed)).unwrap_or(u32::MAX); + members.truncate(listed); + let processes = members + .into_iter() + .map(|(pid, command)| LeftoverProcess { + pid, + command: command + .map(|command| cut(command, MAX_LEFTOVER_COMMAND_BYTES as usize)) + .filter(|command| !command.is_empty()), + }) + .collect(); + Some(LeftoverReport { + disposition, + processes, + omitted_process_count, + }) +} + +fn cut(mut text: String, max: usize) -> String { + if text.len() > max { + let mut end = max; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + } + text +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(unix)] + #[test] + fn signal_deaths_are_named_and_codes_kept() { + use std::os::unix::process::ExitStatusExt as _; + // A raw wait status: low 7 bits the signal, 0x80 the core flag. + let segv = + std::process::ExitStatus::from_raw(rustix::process::Signal::SEGV.as_raw() | 0x80); + assert_eq!( + exit_status(segv), + ExitStatus::Signal { + signal: SignalName::new("SIGSEGV").unwrap(), + core_dumped: true, + } + ); + let term = std::process::ExitStatus::from_raw(rustix::process::Signal::TERM.as_raw()); + assert_eq!( + describe_exit(&exit_status(term)), + "terminated by signal SIGTERM" + ); + let code = std::process::ExitStatus::from_raw(3 << 8); + assert_eq!(exit_status(code), ExitStatus::Code { code: 3 }); + assert_eq!(signal_name(200).as_str(), "SIG200"); + } + + #[test] + fn leftover_reports_are_capped_sorted_and_cut() { + let members = (0..100u32) + .rev() + .map(|pid| (pid + 10, Some("é".repeat(1000)))) + .collect(); + let report = leftover_report(members, LeftoverDisposition::Kept).expect("report"); + assert_eq!(report.processes.len(), MAX_LEFTOVER_PROCESSES as usize); + assert_eq!(report.omitted_process_count, 100 - MAX_LEFTOVER_PROCESSES); + assert_eq!(report.processes[0].pid, 10); + let command = report.processes[0].command.as_deref().unwrap(); + assert!(command.len() <= MAX_LEFTOVER_COMMAND_BYTES as usize); + assert!(command.len() >= MAX_LEFTOVER_COMMAND_BYTES as usize - 1); + assert!(leftover_report(Vec::new(), LeftoverDisposition::Swept).is_none()); + let unnamed = leftover_report(vec![(5, Some(String::new()))], LeftoverDisposition::Swept) + .expect("report"); + assert_eq!(unnamed.processes[0].command, None); + } +} diff --git a/crates/environment-daemon/src/exec/spawn_record.rs b/crates/environment-daemon/src/exec/spawn_record.rs new file mode 100644 index 000000000..1e3c6b4e2 --- /dev/null +++ b/crates/environment-daemon/src/exec/spawn_record.rs @@ -0,0 +1,101 @@ +//! Persisted spawn records: which process groups an earlier daemon +//! instance owned. +//! +//! A record names a group by its id, its leader's start time and the boot +//! it ran in. After a restart in the same boot, a group whose record still +//! matches is the earlier instance's: jobs kill it, processes adopt it as +//! leftovers. A group id cannot be reused while the group has members, so a +//! live group whose leader is gone is still the recorded one; a live leader +//! must also match its recorded start time. + +use std::path::Path; + +use environment_protocol::ids::DaemonInstanceId; +use serde::{Deserialize, Serialize}; + +use crate::process_group::{self, Group}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct SpawnRecord { + pub pgid: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub leader_start_time: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub boot_id: Option, + pub daemon_instance_id: DaemonInstanceId, +} + +impl SpawnRecord { + /// A record of `group`, spawned by this daemon instance. + pub fn new(group: Group, instance_id: &DaemonInstanceId) -> Self { + Self { + pgid: group.pgid, + leader_start_time: group.leader_start_time, + boot_id: process_group::boot_id(), + daemon_instance_id: instance_id.clone(), + } + } + + /// The recorded group. + pub fn group(&self) -> Group { + Group { + pgid: self.pgid, + leader_start_time: self.leader_start_time, + } + } + + /// The record was written in `boot`, the current boot. + pub fn same_boot(&self, boot: Option<&str>) -> bool { + boot.is_some() && self.boot_id.as_deref() == boot + } + + /// The recorded group still has members, and they are the recorded + /// group's. Only meaningful for a record of the current boot. + pub fn group_survives(&self) -> bool { + self.group().alive() + } +} + +/// Writes `value` as JSON to `path` through a sibling temporary file, so a +/// crash leaves either the old or the new content. +pub(crate) fn write_json_atomic(path: &Path, value: &T) -> std::io::Result<()> { + let bytes = serde_json::to_vec(value).map_err(std::io::Error::other)?; + let mut temp = path.as_os_str().to_owned(); + temp.push(".tmp"); + std::fs::write(&temp, bytes)?; + std::fs::rename(&temp, path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[tokio::test(flavor = "current_thread")] + async fn a_live_group_survives_and_a_reused_leader_does_not() { + let mut command = tokio::process::Command::new("/bin/sh"); + command.args(["-c", "sleep 30"]); + process_group::spawn_in_own_group(&mut command); + let mut child = command.spawn().expect("spawn"); + let pgid = child.id().expect("pid"); + let instance = DaemonInstanceId::new("envd_test").unwrap(); + let record = SpawnRecord::new(Group::capture(pgid), &instance); + assert!(record.same_boot(process_group::boot_id().as_deref())); + assert!(!record.same_boot(None)); + assert!(record.group_survives()); + + let reused = SpawnRecord { + leader_start_time: record.leader_start_time.map(|started| started + 1), + ..record.clone() + }; + assert!( + !reused.group_survives(), + "another process with the same pid" + ); + + process_group::signal_group(pgid, process_group::GroupSignal::Kill); + child.wait().await.expect("wait"); + assert!(!record.group_survives()); + } +} diff --git a/crates/environment-daemon/src/filesystem.rs b/crates/environment-daemon/src/filesystem.rs index ed49142e7..af69d711f 100644 --- a/crates/environment-daemon/src/filesystem.rs +++ b/crates/environment-daemon/src/filesystem.rs @@ -1,1119 +1,1407 @@ -#[cfg(any(target_os = "linux", target_os = "macos"))] -mod backend; -#[cfg(any(target_os = "linux", target_os = "macos"))] +//! envd's filesystem: the `fs/*` and `transfer/*` handlers. +//! +//! Every path goes through one [`Confinement`]: resolved beneath the root's +//! directory descriptor, following a symlink only while its target stays +//! inside the root, and never addressing the daemon state directory. Every +//! later operation names the resolved entry by its parent descriptor and +//! name, so nothing follows a link swapped in after resolution. Handlers do +//! their I/O on blocking threads; walks poll the request's cancellation +//! token between entries. + +mod confinement; +mod os; mod scan; -use environment_protocol::data::transfer::*; +mod search; mod transfer; +mod walk; + use std::{ - io, - path::{Component, Path, PathBuf}, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, + ffi::OsString, + fmt::Display, + io::{self, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + sync::Arc, }; +pub use confinement::Confinement; +use confinement::Lookup; use environment_protocol::{ - data::fs::{ - CopyParams, CopyResponse, CreateDirectoryParams, CreateDirectoryResponse, - GetMetadataParams, GetMetadataResponse, GlobFilesParams, GlobFilesResponse, GlobFilesStop, - ReadDirectoryEntry, ReadDirectoryParams, ReadDirectoryResponse, ReadFileParams, - ReadFileResponse, RemoveParams, RemoveResponse, SearchTextMatch, SearchTextParams, - SearchTextResponse, SearchTextStop, WriteFileParams, WriteFileResponse, + data::{ + fs::{ + CreateDirectoryParams, CreateDirectoryResponse, DirectoryEntry, FileKind, FileMetadata, + FileVersion, GetMetadataParams, GetMetadataResponse, GlobFilesParams, + GlobFilesResponse, ReadDirectoryParams, ReadDirectoryResponse, ReadFileParams, + ReadFileResponse, RemoveParams, RemoveResponse, RenameParams, RenameResponse, + SearchTextParams, SearchTextResponse, WriteCondition, WriteFileParams, + WriteFileResponse, + }, + handshake::FilesystemLimits, + scan::{ScanParams, ScanResponse}, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, ListTransferMissingParams, + ListTransferMissingResponse, ReadTransferContentParams, ReadTransferContentResponse, + ReadTransferInventoryParams, ReadTransferInventoryResponse, TransferOperationParams, + TransferStatus, WriteTransferContentParams, WriteTransferContentResponse, + }, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::{ByteChunk, EnvironmentPath}, + error::{Capability, PermissionReason, PreconditionReason, ProtocolError}, + ids::DirectoryCursor, + paths::{EntryName, EnvironmentPath}, }; -use grep_searcher::{BinaryDetection, SearcherBuilder, sinks::UTF8}; -use tokio::fs; +use os::{Directory, EntryStat, RenameMode}; +use rustix::io::Errno; +use sha2::{Digest, Sha256}; + +use crate::dispatch::RequestContext; + +/// Prefix of `fs/writeFile` staging files next to their target. +const WRITE_STAGE_PREFIX: &str = ".lightspeed-write-"; + +/// What the filesystem handlers need from the daemon. +pub struct FilesystemConfig { + /// The root and state directory; transfers journal in + /// `/transfers`. + pub confinement: Arc, + /// Canonical working directory, inside the root. + pub cwd: PathBuf, + pub writable: bool, + pub limits: FilesystemLimits, +} +/// The `fs/*` and `transfer/*` handlers. Cheap to clone. #[derive(Clone)] pub struct LocalFileSystem { - root: PathBuf, + inner: Arc, +} + +struct Inner { + confinement: Arc, cwd: PathBuf, writable: bool, - #[cfg(any(target_os = "linux", target_os = "macos"))] - transfers: std::sync::Arc>, + limits: FilesystemLimits, + transfers: transfer::Transfers, } impl LocalFileSystem { - pub fn new(root: PathBuf, cwd: PathBuf, writable: bool) -> Self { - Self { - root: normalize_path(root), - cwd: normalize_path(cwd), - writable, - #[cfg(any(target_os = "linux", target_os = "macos"))] - transfers: Default::default(), - } - } - - pub fn with_transfer_state_dir( - mut self, - path: PathBuf, - ) -> Result { - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - self.transfers = std::sync::Arc::new(std::sync::Mutex::new( - transfer::session::TransferManager::with_journal(path)?, - )); - let weak = std::sync::Arc::downgrade(&self.transfers); - std::thread::Builder::new() - .name("transfer-expiry".into()) - .spawn(move || { - loop { - std::thread::sleep(Duration::from_secs(30)); - let Some(manager) = weak.upgrade() else { - break; - }; - let directory = - manager.lock().ok().and_then(|mut manager| manager.expire()); - drop(manager); - if let Some(directory) = directory { - transfer::session::TransferManager::cleanup_expired(&directory); - } - } - }) - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - e.to_string(), - ) - })?; - } - Ok(self) - } - - pub async fn transfer( - &self, - mut request: environment_protocol::data::transfer_session::TransferRequest, - ) -> Result< - environment_protocol::data::transfer_session::TransferResponse, - EnvironmentProtocolError, - > { - use environment_protocol::data::transfer_session::{TransferRequest, TransferSelection}; - if let TransferRequest::Begin { selection, .. } = &mut request { - let path = match selection { - TransferSelection::Capture { source } => source, - TransferSelection::Materialize { destination, .. } - | TransferSelection::WriteFile { destination } => { - self.ensure_writable()?; - destination - } - }; - *path = EnvironmentPath::new(self.resolve_scoped_path(path)?.to_string_lossy()) - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - e.to_string(), - ) - })?; - } - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - let transfers = self.transfers.clone(); - let root = self.root.clone(); - tokio::task::spawn_blocking(move || { - transfers - .lock() - .map_err(|_| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - "transfer lock poisoned", - ) - })? - .execute(&root, request) - }) - .await - .map_err(|e| { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::Internal, e.to_string()) - })? - } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - { - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Unsupported, - "filesystem transfer backend unavailable", - )) - } + pub fn new(config: FilesystemConfig) -> Result { + let transfers = transfer::Transfers::new( + config.confinement.clone(), + config.limits.transfer, + config.writable, + )?; + Ok(Self { + inner: Arc::new(Inner { + confinement: config.confinement, + cwd: config.cwd, + writable: config.writable, + limits: config.limits, + transfers, + }), + }) } - pub async fn scan( - &self, - mut params: environment_protocol::data::inventory::ScanParams, - ) -> Result { - for path in &mut params.roots { - *path = EnvironmentPath::new(self.resolve_scoped_path(path)?.to_string_lossy()) - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - e.to_string(), - ) - })?; - } - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - let root = self.root.clone(); - tokio::task::spawn_blocking(move || scan::scan(&root, params)) - .await - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - e.to_string(), - ) - })? - } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - { - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Unsupported, - "filesystem scan backend unavailable", - )) - } + pub fn confinement(&self) -> &Arc { + &self.inner.confinement } - pub async fn capture( - &self, - params: CaptureParams, - ) -> Result { - transfer::capture(self, params).await + pub fn cwd(&self) -> &Path { + &self.inner.cwd } - pub async fn materialize( + async fn blocking( &self, - params: MaterializeParams, - ) -> Result { - self.ensure_writable()?; - transfer::materialize(self, params).await + work: impl FnOnce(&Inner) -> Result + Send + 'static, + ) -> Result { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || work(&inner)) + .await + .map_err(|error| ProtocolError::internal(format!("filesystem worker: {error}")))? } pub async fn read_file( &self, + _: &RequestContext, params: ReadFileParams, - ) -> Result { - let path = self.resolve(¶ms.path)?; - let metadata = fs::metadata(&path) - .await - .map_err(|error| io_error(error, &path))?; - if !metadata.is_file() { - // Preserve the exact outcome a plain read produces for non-file - // paths (directories fail with the authentic io error). - return match fs::read(&path).await { - Ok(data) => Ok(ReadFileResponse { - file_size: Some(data.len() as u64), - data: ByteChunk::from(data), - truncated: false, - }), - Err(error) => Err(io_error(error, &path)), - }; - } - let file_size = metadata.len(); - let offset = params.offset.unwrap_or(0).min(file_size); - let remaining = file_size - offset; - let take = params.max_bytes.map_or(remaining, |max| max.min(remaining)); - - let data = if offset == 0 && take == file_size { - fs::read(&path) - .await - .map_err(|error| io_error(error, &path))? - } else { - use tokio::io::{AsyncReadExt, AsyncSeekExt}; - let mut file = fs::File::open(&path) - .await - .map_err(|error| io_error(error, &path))?; - file.seek(std::io::SeekFrom::Start(offset)) - .await - .map_err(|error| io_error(error, &path))?; - let mut data = Vec::with_capacity(take.min(64 * 1024 * 1024) as usize); - file.take(take) - .read_to_end(&mut data) - .await - .map_err(|error| io_error(error, &path))?; - data - }; - let truncated = offset.saturating_add(data.len() as u64) < file_size; - Ok(ReadFileResponse { - data: ByteChunk::from(data), - file_size: Some(file_size), - truncated, - }) + ) -> Result { + self.blocking(move |inner| inner.read_file(params)).await } pub async fn write_file( &self, + _: &RequestContext, params: WriteFileParams, - ) -> Result { - self.ensure_writable()?; - let path = self.resolve(¶ms.path)?; - fs::write(&path, params.data.into_inner()) - .await - .map_err(|error| io_error(error, &path))?; - Ok(WriteFileResponse {}) + ) -> Result { + self.blocking(move |inner| inner.write_file(params)).await } pub async fn create_directory( &self, + _: &RequestContext, params: CreateDirectoryParams, - ) -> Result { - self.ensure_writable()?; - let path = self.resolve(¶ms.path)?; - if params.recursive.unwrap_or(false) { - fs::create_dir_all(&path) - .await - .map_err(|error| io_error(error, &path))?; - } else { - fs::create_dir(&path) - .await - .map_err(|error| io_error(error, &path))?; - } - Ok(CreateDirectoryResponse {}) + ) -> Result { + self.blocking(move |inner| inner.create_directory(params)) + .await } pub async fn get_metadata( &self, + _: &RequestContext, params: GetMetadataParams, - ) -> Result { - let path = self.resolve(¶ms.path)?; - let metadata = fs::symlink_metadata(&path) - .await - .map_err(|error| io_error(error, &path))?; - let file_type = metadata.file_type(); - let modified = metadata.modified().unwrap_or(UNIX_EPOCH); - let created = metadata.created().unwrap_or(modified); - Ok(GetMetadataResponse { - is_directory: file_type.is_dir(), - is_file: file_type.is_file(), - is_symlink: file_type.is_symlink(), - created_at_ms: system_time_ms(created), - modified_at_ms: system_time_ms(modified), - }) + ) -> Result { + self.blocking(move |inner| inner.get_metadata(params)).await } pub async fn read_directory( &self, + _: &RequestContext, params: ReadDirectoryParams, - ) -> Result { - let path = self.resolve(¶ms.path)?; - let mut directory = fs::read_dir(&path) - .await - .map_err(|error| io_error(error, &path))?; - let mut entries = Vec::new(); - while let Some(entry) = directory - .next_entry() + ) -> Result { + self.blocking(move |inner| inner.read_directory(params)) .await - .map_err(|error| io_error(error, &path))? - { - let metadata = entry - .metadata() - .await - .map_err(|error| io_error(error, &entry.path()))?; - entries.push(ReadDirectoryEntry { - file_name: entry.file_name().to_string_lossy().into_owned(), - is_directory: metadata.is_dir(), - is_file: metadata.is_file(), - }); - } - entries.sort_by(|left, right| left.file_name.cmp(&right.file_name)); - Ok(ReadDirectoryResponse { entries }) } pub async fn remove( &self, + _: &RequestContext, params: RemoveParams, - ) -> Result { - self.ensure_writable()?; - let path = self.resolve(¶ms.path)?; - let metadata = match fs::symlink_metadata(&path).await { - Ok(metadata) => metadata, - Err(error) - if params.force.unwrap_or(false) && error.kind() == io::ErrorKind::NotFound => - { - return Ok(RemoveResponse {}); - } - Err(error) => return Err(io_error(error, &path)), - }; - if metadata.is_dir() { - if params.recursive.unwrap_or(false) { - fs::remove_dir_all(&path) - .await - .map_err(|error| io_error(error, &path))?; - } else { - fs::remove_dir(&path) - .await - .map_err(|error| io_error(error, &path))?; - } - } else { - fs::remove_file(&path) - .await - .map_err(|error| io_error(error, &path))?; - } - Ok(RemoveResponse {}) + ) -> Result { + self.blocking(move |inner| inner.remove(params)).await } - pub async fn copy(&self, params: CopyParams) -> Result { - self.ensure_writable()?; - let source = self.resolve(¶ms.source_path)?; - let destination = self.resolve(¶ms.destination_path)?; - tokio::task::spawn_blocking(move || copy_path(&source, &destination, params.recursive)) - .await - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - error.to_string(), - ) - })??; - Ok(CopyResponse {}) + pub async fn rename( + &self, + _: &RequestContext, + params: RenameParams, + ) -> Result { + self.blocking(move |inner| inner.rename(params)).await } - /// Bounded recursive text search executed in-process with the ripgrep - /// engine crates. No external `rg` binary is involved, and the regex - /// dialect matches the caller-side generic fallback. + /// Bounded recursive regex search with the ripgrep engine crates. pub async fn search_text( &self, + context: &RequestContext, params: SearchTextParams, - ) -> Result { - let root = self.resolve(¶ms.root)?; - tokio::task::spawn_blocking(move || search_text_blocking(root, params)) + ) -> Result { + search::check_search(¶ms, &self.inner.limits.search)?; + let cancel = context.cancel.clone(); + self.blocking(move |inner| search::search_text(&inner.confinement, params, &cancel)) .await - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - error.to_string(), - ) - })? } - /// Bounded recursive file enumeration executed locally, so a broad glob - /// does not become one `fs/readDirectory` round trip per directory. + /// Bounded recursive enumeration of entries matching `filter.include`. pub async fn glob_files( &self, + context: &RequestContext, params: GlobFilesParams, - ) -> Result { - let root = self.resolve(¶ms.root)?; - tokio::task::spawn_blocking(move || glob_files_blocking(root, params)) + ) -> Result { + search::check_glob(¶ms, &self.inner.limits.glob)?; + let cancel = context.cancel.clone(); + self.blocking(move |inner| search::glob_files(&inner.confinement, params, &cancel)) .await - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - error.to_string(), - ) - })? } - fn resolve_scoped_path( + pub async fn scan( &self, - path: &EnvironmentPath, - ) -> Result { - if Path::new(path.as_str()).is_absolute() { - return self.resolve(path); - } - // Cwd and root are administrator-owned configuration. Resolve their aliases, - // then retain the configured root spelling for scope checks and returned paths. - // User-selected components are still opened without following symlinks. - let root = self - .root - .canonicalize() - .map_err(|e| io_error(e, &self.root))?; - let cwd = self - .cwd - .canonicalize() - .map_err(|e| io_error(e, &self.cwd))?; - let relative = cwd.strip_prefix(&root).map_err(|_| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Forbidden, - "configured cwd is outside filesystem root", - ) - })?; - let path = EnvironmentPath::new( - self.root - .join(relative) - .join(path.as_str()) - .to_string_lossy(), - ) - .map_err(|e| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - e.to_string(), - ) - })?; - self.resolve(&path) + context: &RequestContext, + params: ScanParams, + ) -> Result { + scan::check_scan(¶ms, &self.inner.limits.scan)?; + let cancel = context.cancel.clone(); + self.blocking(move |inner| scan::scan(&inner.confinement, params, &cancel)) + .await } - fn resolve(&self, path: &EnvironmentPath) -> Result { - let candidate = if path.is_absolute() { - PathBuf::from(path.as_str()) - } else if path.as_str() == "." { - self.cwd.clone() - } else { - self.cwd.join(path.as_str()) - }; - let normalized = normalize_path(candidate); - if !normalized.starts_with(&self.root) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Forbidden, - format!( - "path is outside bridge fs root: {} (root {})", - normalized.display(), - self.root.display() - ), - )); - } - Ok(normalized) + pub async fn transfer_begin( + &self, + _: &RequestContext, + params: BeginTransferParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.begin(params)) + .await + } + + pub async fn transfer_advance( + &self, + _: &RequestContext, + params: TransferOperationParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.advance(params)) + .await + } + + pub async fn transfer_get_status( + &self, + _: &RequestContext, + params: TransferOperationParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.get_status(params)) + .await } - fn ensure_writable(&self) -> Result<(), EnvironmentProtocolError> { + pub async fn transfer_read_inventory( + &self, + _: &RequestContext, + params: ReadTransferInventoryParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.read_inventory(params)) + .await + } + + pub async fn transfer_append( + &self, + _: &RequestContext, + params: AppendTransferInventoryParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.append(params)) + .await + } + + pub async fn transfer_list_missing( + &self, + _: &RequestContext, + params: ListTransferMissingParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.list_missing(params)) + .await + } + + pub async fn transfer_read( + &self, + _: &RequestContext, + params: ReadTransferContentParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.read(params)) + .await + } + + pub async fn transfer_write( + &self, + _: &RequestContext, + params: WriteTransferContentParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.write(params)) + .await + } + + pub async fn transfer_commit( + &self, + _: &RequestContext, + params: TransferOperationParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.commit(params)) + .await + } + + pub async fn transfer_abort( + &self, + _: &RequestContext, + params: TransferOperationParams, + ) -> Result { + self.blocking(move |inner| inner.transfers.abort(params)) + .await + } +} + +impl Inner { + fn ensure_writable(&self) -> Result<(), ProtocolError> { if self.writable { Ok(()) } else { - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::CapabilityUnavailable, - "bridge filesystem is read-only", + Err(ProtocolError::capability_unavailable( + Capability::FilesystemWrite, + "the filesystem is read-only", )) } } -} -fn search_text_blocking( - root: PathBuf, - params: SearchTextParams, -) -> Result { - if params.pattern.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "search pattern must not be empty", - )); + /// Open non-blocking, `fstat` the descriptor and refuse anything but a + /// regular file before reading; read at most `maxBytes` (plus one byte + /// to learn whether more follow). + fn read_file(&self, params: ReadFileParams) -> Result { + let ceiling = self.limits.max_read_file_bytes; + if params.max_bytes > ceiling { + return Err(ProtocolError::above_ceiling( + "maxBytes", + params.max_bytes, + ceiling, + )); + } + let path = ¶ms.path; + let resolved = self.confinement.lookup(path, Lookup::FOLLOW)?; + let stat = resolved.stat.expect("a followed lookup reports its target"); + require_regular(path, &stat)?; + let mut file = resolved + .parent + .open_read(&resolved.name) + .map_err(|error| io_error(error, path))?; + let opened = EntryStat::of_file(&file).map_err(|error| io_error(error, path))?; + require_regular(path, &opened)?; + if params.offset > 0 { + file.seek(SeekFrom::Start(params.offset)) + .map_err(|error| io_error(error, path))?; + } + let mut data = os::read_up_to(&mut file, params.max_bytes.saturating_add(1)) + .map_err(|error| io_error(error, path))?; + let read = data.len() as u64; + let more = read > params.max_bytes; + data.truncate(params.max_bytes as usize); + // Files such as procfs entries report size 0; their size is what + // could be read. Every other file's size is what `fstat` reported, + // also for an `offset` past its end. + let file_size = if opened.size == 0 && read > 0 { + params.offset.saturating_add(read) + } else { + opened.size + }; + let truncated = more || params.offset.saturating_add(data.len() as u64) < file_size; + Ok(ReadFileResponse { + data: data.into(), + file_size, + truncated, + version: opened.version(), + }) } - let limits = params.limits; - if limits.max_matches == 0 - || limits.max_files == 0 - || limits.max_bytes == 0 - || limits.max_duration_ms == 0 - { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "search limits must all be at least 1", + + /// Stage next to the final target, fsync, set the mode and rename over + /// it. + fn write_file(&self, params: WriteFileParams) -> Result { + self.ensure_writable()?; + let ceiling = self.limits.max_write_file_bytes; + let size = params.data.len() as u64; + if size > ceiling { + return Err(ProtocolError::above_ceiling("data", size, ceiling)); + } + let path = ¶ms.path; + let resolved = self + .confinement + .lookup(path, Lookup::FOLLOW.allowing_missing())?; + write_preconditions( + path, + resolved.stat.as_ref(), + params.create, + params.if_match.as_ref(), + )?; + let base = resolved + .stat + .map_or(os::DEFAULT_FILE_MODE, |stat| stat.mode); + let mode = if params.executable { + os::with_execute_bits(base) + } else { + base + }; + let rename = if params.create == WriteCondition::IfAbsent { + RenameMode::NoReplace + } else { + RenameMode::Replace + }; + let stage = OsString::from(format!( + "{WRITE_STAGE_PREFIX}{:032x}", + rand::random::() )); + let parent = &resolved.parent; + let io = |error| io_error(error, path); + let mut file = parent + .create_file(&stage, os::PRIVATE_FILE_MODE) + .map_err(io)?; + let staged = file + .write_all(params.data.as_slice()) + .and_then(|()| os::set_mode(&file, mode)) + .and_then(|()| file.sync_all()) + .and_then(|()| parent.rename(&stage, parent, &resolved.name, rename)); + if let Err(error) = staged { + let _ = parent.unlink(&stage); + if rename == RenameMode::NoReplace && error.kind() == io::ErrorKind::AlreadyExists { + return Err(create_condition(path, true)); + } + return Err(io(error)); + } + parent.sync().map_err(io)?; + let version = EntryStat::of_file(&file).map_err(io)?.version(); + Ok(WriteFileResponse { version }) } - let matcher = grep_regex::RegexMatcherBuilder::new() - .case_insensitive(!params.case_sensitive) - .build(¶ms.pattern) - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("invalid search regex: {error}"), - ) - })?; - let include = params - .include - .as_deref() - .map(glob::Pattern::new) - .transpose() - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("invalid search include glob: {error}"), - ) - })?; - - let started = Instant::now(); - let deadline = Duration::from_millis(limits.max_duration_ms); - let mut searcher = SearcherBuilder::new() - .line_number(true) - .binary_detection(BinaryDetection::quit(0)) - .build(); - let mut walker = ignore::WalkBuilder::new(&root); - // Mirror the generic traversal: no ignore-file or hidden filtering, no - // symlink following, deterministic order. The generic walker's max_depth - // counts directory descents below the root, so files at walker depth - // `max_depth + 1` are still in range. - walker - .standard_filters(false) - .follow_links(false) - .sort_by_file_path(|left, right| left.cmp(right)); - if let Some(max_depth) = params.max_depth { - walker.max_depth(Some((max_depth as usize).saturating_add(1))); + + fn create_directory( + &self, + params: CreateDirectoryParams, + ) -> Result { + self.ensure_writable()?; + let path = ¶ms.path; + let lookup = if params.recursive { + Lookup::FOLLOW.allowing_missing().creating_parents() + } else { + Lookup::FOLLOW.allowing_missing() + }; + let resolved = self.confinement.lookup(path, lookup)?; + match resolved.stat { + Some(stat) if params.recursive && stat.is_dir() => {} + Some(_) => { + return Err(ProtocolError::already_exists(format!("{path} exists"))); + } + None => match resolved + .parent + .mkdir(&resolved.name, os::DEFAULT_DIRECTORY_MODE) + { + Ok(()) => resolved + .parent + .sync() + .map_err(|error| io_error(error, path))?, + Err(error) + if params.recursive + && error.kind() == io::ErrorKind::AlreadyExists + && resolved + .parent + .stat(&resolved.name) + .is_ok_and(|stat| stat.is_dir()) => {} + Err(error) => return Err(io_error(error, path)), + }, + } + Ok(CreateDirectoryResponse {}) } - let mut matches = Vec::new(); - let mut files_searched = 0u64; - let mut bytes_searched = 0u64; - let mut stopped = None; + fn get_metadata( + &self, + params: GetMetadataParams, + ) -> Result { + let path = ¶ms.path; + let lookup = if params.follow_symlinks { + Lookup::FOLLOW + } else { + Lookup::NO_FOLLOW + }; + let resolved = self.confinement.lookup(path, lookup)?; + let stat = resolved + .parent + .stat_with_birth(&resolved.name) + .map_err(|error| io_error(error, path))?; + Ok(GetMetadataResponse { + metadata: metadata(&resolved.parent, &resolved.name, &stat), + }) + } - 'walk: for entry in walker.build() { - // Unreadable entries are skipped rather than failing the search. - let Ok(entry) = entry else { continue }; - if !entry.file_type().is_some_and(|kind| kind.is_file()) { - continue; - } - let path = entry.path(); - if let Some(include) = &include - && !local_path_matches_include(include, path, &root) - { - continue; + fn read_directory( + &self, + params: ReadDirectoryParams, + ) -> Result { + let ceiling = self.limits.max_directory_entries; + if params.limit == 0 { + return Err(ProtocolError::invalid_field( + "limit", + "limit must be at least 1", + )); } - if started.elapsed() >= deadline { - stopped = Some(SearchTextStop::TimeLimit); - break; + if params.limit > ceiling { + return Err(ProtocolError::above_ceiling( + "limit", + params.limit.into(), + ceiling.into(), + )); } - if files_searched >= limits.max_files { - stopped = Some(SearchTextStop::FileLimit); - break; + let position = params.cursor.as_ref().map(cursor::decode).transpose()?; + let path = ¶ms.path; + let resolved = self.confinement.lookup(path, Lookup::FOLLOW)?; + if !resolved.stat.is_some_and(|stat| stat.is_dir()) { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{path} is not a directory"), + )); } - let file_len = entry.metadata().map(|metadata| metadata.len()).unwrap_or(0); - if bytes_searched.saturating_add(file_len) > limits.max_bytes { - stopped = Some(SearchTextStop::ByteLimit); - break; + let directory = resolved + .open_directory() + .map_err(|error| io_error(error, path))?; + let mut names = directory.names().map_err(|error| io_error(error, path))?; + if resolved + .stat + .is_some_and(|stat| self.confinement.holds_state(&stat)) + { + names.retain(|name| { + directory + .stat(name) + .map_or(true, |stat| !self.confinement.is_state(&stat)) + }); } - files_searched += 1; - bytes_searched = bytes_searched.saturating_add(file_len); - - let match_path = response_path(¶ms.root, path, &root)?; - let mut hit_match_limit = false; - let sink = UTF8(|line_number, line| { - if matches.len() as u64 >= limits.max_matches { - hit_match_limit = true; - return Ok(false); - } - matches.push(SearchTextMatch { - path: match_path.clone(), - line_number, - line: line.trim_end_matches(['\r', '\n']).to_string(), + names.sort_by(|left, right| left.as_encoded_bytes().cmp(right.as_encoded_bytes())); + let start = position.map_or(0, |position| position.start(&names)); + let end = start.saturating_add(params.limit as usize).min(names.len()); + let page = &names[start.min(end)..end]; + let mut response = ReadDirectoryResponse { + entries: Vec::with_capacity(page.len()), + next_cursor: (end < names.len()) + .then(|| { + page.last() + .map(|name| cursor::encode(name.as_encoded_bytes())) + }) + .flatten(), + skipped_entry_count: 0, + }; + for name in page { + let Some(entry_name) = os::utf8_name(name).and_then(|name| EntryName::new(name).ok()) + else { + response.skipped_entry_count += 1; + continue; + }; + let stat = match directory.stat_with_birth(name) { + Ok(stat) => stat, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(io_error(error, format!("{path}/{entry_name}"))), + }; + response.entries.push(DirectoryEntry { + name: entry_name, + metadata: metadata(&directory, name, &stat), }); - Ok(true) - }); - // Per-file search errors (e.g. the file vanished mid-walk) skip the - // file rather than failing the whole search. - let _ = searcher.search_path(&matcher, path, sink); - // Like the generic fallback, the match limit reports truncation only - // when a match beyond the cap actually arrived. - if hit_match_limit { - stopped = Some(SearchTextStop::MatchLimit); - break 'walk; } + Ok(response) } - Ok(SearchTextResponse { - matches, - files_searched, - bytes_searched, - elapsed_ms: started.elapsed().as_millis() as u64, - stopped, - }) -} - -fn glob_files_blocking( - root: PathBuf, - params: GlobFilesParams, -) -> Result { - if params.pattern.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "glob pattern must not be empty", - )); - } - let limits = params.limits; - if limits.max_matches == 0 || limits.max_entries == 0 || limits.max_duration_ms == 0 { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "glob limits must all be at least 1", - )); - } - let pattern = glob::Pattern::new(¶ms.pattern).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("invalid glob pattern: {error}"), - ) - })?; - - let started = Instant::now(); - let deadline = Duration::from_millis(limits.max_duration_ms); - let mut walker = ignore::WalkBuilder::new(&root); - walker - .standard_filters(false) - .follow_links(false) - .sort_by_file_path(|left, right| left.cmp(right)); - if let Some(max_depth) = params.max_depth { - walker.max_depth(Some((max_depth as usize).saturating_add(1))); - } - - let mut matches = Vec::new(); - let mut entries_visited = 0u64; - let mut stopped = None; - - for entry in walker.build() { - let Ok(entry) = entry else { continue }; - if entry.depth() == 0 { - continue; + fn remove(&self, params: RemoveParams) -> Result { + self.ensure_writable()?; + let path = ¶ms.path; + let resolved = match self + .confinement + .lookup(path, Lookup::NO_FOLLOW.allowing_missing()) + { + Ok(resolved) => resolved, + Err(error) + if params.force + && matches!( + error.data, + environment_protocol::error::ErrorData::NotFound { .. } + ) => + { + return Ok(RemoveResponse {}); + } + Err(error) => return Err(error), + }; + if resolved.is_root() { + return Err(filesystem_root()); } - if started.elapsed() >= deadline { - stopped = Some(GlobFilesStop::TimeLimit); - break; + self.confinement.refuse_containing_state(&resolved, path)?; + let io = |error| io_error(error, path); + match resolved.stat { + None if params.force => return Ok(RemoveResponse {}), + None => return Err(ProtocolError::not_found(format!("{path} does not exist"))), + Some(stat) if stat.is_dir() => { + if params.recursive { + resolved.parent.remove_tree(&resolved.name).map_err(io)?; + } else { + resolved.parent.rmdir(&resolved.name).map_err(io)?; + } + } + Some(_) => resolved.parent.unlink(&resolved.name).map_err(io)?, } - if entries_visited >= limits.max_entries { - stopped = Some(GlobFilesStop::EntryLimit); - break; + resolved.parent.sync().map_err(io)?; + Ok(RemoveResponse {}) + } + + fn rename(&self, params: RenameParams) -> Result { + self.ensure_writable()?; + let source = self.confinement.lookup(¶ms.source, Lookup::NO_FOLLOW)?; + let destination = self + .confinement + .lookup(¶ms.destination, Lookup::NO_FOLLOW.allowing_missing())?; + if source.is_root() || destination.is_root() { + return Err(filesystem_root()); } - entries_visited += 1; - if !entry.file_type().is_some_and(|kind| kind.is_file()) { - continue; + self.confinement + .refuse_containing_state(&source, ¶ms.source)?; + self.confinement + .refuse_containing_state(&destination, ¶ms.destination)?; + let source_stat = source.stat.expect("an existing source has a stat"); + if let Some(expected) = ¶ms.if_match + && source_stat.version() != *expected + { + return Err(ProtocolError::failed_precondition( + PreconditionReason::IfMatchFailed, + format!("{} changed since that version", params.source), + )); } - let path = entry.path(); - let caller_path = response_path(¶ms.root, path, &root)?; - if !glob_pattern_matches(&pattern, ¶ms.pattern, &caller_path, path, &root) { - continue; + if destination.segments.starts_with(&source.segments) + && destination.segments != source.segments + { + return Err(ProtocolError::invalid_field( + "destination", + "a directory cannot move into itself", + )); } - // Like the generic fallback, the match limit reports truncation only - // when a match beyond the cap actually arrives. - if matches.len() as u64 >= limits.max_matches { - stopped = Some(GlobFilesStop::MatchLimit); - break; + if let Some(stat) = &destination.stat { + if stat.is_dir() { + return Err(ProtocolError::failed_precondition( + PreconditionReason::IsDirectory, + format!( + "{} is a directory and is never replaced", + params.destination + ), + )); + } + if !params.overwrite { + return Err(ProtocolError::already_exists(format!( + "{} exists", + params.destination + ))); + } } - matches.push(caller_path); + let mode = if params.overwrite { + RenameMode::Replace + } else { + RenameMode::NoReplace + }; + source + .parent + .rename(&source.name, &destination.parent, &destination.name, mode) + .map_err(|error| io_error(error, ¶ms.destination))?; + source + .parent + .sync() + .and_then(|()| destination.parent.sync()) + .map_err(|error| io_error(error, ¶ms.destination))?; + Ok(RenameResponse {}) } +} - Ok(GlobFilesResponse { - matches, - entries_visited, - elapsed_ms: started.elapsed().as_millis() as u64, - stopped, - }) +fn require_regular(path: &EnvironmentPath, stat: &EntryStat) -> Result<(), ProtocolError> { + match stat.kind { + FileKind::File => Ok(()), + FileKind::Directory => Err(ProtocolError::failed_precondition( + PreconditionReason::IsDirectory, + format!("{path} is a directory"), + )), + FileKind::Symlink | FileKind::Other => Err(ProtocolError::failed_precondition( + PreconditionReason::NotRegularFile, + format!("{path} is not a regular file"), + )), + } } -/// Glob pattern semantics mirror the caller-side generic tool: an absolute -/// pattern matches the caller-space path; otherwise the root-relative path -/// matches, and a pattern without `/` also matches the bare file name. -fn glob_pattern_matches( - pattern: &glob::Pattern, - pattern_text: &str, - caller_path: &EnvironmentPath, - path: &Path, - root: &Path, -) -> bool { - if pattern_text.starts_with('/') { - return pattern.matches(caller_path.as_str()); +/// The checks of `fs/writeFile` and the transfer `writeFile` selection on +/// the final target as it is now. +pub(crate) fn write_preconditions( + path: &EnvironmentPath, + current: Option<&EntryStat>, + create: WriteCondition, + if_match: Option<&FileVersion>, +) -> Result<(), ProtocolError> { + if let Some(stat) = current { + require_regular(path, stat)?; } - let relative = path - .strip_prefix(root) - .map(|relative| { - relative - .to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/") - }) - .unwrap_or_else(|_| path.to_string_lossy().into_owned()); - pattern.matches(&relative) - || (!pattern_text.contains('/') - && path - .file_name() - .is_some_and(|file_name| pattern.matches(&file_name.to_string_lossy()))) + match (current, create) { + (Some(_), WriteCondition::IfAbsent) => return Err(create_condition(path, true)), + (None, WriteCondition::IfPresent) => return Err(create_condition(path, false)), + _ => {} + } + if let Some(expected) = if_match + && current.is_none_or(|stat| stat.version() != *expected) + { + return Err(ProtocolError::failed_precondition( + PreconditionReason::IfMatchFailed, + format!("{path} is not at the expected version"), + )); + } + Ok(()) } -/// Include globs match the root-relative path or the bare file name, exactly -/// like the caller-side generic fallback. -fn local_path_matches_include(pattern: &glob::Pattern, path: &Path, root: &Path) -> bool { - let relative = path - .strip_prefix(root) - .map(|relative| { - relative - .to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/") - }) - .unwrap_or_else(|_| path.to_string_lossy().into_owned()); - pattern.matches(&relative) - || path - .file_name() - .is_some_and(|file_name| pattern.matches(&file_name.to_string_lossy())) +fn create_condition(path: &EnvironmentPath, exists: bool) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::CreateCondition, + if exists { + format!("{path} exists") + } else { + format!("{path} does not exist") + }, + ) } -/// Map a matched local path back into the caller's path space by joining its -/// root-relative suffix onto the requested root. -fn response_path( - requested_root: &EnvironmentPath, - path: &Path, - resolved_root: &Path, -) -> Result { - let relative = path.strip_prefix(resolved_root).map_err(|_| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - format!("search match escaped its root: {}", path.display()), - ) - })?; - let relative = relative - .to_string_lossy() - .replace(std::path::MAIN_SEPARATOR, "/"); - if relative.is_empty() { - return Ok(requested_root.clone()); +fn filesystem_root() -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::FilesystemRoot, + "the filesystem root cannot be removed, renamed or replaced", + ) +} + +fn metadata(parent: &Directory, name: &std::ffi::OsStr, stat: &EntryStat) -> FileMetadata { + let symlink_target = (stat.kind == FileKind::Symlink) + .then(|| parent.read_link(name).ok()) + .flatten() + .and_then(|target| target.into_string().ok()); + FileMetadata { + kind: stat.kind, + size_bytes: stat.size, + executable: stat.kind == FileKind::File && stat.executable(), + modified_at_ms: stat.modified_ms(), + created_at_ms: stat.birth_ms, + symlink_target, + version: stat.version(), } - let joined = format!( - "{}/{}", - requested_root.as_str().trim_end_matches('/'), - relative - ); - EnvironmentPath::new(&joined).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - format!("search match produced an invalid environment path {joined}: {error}"), - ) - }) } -fn copy_path( - source: &Path, - destination: &Path, - recursive: bool, -) -> Result<(), EnvironmentProtocolError> { - let metadata = std::fs::symlink_metadata(source).map_err(|error| io_error(error, source))?; - if metadata.is_dir() { - if !recursive { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "copy requires recursive=true when source is a directory", - )); +/// Bytes `value` takes as a JSON string, quotes included: `"` and `\` +/// and the short control escapes take two bytes, every other control +/// character six (`\u00XX`). What response budgets charge, so an answer +/// within its budget also encodes within it. +pub(crate) fn json_string_len(value: &str) -> u64 { + let escapes: u64 = value + .bytes() + .map(|byte| match byte { + b'"' | b'\\' | b'\n' | b'\r' | b'\t' | 0x08 | 0x0c => 1, + 0x00..=0x1f => 5, + _ => 0, + }) + .sum(); + value.len() as u64 + escapes + 2 +} + +/// envd's one mapping of OS errors to protocol error kinds. +pub(crate) fn io_error(error: io::Error, subject: impl Display) -> ProtocolError { + let message = format!("{subject}: {error}"); + let precondition = |reason| ProtocolError::failed_precondition(reason, message.clone()); + match Errno::from_io_error(&error) { + Some(Errno::LOOP) => return precondition(PreconditionReason::SymlinkLoop), + Some(Errno::NOTDIR) => return precondition(PreconditionReason::NotDirectory), + Some(Errno::ISDIR) => return precondition(PreconditionReason::IsDirectory), + Some(Errno::NOTEMPTY) => return precondition(PreconditionReason::NotEmpty), + Some(Errno::NOSPC | Errno::DQUOT) => return precondition(PreconditionReason::NoSpace), + Some(Errno::XDEV) => return precondition(PreconditionReason::CrossDevice), + Some(Errno::ROFS) => { + return ProtocolError::capability_unavailable(Capability::FilesystemWrite, message); + } + _ => {} + } + match error.kind() { + io::ErrorKind::NotFound => ProtocolError::not_found(message), + io::ErrorKind::AlreadyExists => ProtocolError::already_exists(message), + io::ErrorKind::PermissionDenied => { + ProtocolError::permission_denied(Some(PermissionReason::FilePermission), message) } - copy_directory(source, destination) - } else { - if let Some(parent) = destination.parent() { - std::fs::create_dir_all(parent).map_err(|error| io_error(error, parent))?; + io::ErrorKind::Interrupted | io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut => { + ProtocolError::unavailable(message) } - std::fs::copy(source, destination) - .map(|_| ()) - .map_err(|error| io_error(error, destination)) + _ => ProtocolError::internal(message), } } -fn copy_directory(source: &Path, destination: &Path) -> Result<(), EnvironmentProtocolError> { - std::fs::create_dir_all(destination).map_err(|error| io_error(error, destination))?; - for entry in std::fs::read_dir(source).map_err(|error| io_error(error, source))? { - let entry = entry.map_err(|error| io_error(error, source))?; - let source_child = entry.path(); - let destination_child = destination.join(entry.file_name()); - let metadata = std::fs::symlink_metadata(&source_child) - .map_err(|error| io_error(error, &source_child))?; - if metadata.is_dir() { - copy_directory(&source_child, &destination_child)?; +/// `fs/readDirectory` cursors: the last name returned, as hex. Names too +/// long for the 255-byte cursor carry a prefix and the name's SHA-256. +mod cursor { + use super::*; + + /// Longest name encoded in full (`x` and two hex digits per byte). + const MAX_EXACT_NAME_BYTES: usize = 127; + const PREFIX_BYTES: usize = 64; + + pub(super) enum Position { + After(Vec), + AfterHashed { prefix: Vec, sha256: Vec }, + } + + pub(super) fn encode(name: &[u8]) -> DirectoryCursor { + let value = if name.len() <= MAX_EXACT_NAME_BYTES { + format!("x{}", hex::encode(name)) } else { - std::fs::copy(&source_child, &destination_child) - .map(|_| ()) - .map_err(|error| io_error(error, &destination_child))?; - } + format!( + "h{}.{}", + hex::encode(&name[..PREFIX_BYTES]), + hex::encode(Sha256::digest(name)) + ) + }; + DirectoryCursor::new(value).expect("hex cursors are short and printable") } - Ok(()) -} -fn normalize_path(path: PathBuf) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(std::path::MAIN_SEPARATOR.to_string()), - Component::CurDir => {} - Component::ParentDir => { - normalized.pop(); - } - Component::Normal(segment) => normalized.push(segment), + pub(super) fn decode(cursor: &DirectoryCursor) -> Result { + let malformed = + || ProtocolError::invalid_field("cursor", "not a cursor this daemon minted"); + let value = cursor.as_str(); + if let Some(exact) = value.strip_prefix('x') { + return hex::decode(exact) + .map(Position::After) + .map_err(|_| malformed()); + } + let (prefix, sha256) = value + .strip_prefix('h') + .and_then(|hashed| hashed.split_once('.')) + .ok_or_else(malformed)?; + let prefix = hex::decode(prefix).map_err(|_| malformed())?; + let sha256 = hex::decode(sha256).map_err(|_| malformed())?; + if prefix.len() != PREFIX_BYTES || sha256.len() != 32 { + return Err(malformed()); } + Ok(Position::AfterHashed { prefix, sha256 }) } - normalized -} -fn io_error(error: io::Error, path: &Path) -> EnvironmentProtocolError { - let code = match error.kind() { - io::ErrorKind::NotFound => EnvironmentProtocolErrorCode::NotFound, - io::ErrorKind::PermissionDenied => EnvironmentProtocolErrorCode::Forbidden, - io::ErrorKind::AlreadyExists => EnvironmentProtocolErrorCode::Conflict, - io::ErrorKind::InvalidInput | io::ErrorKind::InvalidData => { - EnvironmentProtocolErrorCode::InvalidRequest + impl Position { + /// Index of the first name after this position in sorted `names`. + pub(super) fn start(&self, names: &[OsString]) -> usize { + match self { + Self::After(last) => { + names.partition_point(|name| name.as_encoded_bytes() <= last.as_slice()) + } + Self::AfterHashed { prefix, sha256 } => names + .iter() + .position(|name| { + let bytes = name.as_encoded_bytes(); + bytes.starts_with(prefix) && Sha256::digest(bytes).as_slice() == sha256 + }) + .map(|index| index + 1) + // The last name vanished: continue after its prefix. + .unwrap_or_else(|| { + names.partition_point(|name| name.as_encoded_bytes() <= prefix.as_slice()) + }), + } } - _ => EnvironmentProtocolErrorCode::Internal, - }; - EnvironmentProtocolError::new(code, format!("{}: {}", path.display(), error)) -} - -fn system_time_ms(value: SystemTime) -> i64 { - value - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_millis().min(i64::MAX as u128) as i64) - .unwrap_or_default() + } } #[cfg(test)] mod tests { + use std::os::unix::fs::{PermissionsExt, symlink}; + + use environment_protocol::{ + data::handshake::{GlobCeilings, ScanCeilings, SearchCeilings, TransferCeilings}, + error::{ErrorData, ProtocolErrorKind}, + }; + use super::*; - #[tokio::test(flavor = "current_thread")] - async fn filesystem_reads_and_writes_under_root() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let fs = LocalFileSystem::new(root.clone(), root.clone(), true); - let path = EnvironmentPath::new(root.join("file.txt").to_string_lossy()) - .expect("environment path"); - - fs.write_file(WriteFileParams { - path: path.clone(), - data: ByteChunk::from(b"hello".as_slice()), - }) - .await - .expect("write"); - let read = fs - .read_file(ReadFileParams { - path, - offset: None, - max_bytes: None, - }) - .await - .expect("read") - .data - .into_inner(); + pub(crate) fn limits() -> FilesystemLimits { + FilesystemLimits { + max_read_file_bytes: 8 * 1024 * 1024, + max_write_file_bytes: 8 * 1024 * 1024, + max_directory_entries: 1000, + search: SearchCeilings { + max_matches: 10_000, + max_files: 100_000, + max_bytes: 1 << 30, + max_duration_ms: 60_000, + max_line_bytes: 4096, + }, + glob: GlobCeilings { + max_matches: 100_000, + max_files: 1_000_000, + max_duration_ms: 60_000, + }, + scan: ScanCeilings { + max_roots: 32, + max_patterns: 32, + max_entries: 10_000, + max_file_bytes: 256 * 1024, + max_total_bytes: 4 * 1024 * 1024, + max_duration_ms: 30_000, + }, + transfer: TransferCeilings { + max_active_operations: 8, + max_content_chunk_bytes: 4 * 1024 * 1024, + max_inventory_page_entries: 1000, + receipt_retention_ms: 86_400_000, + max_entries: 100_000, + max_depth: 64, + max_file_bytes: 1 << 40, + max_total_bytes: 1 << 40, + max_manifest_bytes: 32 * 1024 * 1024, + max_duration_ms: 86_400_000, + }, + } + } - assert_eq!(read, b"hello"); + pub(crate) fn context() -> RequestContext { + RequestContext::detached() } - #[tokio::test(flavor = "current_thread")] - async fn filesystem_rejects_root_escape() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().join("root"); - std::fs::create_dir(&root).expect("root"); - let root = root.canonicalize().expect("canonical root"); - let fs = LocalFileSystem::new(root, temp.path().to_path_buf(), true); - let outside = - EnvironmentPath::new(temp.path().join("outside.txt").to_string_lossy()).unwrap(); + /// A filesystem rooted at `root` with its state directory inside. + pub(crate) fn filesystem(root: &Path, writable: bool) -> LocalFileSystem { + let confinement = + Arc::new(Confinement::new(root.to_path_buf(), root.join(".state")).unwrap()); + LocalFileSystem::new(FilesystemConfig { + confinement, + cwd: root.to_path_buf(), + writable, + limits: limits(), + }) + .unwrap() + } - let error = fs - .read_file(ReadFileParams { - path: outside, - offset: None, - max_bytes: None, - }) - .await - .expect_err("escape should fail"); + fn path(root: &Path, relative: &str) -> EnvironmentPath { + EnvironmentPath::new(format!("{}/{relative}", root.display())).unwrap() + } - assert_eq!(error.code, EnvironmentProtocolErrorCode::Forbidden); + fn reason(error: &ProtocolError) -> Option { + match error.data { + ErrorData::FailedPrecondition { reason } => Some(reason), + _ => None, + } } - fn search_fixture() -> (tempfile::TempDir, LocalFileSystem, EnvironmentPath) { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - std::fs::create_dir_all(root.join("src/deep")).expect("dirs"); - std::fs::write( - root.join("src/lib.rs"), - "pub fn target() {}\nfn other() {}\n", - ) - .expect("lib"); - std::fs::write(root.join("src/deep/nested.rs"), "target nested\n").expect("nested"); - std::fs::write(root.join("readme.md"), "target readme\n").expect("readme"); - std::fs::write(root.join("binary.bin"), b"tar\x00get\n").expect("binary"); - let fs = LocalFileSystem::new(root.clone(), root.clone(), false); - let environment_root = - EnvironmentPath::new(root.to_string_lossy()).expect("environment root"); - (temp, fs, environment_root) + fn read(path: EnvironmentPath, max_bytes: u64) -> ReadFileParams { + ReadFileParams { + path, + offset: 0, + max_bytes, + } } - fn limits() -> environment_protocol::data::fs::SearchTextLimits { - environment_protocol::data::fs::SearchTextLimits { - max_matches: 100, - max_files: 100, - max_bytes: 1024 * 1024, - max_duration_ms: 10_000, + fn write(path: EnvironmentPath, data: &[u8]) -> WriteFileParams { + WriteFileParams { + path, + data: data.into(), + create: WriteCondition::Always, + if_match: None, + executable: false, } } - fn search_params(root: &EnvironmentPath) -> SearchTextParams { - SearchTextParams { - root: root.clone(), - pattern: "target".to_owned(), - include: None, - case_sensitive: true, - max_depth: None, + #[tokio::test(flavor = "current_thread")] + async fn special_files_are_refused_before_any_read() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + // The root is `/` so a workspace link to a device stays inside it. + let confinement = Arc::new(Confinement::new("/".into(), base.join("state")).unwrap()); + let fs = LocalFileSystem::new(FilesystemConfig { + confinement, + cwd: base.clone(), + writable: false, limits: limits(), + }) + .unwrap(); + symlink("/dev/zero", base.join("zero")).unwrap(); + let status = std::process::Command::new("mkfifo") + .arg(base.join("fifo")) + .status() + .unwrap(); + assert!(status.success()); + for subject in [ + EnvironmentPath::new("/dev/zero").unwrap(), + path(&base, "zero"), + path(&base, "fifo"), + ] { + let error = fs + .read_file(&context(), read(subject.clone(), 1024)) + .await + .unwrap_err(); + assert_eq!( + reason(&error), + Some(PreconditionReason::NotRegularFile), + "{subject}" + ); } + let error = fs + .read_file( + &context(), + read(EnvironmentPath::new(base.to_str().unwrap()).unwrap(), 1), + ) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::IsDirectory)); + } + + #[cfg(target_os = "linux")] + #[tokio::test(flavor = "current_thread")] + async fn zero_size_proc_files_report_the_bytes_read() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let confinement = Arc::new(Confinement::new("/".into(), base.join("state")).unwrap()); + let fs = LocalFileSystem::new(FilesystemConfig { + confinement, + cwd: base, + writable: false, + limits: limits(), + }) + .unwrap(); + let response = fs + .read_file( + &context(), + read(EnvironmentPath::new("/proc/self/status").unwrap(), 1 << 20), + ) + .await + .unwrap(); + assert!(!response.data.is_empty()); + assert_eq!(response.file_size, response.data.len() as u64); + assert!(!response.truncated); } #[tokio::test(flavor = "current_thread")] - async fn search_finds_matches_with_include_and_depth_semantics() { - let (_temp, fs, root) = search_fixture(); - - let mut params = search_params(&root); - params.include = Some("*.rs".to_owned()); - let result = fs.search_text(params).await.expect("search"); - let mut paths = result - .matches - .iter() - .map(|item| item.path.as_str().to_owned()) - .collect::>(); - paths.sort(); + async fn ranged_reads_bound_every_read_and_report_truncation() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + std::fs::write(root.join("file"), b"0123456789").unwrap(); + let ranged = fs + .read_file( + &context(), + ReadFileParams { + path: path(&root, "file"), + offset: 2, + max_bytes: 3, + }, + ) + .await + .unwrap(); + assert_eq!(ranged.data.as_slice(), b"234"); + assert_eq!(ranged.file_size, 10); + assert!(ranged.truncated); + let past_end = fs + .read_file( + &context(), + ReadFileParams { + path: path(&root, "file"), + offset: 100, + max_bytes: 3, + }, + ) + .await + .unwrap(); + assert!(past_end.data.as_slice().is_empty()); assert_eq!( - paths, - vec![ - format!("{}/src/deep/nested.rs", root.as_str()), - format!("{}/src/lib.rs", root.as_str()), - ] - ); - assert_eq!(result.matches[0].line_number, 1); - assert!(result.stopped.is_none()); - // NUL-containing files are treated as binary and skipped, so the - // binary fixture never matches. - assert!( - result - .matches - .iter() - .all(|item| !item.path.as_str().ends_with(".bin")) + past_end.file_size, 10, + "the size of the file, not the offset" ); - - // max_depth 0 mirrors the generic traversal: files directly under - // the root only. - let mut params = search_params(&root); - params.max_depth = Some(0); - let shallow = fs.search_text(params).await.expect("shallow search"); - assert_eq!(shallow.matches.len(), 1); - assert!(shallow.matches[0].path.as_str().ends_with("readme.md")); + assert!(!past_end.truncated); + let whole = fs + .read_file(&context(), read(path(&root, "file"), 10)) + .await + .unwrap(); + assert!(!whole.truncated); + let error = fs + .read_file(&context(), read(path(&root, "file"), u64::MAX)) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::InvalidParams { + ceiling: Some(_), + .. + } + )); } #[tokio::test(flavor = "current_thread")] - async fn search_stops_at_each_limit_with_a_typed_reason() { - let (_temp, fs, root) = search_fixture(); - - let mut params = search_params(&root); - params.limits.max_matches = 1; - let matches = fs.search_text(params).await.expect("match limit"); - assert_eq!(matches.matches.len(), 1); - assert_eq!(matches.stopped, Some(SearchTextStop::MatchLimit)); - - let mut params = search_params(&root); - params.limits.max_files = 1; - let files = fs.search_text(params).await.expect("file limit"); - assert_eq!(files.stopped, Some(SearchTextStop::FileLimit)); - assert_eq!(files.files_searched, 1); - - let mut params = search_params(&root); - params.limits.max_bytes = 1; - let bytes = fs.search_text(params).await.expect("byte limit"); - assert_eq!(bytes.stopped, Some(SearchTextStop::ByteLimit)); - assert!(bytes.matches.is_empty()); + async fn writes_are_atomic_conditional_and_keep_links_and_modes() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + std::fs::create_dir(root.join("src")).unwrap(); + std::fs::write(root.join("src/tool.sh"), b"old").unwrap(); + std::fs::set_permissions( + root.join("src/tool.sh"), + std::fs::Permissions::from_mode(0o700), + ) + .unwrap(); + symlink("src/tool.sh", root.join("link")).unwrap(); + + let current = fs + .read_file(&context(), read(path(&root, "link"), 16)) + .await + .unwrap() + .version; + let written = fs + .write_file( + &context(), + WriteFileParams { + if_match: Some(current.clone()), + ..write(path(&root, "link"), b"new") + }, + ) + .await + .unwrap(); + assert!( + std::fs::symlink_metadata(root.join("link")) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(std::fs::read(root.join("src/tool.sh")).unwrap(), b"new"); + let mode = std::fs::metadata(root.join("src/tool.sh")) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o700); + assert_ne!(written.version, current); + + let stale = fs + .write_file( + &context(), + WriteFileParams { + if_match: Some(current), + ..write(path(&root, "src/tool.sh"), b"lost") + }, + ) + .await + .unwrap_err(); + assert_eq!(reason(&stale), Some(PreconditionReason::IfMatchFailed)); + let exists = fs + .write_file( + &context(), + WriteFileParams { + create: WriteCondition::IfAbsent, + ..write(path(&root, "src/tool.sh"), b"x") + }, + ) + .await + .unwrap_err(); + assert_eq!(reason(&exists), Some(PreconditionReason::CreateCondition)); + let missing = fs + .write_file( + &context(), + WriteFileParams { + create: WriteCondition::IfPresent, + ..write(path(&root, "src/new"), b"x") + }, + ) + .await + .unwrap_err(); + assert_eq!(reason(&missing), Some(PreconditionReason::CreateCondition)); + fs.write_file( + &context(), + WriteFileParams { + executable: true, + ..write(path(&root, "src/run"), b"#!/bin/sh\n") + }, + ) + .await + .unwrap(); + let mode = std::fs::metadata(root.join("src/run")) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o755); + let error = fs + .write_file(&context(), write(path(&root, "src"), b"x")) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::IsDirectory)); + let error = fs + .write_file(&context(), write(path(&root, "absent/file"), b"x")) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + // No staging file is left behind. + let leftovers: Vec<_> = std::fs::read_dir(root.join("src")) + .unwrap() + .flatten() + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(WRITE_STAGE_PREFIX) + }) + .collect(); + assert!(leftovers.is_empty()); } #[tokio::test(flavor = "current_thread")] - async fn search_rejects_invalid_requests_and_root_escape() { - let (temp, fs, root) = search_fixture(); - - let mut params = search_params(&root); - params.pattern = "(unclosed".to_owned(); - let error = fs.search_text(params).await.expect_err("invalid regex"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::InvalidRequest); - - let mut params = search_params(&root); - params.limits.max_matches = 0; - let error = fs.search_text(params).await.expect_err("zero limit"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::InvalidRequest); - - let nested_root = temp.path().join("root"); - std::fs::create_dir(&nested_root).expect("nested root"); - let nested_root = nested_root.canonicalize().expect("canonical nested root"); - let confined = LocalFileSystem::new(nested_root, temp.path().to_path_buf(), false); - let error = confined - .search_text(search_params(&root)) + async fn rename_never_replaces_directories_or_unasked_files() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + std::fs::create_dir(root.join("dir")).unwrap(); + std::fs::write(root.join("a"), b"a").unwrap(); + std::fs::write(root.join("b"), b"b").unwrap(); + let rename = |source: &str, destination: &str, overwrite: bool| RenameParams { + source: path(&root, source), + destination: path(&root, destination), + overwrite, + if_match: None, + }; + let error = fs + .rename(&context(), rename("a", "b", false)) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::AlreadyExists); + let error = fs + .rename(&context(), rename("a", "dir", true)) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::IsDirectory)); + fs.rename(&context(), rename("a", "b", true)).await.unwrap(); + assert_eq!(std::fs::read(root.join("b")).unwrap(), b"a"); + fs.rename(&context(), rename("b", "dir/c", false)) + .await + .unwrap(); + assert!(root.join("dir/c").exists()); + let error = fs + .rename(&context(), rename(".state", "moved", false)) .await - .expect_err("escape should fail"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::Forbidden); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::PermissionDenied); } - fn glob_params(root: &EnvironmentPath, pattern: &str) -> GlobFilesParams { - GlobFilesParams { - root: root.clone(), - pattern: pattern.to_owned(), - max_depth: None, - limits: environment_protocol::data::fs::GlobFilesLimits { - max_matches: 100, - max_entries: 100, - max_duration_ms: 10_000, + #[tokio::test(flavor = "current_thread")] + async fn removal_refuses_the_root_and_the_state_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + let remove = |path: EnvironmentPath| RemoveParams { + path, + recursive: true, + force: false, + }; + let error = fs + .remove( + &context(), + remove(EnvironmentPath::new(root.to_str().unwrap()).unwrap()), + ) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::FilesystemRoot)); + let error = fs + .remove(&context(), remove(path(&root, ".state"))) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::DaemonState) + } + )); + fs.remove( + &context(), + RemoveParams { + force: true, + ..remove(path(&root, "missing/deeper")) }, - } + ) + .await + .unwrap(); + let read_only = filesystem(&root, false); + let error = read_only + .remove(&context(), remove(path(&root, "anything"))) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::CapabilityUnavailable); } #[tokio::test(flavor = "current_thread")] - async fn glob_enumerates_with_generic_pattern_semantics_and_limits() { - let (_temp, fs, root) = search_fixture(); - - let result = fs - .glob_files(glob_params(&root, "**/*.rs")) - .await - .expect("glob"); - let mut paths = result - .matches - .iter() - .map(|path| path.as_str().to_owned()) - .collect::>(); - paths.sort(); - assert_eq!( - paths, - vec![ - format!("{}/src/deep/nested.rs", root.as_str()), - format!("{}/src/lib.rs", root.as_str()), - ] - ); - assert!(result.stopped.is_none()); - assert!(result.entries_visited >= 4); - - // A pattern without '/' also matches bare file names anywhere. - let by_name = fs - .glob_files(glob_params(&root, "*.md")) + async fn directories_page_by_cursor_and_hide_the_state_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + for name in ["c", "a", "b\\slash", "d"] { + std::fs::write(root.join(name), name).unwrap(); + } + symlink("a", root.join("link")).unwrap(); + let long = "n".repeat(200); + std::fs::write(root.join(&long), "long").unwrap(); + let mut names = Vec::new(); + let mut cursor = None; + loop { + let page = fs + .read_directory( + &context(), + ReadDirectoryParams { + path: EnvironmentPath::new(root.to_str().unwrap()).unwrap(), + limit: 2, + cursor: cursor.take(), + }, + ) + .await + .unwrap(); + names.extend( + page.entries + .iter() + .map(|entry| (entry.name.as_str().to_owned(), entry.metadata.kind)), + ); + match page.next_cursor { + Some(next) => cursor = Some(next), + None => break, + } + } + let expected: Vec<(String, FileKind)> = [ + ("a", FileKind::File), + ("b\\slash", FileKind::File), + ("c", FileKind::File), + ("d", FileKind::File), + ("link", FileKind::Symlink), + (long.as_str(), FileKind::File), + ] + .into_iter() + .map(|(name, kind)| (name.to_owned(), kind)) + .collect(); + assert_eq!(names, expected); + let error = fs + .read_directory( + &context(), + ReadDirectoryParams { + path: EnvironmentPath::new(root.to_str().unwrap()).unwrap(), + limit: 1, + cursor: Some(DirectoryCursor::new("garbage").unwrap()), + }, + ) .await - .expect("glob by name"); - assert_eq!(by_name.matches.len(), 1); - assert!(by_name.matches[0].as_str().ends_with("readme.md")); - - let mut params = glob_params(&root, "**/*"); - params.limits.max_matches = 1; - let matches = fs.glob_files(params).await.expect("match limit"); - assert_eq!(matches.matches.len(), 1); - assert_eq!(matches.stopped, Some(GlobFilesStop::MatchLimit)); - - let mut params = glob_params(&root, "**/*"); - params.limits.max_entries = 2; - let entries = fs.glob_files(params).await.expect("entry limit"); - assert_eq!(entries.stopped, Some(GlobFilesStop::EntryLimit)); - assert_eq!(entries.entries_visited, 2); - - let mut params = glob_params(&root, "**/*"); - params.pattern = String::new(); - let error = fs.glob_files(params).await.expect_err("empty pattern"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::InvalidRequest); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); } #[tokio::test(flavor = "current_thread")] - async fn ranged_read_truncates_at_the_source_and_reports_true_size() { - let (_temp, fs, root) = search_fixture(); - let path = EnvironmentPath::new(format!("{}/readme.md", root.as_str())).expect("path"); - let full = std::fs::read(std::path::Path::new(root.as_str()).join("readme.md")) - .expect("full contents"); - - let ranged = fs - .read_file(ReadFileParams { - path: path.clone(), - offset: Some(0), - max_bytes: Some(6), - }) + async fn metadata_describes_links_and_follows_on_request() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + std::fs::create_dir(root.join("dir")).unwrap(); + symlink("dir", root.join("link")).unwrap(); + let link = fs + .get_metadata( + &context(), + GetMetadataParams { + path: path(&root, "link"), + follow_symlinks: false, + }, + ) .await - .expect("ranged read"); - assert_eq!(ranged.data.as_slice(), &full[..6]); - assert_eq!(ranged.file_size, Some(full.len() as u64)); - assert!(ranged.truncated); - - let tail = fs - .read_file(ReadFileParams { - path: path.clone(), - offset: Some(7), - max_bytes: None, - }) + .unwrap() + .metadata; + assert_eq!(link.kind, FileKind::Symlink); + assert_eq!(link.symlink_target.as_deref(), Some("dir")); + let followed = fs + .get_metadata( + &context(), + GetMetadataParams { + path: path(&root, "link"), + follow_symlinks: true, + }, + ) .await - .expect("tail read"); - assert_eq!(tail.data.as_slice(), &full[7..]); - assert!(!tail.truncated); + .unwrap() + .metadata; + assert_eq!(followed.kind, FileKind::Directory); + } - let whole = fs - .read_file(ReadFileParams { - path, - offset: None, - max_bytes: Some(u64::MAX), - }) - .await - .expect("whole read"); - assert_eq!(whole.data.as_slice(), full.as_slice()); - assert!(!whole.truncated); - assert_eq!(whole.file_size, Some(full.len() as u64)); + #[test] + fn io_errors_map_to_protocol_kinds() { + let error = |errno: Errno| io_error(io::Error::from(errno), "subject"); + assert_eq!( + reason(&error(Errno::LOOP)), + Some(PreconditionReason::SymlinkLoop) + ); + assert_eq!( + reason(&error(Errno::NOTEMPTY)), + Some(PreconditionReason::NotEmpty) + ); + assert_eq!( + reason(&error(Errno::XDEV)), + Some(PreconditionReason::CrossDevice) + ); + assert_eq!(error(Errno::NOENT).kind(), ProtocolErrorKind::NotFound); + assert_eq!(error(Errno::EXIST).kind(), ProtocolErrorKind::AlreadyExists); + assert!(matches!( + error(Errno::ACCESS).data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::FilePermission) + } + )); + assert_eq!( + error(Errno::ROFS).kind(), + ProtocolErrorKind::CapabilityUnavailable + ); + } + + #[test] + fn json_string_lengths_count_escapes() { + for value in [ + "", + "plain", + "a\"b\\c", + "line\n\ttab", + "\u{1b}[31mred\u{1b}[0m", + "\0", + "ünï", + ] { + let encoded = serde_json::to_string(value).unwrap(); + assert_eq!(json_string_len(value), encoded.len() as u64, "{value:?}"); + } } } diff --git a/crates/environment-daemon/src/filesystem/backend/mod.rs b/crates/environment-daemon/src/filesystem/backend/mod.rs deleted file mode 100644 index d07fa92bc..000000000 --- a/crates/environment-daemon/src/filesystem/backend/mod.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! OS boundary for confined traversal and atomic publication. Wire paths and inventories -//! contain no descriptors, inode numbers, or Unix permission bits. A Windows backend can -//! implement this interface with directory handles, reparse-point checks and native rename. -#[cfg(any(target_os = "linux", target_os = "macos"))] -mod unix; -#[cfg(any(target_os = "linux", target_os = "macos"))] -pub use unix::*; - -use environment_protocol::{ - data::inventory::MAX_INVENTORY_PATH_BYTES, - error::{EnvironmentProtocolError as Error, EnvironmentProtocolErrorCode as Code}, -}; -use sha2::{Digest, Sha256}; -use std::path::Path; -pub(crate) type Result = std::result::Result; -pub(crate) fn error(code: Code, message: impl Into) -> Error { - Error::new(code, message) -} -pub(crate) fn invalid(message: &str) -> Error { - error(Code::InvalidRequest, message) -} -pub(crate) fn io(e: std::io::Error) -> Error { - use std::io::ErrorKind::*; - if is_path_violation(&e) { - return error(Code::Forbidden, e.to_string()); - } - error( - match e.kind() { - NotFound => Code::NotFound, - AlreadyExists => Code::Conflict, - PermissionDenied => Code::Forbidden, - Unsupported => Code::Unsupported, - InvalidInput => Code::InvalidRequest, - _ => Code::Internal, - }, - e.to_string(), - ) -} -pub(crate) fn conflict() -> Error { - error( - Code::Conflict, - "filesystem observation changed; retry the filesystem operation", - ) -} -pub(crate) fn valid_path(path: &str) -> bool { - path.len() <= MAX_INVENTORY_PATH_BYTES - && (path.is_empty() - || (!path.contains(['\\', '\0']) - && path - .split('/') - .all(|p| !p.is_empty() && p != "." && p != ".."))) -} -pub(crate) fn valid_digest(value: &str) -> bool { - value.strip_prefix("sha256:").is_some_and(|s| { - s.len() == 64 - && s.bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) - }) -} -pub(crate) fn digest(hash: Sha256) -> String { - format!("sha256:{}", hex::encode(hash.finalize())) -} -pub(crate) fn relative( - root: &Path, - path: &environment_protocol::shared::EnvironmentPath, -) -> Result { - let path = Path::new(path.as_str()) - .strip_prefix(root) - .map_err(|_| error(Code::Forbidden, "selection outside filesystem root"))?; - let value = path - .components() - .map(|component| match component { - std::path::Component::Normal(name) => name - .to_str() - .map(str::to_owned) - .ok_or_else(|| invalid("non-UTF-8 selection")), - _ => Err(invalid("invalid selection component")), - }) - .collect::>>()? - .join("/"); - if !valid_path(&value) { - return Err(invalid("invalid selection")); - } - Ok(value) -} diff --git a/crates/environment-daemon/src/filesystem/backend/unix.rs b/crates/environment-daemon/src/filesystem/backend/unix.rs deleted file mode 100644 index 7be100dd7..000000000 --- a/crates/environment-daemon/src/filesystem/backend/unix.rs +++ /dev/null @@ -1,504 +0,0 @@ -use std::{ - ffi::{CStr, CString}, - fs::{File, Metadata}, - io, - os::unix::fs::{MetadataExt, PermissionsExt}, - path::{Component, Path}, -}; - -use rustix::fs::{self, AtFlags, FileType, Mode, OFlags}; - -pub struct Directory(File); -#[derive(Clone, Debug)] -pub struct Observation(Metadata); -impl Observation { - pub fn is_dir(&self) -> bool { - self.0.is_dir() - } - pub fn size(&self) -> u64 { - self.0.len() - } - pub fn executable(&self) -> bool { - self.0.mode() & 0o111 != 0 - } - pub fn matches(&self, other: &Self) -> bool { - let (a, b) = (&self.0, &other.0); - a.dev() == b.dev() - && a.ino() == b.ino() - && a.mode() == b.mode() - && a.len() == b.len() - && a.mtime() == b.mtime() - && a.mtime_nsec() == b.mtime_nsec() - && a.ctime() == b.ctime() - && a.ctime_nsec() == b.ctime_nsec() - } -} -fn c(name: &str) -> io::Result { - CString::new(name).map_err(|_| io::Error::from(io::ErrorKind::InvalidInput)) -} -fn check(meta: Metadata) -> io::Result { - if meta.is_file() || meta.is_dir() { - Ok(Observation(meta)) - } else { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "scoped filesystem access does not support symlinks or special files", - )) - } -} -pub fn observe(file: &File) -> io::Result { - check(file.metadata()?) -} -pub fn set_executable(file: &File, executable: bool) -> io::Result<()> { - file.set_permissions(std::fs::Permissions::from_mode(if executable { - 0o755 - } else { - 0o644 - })) -} -/// Streaming directory enumeration; cleanup never collects an entire retired tree. -struct Entries(fs::Dir); -impl Iterator for Entries { - type Item = io::Result; - fn next(&mut self) -> Option { - self.0.find_map(|entry| match entry { - Ok(entry) => { - let name = entry.file_name(); - if name.to_bytes() == b"." || name.to_bytes() == b".." { - None - } else { - Some(Ok(name.to_owned())) - } - } - Err(error) => Some(Err(error.into())), - }) - } -} -fn entries(file: &File) -> io::Result { - // Open a separate description so enumeration never shares the caller's offset. - let fd = fs::openat( - file, - ".", - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, - Mode::empty(), - )?; - Ok(Entries(fs::Dir::new(fd)?)) -} - -pub fn is_path_violation(error: &io::Error) -> bool { - matches!( - rustix::io::Errno::from_io_error(error), - Some(rustix::io::Errno::LOOP | rustix::io::Errno::NOTDIR) - ) -} -pub fn create_private_directory(path: &Path) -> io::Result<()> { - use std::os::unix::fs::DirBuilderExt; - std::fs::DirBuilder::new() - .recursive(true) - .mode(0o700) - .create(path) -} -pub fn sync_directory(path: &Path) -> io::Result<()> { - Directory::anchor(path)?.sync() -} - -impl Directory { - pub fn anchor(root: &Path) -> io::Result { - // Configured roots may contain platform aliases (/var on macOS). Resolve this - // administrator-owned anchor once; user selections are always opened relative to it. - let root = root.canonicalize()?; - let mut dir = Self( - fs::open( - "/", - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::empty(), - )? - .into(), - ); - for part in root.components() { - if let Component::Normal(name) = part { - let name = name - .to_str() - .ok_or_else(|| io::Error::from(io::ErrorKind::InvalidInput))?; - dir = dir.child(name)?; - } - } - Ok(dir) - } - pub fn sync(&self) -> io::Result<()> { - self.0.sync_all() - } - pub fn clone_dir(&self) -> io::Result { - Ok(Self(self.0.try_clone()?)) - } - pub fn child(&self, name: &str) -> io::Result { - self.child_name(name) - } - fn child_name(&self, name: impl rustix::path::Arg) -> io::Result { - Ok(Self( - fs::openat( - &self.0, - name, - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::empty(), - )? - .into(), - )) - } - pub fn parent(&self, path: &str) -> io::Result<(Self, String)> { - self.parent_with_creation(path, false) - } - /// Prepare a destination without following symlinks in existing or raced-in parents. - pub fn ensure_parent(&self, path: &str) -> io::Result<(Self, String)> { - self.parent_with_creation(path, true) - } - fn parent_with_creation(&self, path: &str, create_missing: bool) -> io::Result<(Self, String)> { - let mut parts = path.split('/').collect::>(); - let name = parts.pop().unwrap_or("").to_owned(); - let mut dir = self.clone_dir()?; - for part in parts { - dir = match dir.child(part) { - Ok(child) => child, - Err(error) if create_missing && error.kind() == io::ErrorKind::NotFound => { - let child = match dir.mkdir(part, false) { - Ok(child) => child, - // Another creator may win the mkdir race. Reopen with the same - // directory-only, no-follow checks used for existing parents. - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { - dir.child(part)? - } - Err(error) => return Err(error), - }; - dir.sync()?; - child - } - Err(error) => return Err(error), - }; - } - Ok((dir, if name.is_empty() { ".".into() } else { name })) - } - pub fn open(&self, path: &str) -> io::Result { - self.open_kind(path, false) - } - pub fn metadata(&self, path: &str) -> io::Result { - self.open_kind(path, true) - } - /// Check the same existing-file write permission as a normal write, without - /// truncation, and without following filesystem aliases outside the scope. - pub fn writable_file(&self, path: &str) -> io::Result { - let (dir, name) = self.parent(path)?; - let file: File = fs::openat( - &dir.0, - &name, - OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC, - Mode::empty(), - )? - .into(); - if !file.metadata()?.is_file() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "write requires a regular file", - )); - } - Ok(file) - } - fn open_kind(&self, path: &str, metadata_only: bool) -> io::Result { - let (dir, name) = self.parent(path)?; - #[cfg(target_os = "linux")] - let access = if metadata_only { - OFlags::PATH - } else { - OFlags::RDONLY - }; - #[cfg(target_os = "macos")] - let access = if metadata_only { - // Rustix does not name the macOS metadata-only flag. - OFlags::from_bits_retain(libc::O_EVTONLY as u32) - } else { - OFlags::RDONLY - }; - let file = fs::openat( - &dir.0, - &name, - access | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC, - Mode::empty(), - )? - .into(); - observe(&file)?; - Ok(file) - } - /// Inspect replacement targets without requiring read permission on their content. - pub fn target_exists(&self, name: &str) -> io::Result { - let stat = match fs::statat(&self.0, name, AtFlags::SYMLINK_NOFOLLOW) { - Ok(stat) => stat, - Err(rustix::io::Errno::NOENT) => return Ok(false), - Err(error) => return Err(error.into()), - }; - if !matches!( - FileType::from_raw_mode(stat.st_mode), - FileType::Directory | FileType::RegularFile - ) { - return Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "target is a symlink or special file", - )); - } - Ok(true) - } - pub fn names(file: &File, limit: usize) -> io::Result> { - let mut names = Vec::new(); - for name in entries(file)? { - if names.len() >= limit { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "directory entry limit exceeded", - )); - } - names.push( - name?.into_string().map_err(|_| { - io::Error::new(io::ErrorKind::Unsupported, "non-UTF-8 filename") - })?, - ); - } - names.sort(); - Ok(names) - } - pub fn mkdir(&self, name: &str, private: bool) -> io::Result { - fs::mkdirat( - &self.0, - name, - Mode::from_raw_mode(if private { 0o700 } else { 0o755 }), - )?; - self.child(name) - } - pub fn create(&self, path: &str) -> io::Result { - let (dir, name) = self.parent(path)?; - Ok(fs::openat( - &dir.0, - &name, - OFlags::RDWR | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, - Mode::RUSR | Mode::WUSR, - )? - .into()) - } - pub fn publish(&self, stage: &Self, target: &str, replace: bool) -> io::Result { - // Rustix handles flagged renames on both Linux libc variants and macOS. - let rename = |exchange: bool| { - fs::renameat_with( - &stage.0, - "tree", - &self.0, - target, - if exchange { - fs::RenameFlags::EXCHANGE - } else { - fs::RenameFlags::NOREPLACE - }, - ) - .map_err(io::Error::from) - }; - match rename(false) { - Ok(()) => Ok(false), - Err(error) if replace && error.kind() == io::ErrorKind::AlreadyExists => { - rename(true)?; - Ok(true) - } - Err(error) => Err(error), - } - } - - /// A normal rename atomically refuses to replace a directory with a file, - /// including a directory created after the caller's destination check. - pub fn publish_file(&self, stage: &Self, target: &str) -> io::Result<()> { - fs::renameat(&stage.0, "tree", &self.0, target).map_err(io::Error::from) - } - pub fn remove_tree(&self, name: &str) -> io::Result<()> { - struct Frame { - name: CString, - directory: Directory, - entries: Entries, - } - let name = c(name)?; - let unlink = |parent: &Directory, name: &CStr, flags: AtFlags| -> io::Result<()> { - Ok(fs::unlinkat(&parent.0, name, flags)?) - }; - let directory = match self.child_name(&name) { - Ok(dir) => dir, - Err(_) => return unlink(self, &name, AtFlags::empty()), - }; - let mut stack = vec![Frame { - name, - entries: entries(&directory.0)?, - directory, - }]; - while let Some(frame) = stack.last_mut() { - if let Some(name) = frame.entries.next() { - let name = name?; - match frame.directory.child_name(&name) { - Ok(directory) => { - if stack.len() >= 256 { - return Err(io::Error::new( - io::ErrorKind::Unsupported, - "retirement cleanup depth limit exceeded", - )); - } - stack.push(Frame { - name, - entries: entries(&directory.0)?, - directory, - }); - } - Err(_) => unlink(&frame.directory, &name, AtFlags::empty())?, - } - } else { - let frame = stack.pop().unwrap(); - let parent = stack.last().map_or(self, |frame| &frame.directory); - unlink(parent, &frame.name, AtFlags::REMOVEDIR)?; - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn file_publication_cannot_replace_a_directory() { - let root = tempfile::tempdir().unwrap(); - let parent = Directory::anchor(root.path()).unwrap(); - let stage = parent.mkdir("staging", true).unwrap(); - std::fs::write(root.path().join("staging/tree"), b"new bytes").unwrap(); - // The file was valid when checked, but another actor replaced it by a - // nonempty directory before publication. - std::fs::write(root.path().join("target"), b"old bytes").unwrap(); - parent.writable_file("target").unwrap(); - std::fs::remove_file(root.path().join("target")).unwrap(); - std::fs::create_dir(root.path().join("target")).unwrap(); - std::fs::write(root.path().join("target/keep"), b"keep").unwrap(); - assert!(parent.publish_file(&stage, "target").is_err()); - assert_eq!( - std::fs::read(root.path().join("target/keep")).unwrap(), - b"keep" - ); - assert_eq!( - std::fs::read(root.path().join("staging/tree")).unwrap(), - b"new bytes" - ); - std::fs::remove_dir_all(root.path().join("target")).unwrap(); - std::fs::write(root.path().join("target"), b"old bytes").unwrap(); - parent.publish_file(&stage, "target").unwrap(); - assert_eq!( - std::fs::read(root.path().join("target")).unwrap(), - b"new bytes" - ); - } - - #[test] - fn directory_enumerations_have_independent_offsets_and_enforce_limits() { - let root = tempfile::tempdir().unwrap(); - for name in ["a", "b", "c"] { - std::fs::write(root.path().join(name), name).unwrap(); - } - let dir = Directory::anchor(root.path()).unwrap(); - let mut first = entries(&dir.0).unwrap(); - let mut observed = vec![first.next().unwrap().unwrap().into_string().unwrap()]; - assert_eq!(Directory::names(&dir.0, 3).unwrap(), ["a", "b", "c"]); - observed.extend(first.map(|name| name.unwrap().into_string().unwrap())); - observed.sort(); - assert_eq!(observed, ["a", "b", "c"]); - assert_eq!( - Directory::names(&dir.0, 2).unwrap_err().kind(), - io::ErrorKind::InvalidInput - ); - assert_eq!(Directory::names(&dir.0, 3).unwrap(), ["a", "b", "c"]); - } - - #[test] - fn cleanup_removes_nested_trees_without_following_symlinks() { - use std::os::unix::fs::symlink; - - let root = tempfile::tempdir().unwrap(); - let outside = tempfile::tempdir().unwrap(); - std::fs::write(outside.path().join("keep"), "untouched").unwrap(); - let dir = Directory::anchor(root.path()).unwrap(); - let retired = dir.mkdir("retired", true).unwrap(); - retired.mkdir("nested", true).unwrap(); - std::fs::write(root.path().join("retired/nested/file"), "content").unwrap(); - symlink(outside.path(), root.path().join("retired/link")).unwrap(); - symlink("missing", root.path().join("retired/dangling")).unwrap(); - dir.remove_tree("retired").unwrap(); - assert!(!root.path().join("retired").exists()); - assert_eq!( - std::fs::read_to_string(outside.path().join("keep")).unwrap(), - "untouched" - ); - } - - // macOS filesystems reject invalid UTF-8 at creation time. - #[cfg(target_os = "linux")] - #[test] - fn cleanup_handles_non_utf8_names() { - use std::os::unix::ffi::OsStrExt; - - let root = tempfile::tempdir().unwrap(); - let dir = Directory::anchor(root.path()).unwrap(); - dir.mkdir("retired", true).unwrap(); - let raw_path = root - .path() - .join("retired") - .join(std::ffi::OsStr::from_bytes(b"raw-\xff")); - std::fs::create_dir(&raw_path).unwrap(); - std::fs::write( - raw_path.join(std::ffi::OsStr::from_bytes(b"file-\xfe")), - "data", - ) - .unwrap(); - dir.remove_tree("retired").unwrap(); - assert!(!root.path().join("retired").exists()); - } - - #[test] - fn publication_refuses_overwrite_and_exchanges_replaced_files_and_trees() { - for directory in [false, true] { - let root = tempfile::tempdir().unwrap(); - let parent = Directory::anchor(root.path()).unwrap(); - let stage = parent.mkdir("stage", true).unwrap(); - let staged = root.path().join("stage/tree"); - let target = root.path().join("target"); - let write = |path: &Path, bytes: &[u8]| { - if directory { - std::fs::create_dir(path).unwrap(); - std::fs::write(path.join("content"), bytes).unwrap(); - } else { - std::fs::write(path, bytes).unwrap(); - } - }; - let read = |path: &Path| { - std::fs::read(if directory { - path.join("content") - } else { - path.to_owned() - }) - .unwrap() - }; - - write(&staged, b"original"); - assert!(!parent.publish(&stage, "target", false).unwrap()); - assert_eq!(read(&target), b"original"); - assert!(!staged.exists()); - - write(&staged, b"replacement"); - let error = parent.publish(&stage, "target", false).unwrap_err(); - assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); - assert_eq!(read(&target), b"original"); - assert_eq!(read(&staged), b"replacement"); - - assert!(parent.publish(&stage, "target", true).unwrap()); - assert_eq!(read(&target), b"replacement"); - // Exchange retains the old target in staging for later cleanup. - assert_eq!(read(&staged), b"original"); - } - } -} diff --git a/crates/environment-daemon/src/filesystem/confinement.rs b/crates/environment-daemon/src/filesystem/confinement.rs new file mode 100644 index 000000000..b4fe081c9 --- /dev/null +++ b/crates/environment-daemon/src/filesystem/confinement.rs @@ -0,0 +1,721 @@ +//! The one confinement rule of every data-plane path, a command's `cwd` +//! included. +//! +//! A wire path that is neither the filesystem root nor below it is +//! `permissionDenied` (`outsideRoot`). Otherwise the root prefix is stripped +//! and the remaining segments are resolved one at a time beneath the root's +//! directory descriptor. A symlink is followed only while its target stays +//! inside the root; a link that leaves it is refused the same way. A path +//! that names the daemon state directory, or passes through or below it, is +//! `permissionDenied` (`daemonState`). +//! +//! The state directory is recognized by its identity (device and inode), +//! never by name: every entry a lookup meets is compared with it, so another +//! spelling on a case-insensitive filesystem, a bind mount or a link cannot +//! reach it. + +use std::{ + collections::VecDeque, + ffi::{OsStr, OsString}, + os::unix::ffi::OsStrExt, + path::{Path, PathBuf}, +}; + +use environment_protocol::{ + error::{PermissionReason, PreconditionReason, ProtocolError}, + paths::EnvironmentPath, +}; + +use super::{ + io_error, + os::{DEFAULT_DIRECTORY_MODE, Directory, EntryStat, PRIVATE_DIRECTORY_MODE, SELF}, +}; + +/// Most symlinks one resolution follows before it is a loop. +const MAX_SYMLINK_FOLLOWS: u32 = 40; + +/// The filesystem root, its directory descriptor and the daemon state +/// directory. Built once per daemon and shared by every manager. +#[derive(Debug)] +pub struct Confinement { + root: PathBuf, + root_path: EnvironmentPath, + root_directory: Directory, + state_dir: PathBuf, + /// Device and inode of the state directory. + state_identity: Identity, + /// Device and inode of each of its ancestors, its parent first. + state_ancestors: Vec, +} + +/// Device and inode of an entry. +pub(crate) type Identity = (u64, u64); + +/// How one lookup treats the final entry and missing entries. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Lookup { + /// Follow a symlink in the final position (inside the root only). + pub follow_final: bool, + /// A missing final entry resolves with `stat: None` instead of + /// `notFound`. + pub allow_missing: bool, + /// Create missing intermediate directories. + pub create_parents: bool, + /// Refuse every symlink, final or not: re-opening a path recorded as + /// canonical, where a link means the tree changed. + pub no_symlinks: bool, +} + +impl Lookup { + pub const FOLLOW: Self = Self { + follow_final: true, + allow_missing: false, + create_parents: false, + no_symlinks: false, + }; + pub const NO_FOLLOW: Self = Self { + follow_final: false, + allow_missing: false, + create_parents: false, + no_symlinks: false, + }; + + pub fn allowing_missing(mut self) -> Self { + self.allow_missing = true; + self + } + + pub fn creating_parents(mut self) -> Self { + self.create_parents = true; + self + } +} + +/// A resolved entry: the directory that holds it and its name there. Every +/// later operation names the entry through `parent` and `name`, so a link +/// swapped in after resolution is never followed. +#[derive(Debug)] +pub(crate) struct Resolved { + pub parent: Directory, + /// `.` for the filesystem root itself. + pub name: OsString, + /// Canonical segments below the root; empty for the root. + pub segments: Vec, + /// `lstat` of the final entry; `None` when it is missing and the lookup + /// allowed that. + pub stat: Option, + /// Canonical segments of the directories this lookup created + /// (`creating_parents`), outermost first. + pub created: Vec>, +} + +impl Resolved { + pub fn is_root(&self) -> bool { + self.segments.is_empty() + } + + /// Open the resolved directory. + pub fn open_directory(&self) -> std::io::Result { + if self.is_root() { + self.parent.try_clone() + } else { + self.parent.child(&self.name) + } + } +} + +impl Confinement { + /// `root` is canonicalized and opened. `state_dir` is created (private) + /// when missing and canonicalized; it must not be the root itself. + pub fn new(root: PathBuf, state_dir: PathBuf) -> Result { + let root = root + .canonicalize() + .map_err(|error| io_error(error, root.display()))?; + let root_path = root + .to_str() + .and_then(|root| EnvironmentPath::new(root).ok()) + .ok_or_else(|| { + ProtocolError::internal(format!( + "filesystem root {} is not a valid environment path", + root.display() + )) + })?; + let root_directory = + Directory::open_host(&root).map_err(|error| io_error(error, root.display()))?; + if !state_dir.exists() { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new() + .recursive(true) + .mode(PRIVATE_DIRECTORY_MODE) + .create(&state_dir) + .map_err(|error| io_error(error, state_dir.display()))?; + } + let state_dir = state_dir + .canonicalize() + .map_err(|error| io_error(error, state_dir.display()))?; + let identity_of = |path: &Path| { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(path) + .map(|metadata| (metadata.dev(), metadata.ino())) + .map_err(|error| io_error(error, path.display())) + }; + let state_identity = identity_of(&state_dir)?; + let state_ancestors = state_dir + .ancestors() + .skip(1) + .map(identity_of) + .collect::, _>>()?; + let root_identity = EntryStat::of_directory(&root_directory) + .map_err(|error| io_error(error, root.display()))? + .identity(); + if root_identity == state_identity { + return Err(ProtocolError::internal( + "the daemon state directory must not be the filesystem root", + )); + } + Ok(Self { + root, + root_path, + root_directory, + state_dir, + state_identity, + state_ancestors, + }) + } + + pub fn root(&self) -> &Path { + &self.root + } + + pub fn state_dir(&self) -> &Path { + &self.state_dir + } + + /// The root as a wire path. + pub fn root_path(&self) -> &EnvironmentPath { + &self.root_path + } + + /// The confinement rule: the resolved host path of the final target, + /// following in-root symlinks. Missing entries are `notFound`. + pub fn resolve(&self, path: &EnvironmentPath) -> Result { + let resolved = self.lookup(path, Lookup::FOLLOW)?; + Ok(self.host_path(&resolved.segments)) + } + + /// [`Self::resolve`], then require a directory + /// (`failedPrecondition` `notDirectory` otherwise). + pub fn resolve_directory(&self, path: &EnvironmentPath) -> Result { + let resolved = self.lookup(path, Lookup::FOLLOW)?; + if !resolved.stat.is_some_and(|stat| stat.is_dir()) { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{path} is not a directory"), + )); + } + Ok(self.host_path(&resolved.segments)) + } + + /// Resolve a wire path under the rule. + pub(crate) fn lookup( + &self, + path: &EnvironmentPath, + lookup: Lookup, + ) -> Result { + let Some(relative) = path.relative_to(&self.root_path) else { + return Err(outside_root(path)); + }; + let components = relative.segments().map(OsString::from).collect(); + self.walk(components, lookup, path) + } + + /// Resolve canonical segments below the root plus a trailing name, + /// following a symlink in final position: how a walker follows a link + /// it met at `segments/name`. + pub(crate) fn lookup_link( + &self, + segments: &[OsString], + name: &OsStr, + subject: &dyn std::fmt::Display, + ) -> Result { + let mut components = segments.to_vec(); + components.push(name.to_owned()); + self.walk(components.into(), Lookup::FOLLOW, subject) + } + + /// Re-open canonical segments recorded earlier. Any symlink on the way + /// means the tree changed: it fails with `ELOOP`'s mapping. + pub(crate) fn reopen( + &self, + segments: &[OsString], + subject: &dyn std::fmt::Display, + ) -> Result { + let lookup = Lookup { + no_symlinks: true, + ..Lookup::NO_FOLLOW + }; + self.walk(segments.iter().cloned().collect(), lookup, subject) + } + + /// The host path of canonical segments. + pub(crate) fn host_path(&self, segments: &[OsString]) -> PathBuf { + let mut path = self.root.clone(); + path.extend(segments); + path + } + + /// The wire path of canonical segments, when they are UTF-8. + pub(crate) fn wire_path(&self, segments: &[OsString]) -> Option { + let mut path = self.root_path.as_str().trim_end_matches('/').to_owned(); + for segment in segments { + path.push('/'); + path.push_str(segment.to_str()?); + } + if path.is_empty() { + path.push('/'); + } + EnvironmentPath::new(path).ok() + } + + /// The entry is the state directory. + pub(crate) fn is_state(&self, stat: &EntryStat) -> bool { + stat.identity() == self.state_identity + } + + /// The entry is the state directory or one of its ancestors: removing, + /// replacing or capturing it would address the state directory. + pub(crate) fn contains_state(&self, stat: &EntryStat) -> bool { + self.is_state(stat) || self.state_ancestors.contains(&stat.identity()) + } + + /// The directory is the state directory's parent, so one of its + /// entries may be the state directory. + pub(crate) fn holds_state(&self, directory: &EntryStat) -> bool { + self.state_ancestors.first() == Some(&directory.identity()) + } + + /// Refuse an operation whose subject contains the state directory. + pub(crate) fn refuse_containing_state( + &self, + resolved: &Resolved, + subject: &dyn std::fmt::Display, + ) -> Result<(), ProtocolError> { + if resolved.stat.is_some_and(|stat| self.contains_state(&stat)) { + return Err(daemon_state(subject)); + } + Ok(()) + } + + /// Remove directories a lookup created (see [`Resolved::created`]) + /// that are still empty, innermost first. Best effort. + pub(crate) fn remove_created(&self, created: &[Vec]) { + for segments in created.iter().rev() { + let Some((name, parent)) = segments.split_last() else { + continue; + }; + let Ok(resolved) = self.reopen(parent, &"created directory") else { + return; + }; + let Ok(directory) = resolved.open_directory() else { + return; + }; + if directory.rmdir(name).is_err() { + return; + } + } + } + + fn walk( + &self, + mut pending: VecDeque, + lookup: Lookup, + subject: &dyn std::fmt::Display, + ) -> Result { + let io = |error| io_error(error, subject); + let mut stack = vec![self.root_directory.try_clone().map_err(io)?]; + let mut segments: Vec = Vec::new(); + let mut created = Vec::new(); + let mut follows = 0; + while let Some(name) = pending.pop_front() { + if name.is_empty() || name == SELF { + continue; + } + if name == ".." { + if segments.pop().is_none() { + return Err(outside_root(subject)); + } + stack.pop(); + continue; + } + let last = pending.is_empty(); + let mut candidate = segments.clone(); + candidate.push(name.clone()); + let directory = stack.last().expect("the root stays on the stack"); + let stat = match directory.stat(&name) { + Ok(stat) => stat, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if last && lookup.allow_missing { + return Ok(Resolved { + parent: directory.try_clone().map_err(io)?, + name, + segments: candidate, + stat: None, + created, + }); + } + if !last && lookup.create_parents { + match directory.mkdir(&name, DEFAULT_DIRECTORY_MODE) { + Ok(()) => created.push(candidate), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(io(error)), + } + directory.sync().map_err(io)?; + pending.push_front(name); + continue; + } + return Err(io(error)); + } + Err(error) => return Err(io(error)), + }; + if self.is_state(&stat) { + return Err(daemon_state(subject)); + } + if stat.kind == environment_protocol::data::fs::FileKind::Symlink + && (!last || lookup.follow_final) + { + if lookup.no_symlinks { + return Err(ProtocolError::failed_precondition( + PreconditionReason::SymlinkLoop, + format!("{subject}: a symlink replaced a recorded entry"), + )); + } + follows += 1; + if follows > MAX_SYMLINK_FOLLOWS { + return Err(ProtocolError::failed_precondition( + PreconditionReason::SymlinkLoop, + format!("{subject}: too many levels of symbolic links"), + )); + } + let target = directory.read_link(&name).map_err(io)?; + let target = Path::new(&target); + let components: Vec = if target.is_absolute() { + let Ok(rest) = target.strip_prefix(&self.root) else { + return Err(outside_root(subject)); + }; + stack.truncate(1); + segments.clear(); + raw_components(rest) + } else { + raw_components(target) + }; + for component in components.into_iter().rev() { + pending.push_front(component); + } + continue; + } + if last { + return Ok(Resolved { + parent: directory.try_clone().map_err(io)?, + name, + segments: candidate, + stat: Some(stat), + created, + }); + } + if !stat.is_dir() { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{subject}: a path component is not a directory"), + )); + } + let child = directory.child(&name).map_err(io)?; + stack.push(child); + segments.push(name); + } + // The path ended on a directory already open: the root, or a + // directory reached through `..` or a link ending in one. + let current = stack.pop().expect("the root stays on the stack"); + let stat = EntryStat::of_directory(¤t).map_err(io)?; + if self.is_state(&stat) { + return Err(daemon_state(subject)); + } + match (stack.pop(), segments.last()) { + (Some(parent), Some(name)) => Ok(Resolved { + parent, + name: name.clone(), + segments, + stat: Some(stat), + created, + }), + _ => Ok(Resolved { + parent: current, + name: OsString::from(SELF), + segments, + stat: Some(stat), + created, + }), + } + } +} + +/// Components of a link target, keeping `.` and `..` for the walk to apply +/// against the open directories. +fn raw_components(path: &Path) -> Vec { + path.as_os_str() + .as_bytes() + .split(|byte| *byte == b'/') + .filter(|segment| !segment.is_empty()) + .map(|segment| OsStr::from_bytes(segment).to_owned()) + .collect() +} + +pub(crate) fn outside_root(subject: &dyn std::fmt::Display) -> ProtocolError { + ProtocolError::permission_denied( + Some(PermissionReason::OutsideRoot), + format!("{subject} is outside the filesystem root"), + ) +} + +pub(crate) fn daemon_state(subject: &dyn std::fmt::Display) -> ProtocolError { + ProtocolError::permission_denied( + Some(PermissionReason::DaemonState), + format!("{subject} addresses the daemon state directory"), + ) +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::symlink; + + use environment_protocol::error::{ErrorData, ProtocolErrorKind}; + + use super::*; + + struct Fixture { + _temp: tempfile::TempDir, + outside: PathBuf, + root: PathBuf, + confinement: Confinement, + } + + fn fixture() -> Fixture { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("root"); + let outside = base.join("outside"); + std::fs::create_dir_all(root.join("src/deep")).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + std::fs::write(root.join("src/lib.rs"), "lib").unwrap(); + std::fs::write(outside.join("secret"), "secret").unwrap(); + let confinement = Confinement::new(root.clone(), root.join(".state")).unwrap(); + Fixture { + _temp: temp, + outside, + root, + confinement, + } + } + + fn path(root: &Path, relative: &str) -> EnvironmentPath { + EnvironmentPath::new(format!("{}/{relative}", root.display())).unwrap() + } + + fn permission_reason(error: &ProtocolError) -> Option { + match error.data { + ErrorData::PermissionDenied { reason } => reason, + _ => None, + } + } + + #[test] + fn paths_outside_the_root_are_refused_before_any_lookup() { + let fixture = fixture(); + let error = fixture + .confinement + .resolve( + &EnvironmentPath::new(fixture.outside.join("secret").to_str().unwrap()).unwrap(), + ) + .unwrap_err(); + assert_eq!( + permission_reason(&error), + Some(PermissionReason::OutsideRoot) + ); + } + + #[test] + fn in_root_links_are_followed_and_escaping_links_refused() { + let fixture = fixture(); + symlink("src", fixture.root.join("inside")).unwrap(); + // Relative to the link's directory: up to the root, then back down. + symlink("../../src/lib.rs", fixture.root.join("src/deep/up")).unwrap(); + symlink(&fixture.outside, fixture.root.join("absolute-escape")).unwrap(); + symlink("../outside/secret", fixture.root.join("relative-escape")).unwrap(); + symlink( + fixture.root.join("src"), + fixture.root.join("absolute-inside"), + ) + .unwrap(); + + assert_eq!( + fixture + .confinement + .resolve(&path(&fixture.root, "inside/lib.rs")) + .unwrap(), + fixture.root.join("src/lib.rs") + ); + assert_eq!( + fixture + .confinement + .resolve(&path(&fixture.root, "src/deep/up")) + .unwrap(), + fixture.root.join("src/lib.rs") + ); + assert_eq!( + fixture + .confinement + .resolve_directory(&path(&fixture.root, "absolute-inside")) + .unwrap(), + fixture.root.join("src") + ); + for escape in [ + "absolute-escape/secret", + "relative-escape", + "absolute-escape", + ] { + let error = fixture + .confinement + .resolve(&path(&fixture.root, escape)) + .unwrap_err(); + assert_eq!( + permission_reason(&error), + Some(PermissionReason::OutsideRoot), + "{escape}" + ); + } + } + + #[test] + fn the_state_directory_is_never_addressed() { + let fixture = fixture(); + std::fs::write(fixture.root.join(".state/daemon-key"), "key").unwrap(); + symlink(".state", fixture.root.join("state-link")).unwrap(); + for subject in [".state", ".state/daemon-key", "state-link/daemon-key"] { + let error = fixture + .confinement + .resolve(&path(&fixture.root, subject)) + .unwrap_err(); + assert_eq!( + permission_reason(&error), + Some(PermissionReason::DaemonState), + "{subject}" + ); + } + let root = fixture + .confinement + .lookup(fixture.confinement.root_path(), Lookup::NO_FOLLOW) + .unwrap(); + assert!(root.is_root()); + assert!(fixture.confinement.contains_state(&root.stat.unwrap())); + let src = fixture + .confinement + .lookup(&path(&fixture.root, "src"), Lookup::NO_FOLLOW) + .unwrap(); + assert!(!fixture.confinement.contains_state(&src.stat.unwrap())); + } + + #[test] + fn the_state_directory_is_recognized_by_identity_not_by_name() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap().join("root"); + std::fs::create_dir_all(root.join("local/state")).unwrap(); + let confinement = Confinement::new(root.clone(), root.join("local/state/envd")).unwrap(); + std::fs::write(root.join("local/state/envd/daemon-key"), "key").unwrap(); + // Another name for the same directories: no recorded spelling of + // the state directory matches it any more. + std::fs::rename(root.join("local"), root.join("moved")).unwrap(); + // Where the filesystem folds case (APFS), other spellings reach the + // same entries; elsewhere they are missing. + for subject in [ + "moved/state/envd/daemon-key", + "moved/state/envd", + "MOVED/STATE/ENVD/daemon-key", + ] { + match confinement.resolve(&path(&root, subject)) { + Err(error) if error.kind() == ProtocolErrorKind::NotFound => { + assert!(subject.starts_with("MOVED"), "{subject}"); + } + Err(error) => assert_eq!( + permission_reason(&error), + Some(PermissionReason::DaemonState), + "{subject}" + ), + Ok(resolved) => panic!("{subject} resolved to {}", resolved.display()), + } + } + for ancestor in ["moved", "moved/state"] { + let resolved = confinement + .lookup(&path(&root, ancestor), Lookup::NO_FOLLOW) + .unwrap(); + assert!( + confinement + .refuse_containing_state(&resolved, &ancestor) + .is_err(), + "{ancestor}" + ); + assert_eq!( + confinement.holds_state(&resolved.stat.unwrap()), + ancestor == "moved/state" + ); + } + } + + #[test] + fn loops_and_missing_entries_are_typed() { + let fixture = fixture(); + symlink("b", fixture.root.join("a")).unwrap(); + symlink("a", fixture.root.join("b")).unwrap(); + let error = fixture + .confinement + .resolve(&path(&fixture.root, "a")) + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::SymlinkLoop + } + )); + let error = fixture + .confinement + .resolve(&path(&fixture.root, "missing")) + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + let error = fixture + .confinement + .resolve_directory(&path(&fixture.root, "src/lib.rs")) + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::NotDirectory + } + )); + } + + #[test] + fn dangling_in_root_links_resolve_to_their_missing_target() { + let fixture = fixture(); + symlink("src/new.rs", fixture.root.join("dangling")).unwrap(); + let resolved = fixture + .confinement + .lookup( + &path(&fixture.root, "dangling"), + Lookup::FOLLOW.allowing_missing(), + ) + .unwrap(); + assert!(resolved.stat.is_none()); + assert_eq!( + fixture.confinement.host_path(&resolved.segments), + fixture.root.join("src/new.rs") + ); + } +} diff --git a/crates/environment-daemon/src/filesystem/os.rs b/crates/environment-daemon/src/filesystem/os.rs new file mode 100644 index 000000000..9119d35dd --- /dev/null +++ b/crates/environment-daemon/src/filesystem/os.rs @@ -0,0 +1,486 @@ +//! Descriptor-relative filesystem primitives. Every operation names an entry +//! by a directory descriptor and one name, never by a host path, so nothing +//! here follows a symlink implicitly. Wire paths, inventories and receipts +//! never contain descriptors or inode numbers; only [`FileVersion`] tokens +//! are derived from them. + +use std::{ + ffi::{OsStr, OsString}, + fs::File, + io::{self, Read}, + os::{ + fd::OwnedFd, + unix::ffi::{OsStrExt, OsStringExt}, + }, + path::Path, +}; + +use environment_protocol::data::fs::{FileKind, FileVersion}; +use rustix::fs::{self as rfs, AtFlags, FileType, Mode, OFlags, RenameFlags}; + +/// An open directory. Children are opened beneath it without following +/// symlinks. +#[derive(Debug)] +pub(crate) struct Directory(OwnedFd); + +/// The entry a directory descriptor itself names. +pub(crate) const SELF: &str = "."; + +impl Directory { + /// Open an administrator-configured directory by its host path. Only the + /// filesystem root and the transfer journal are opened this way; every + /// wire path is resolved beneath the root's descriptor. + pub fn open_host(path: &Path) -> io::Result { + Ok(Self(rfs::open( + path, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + )?)) + } + + pub fn try_clone(&self) -> io::Result { + Ok(Self(self.0.try_clone()?)) + } + + /// Open the child directory `name`; a symlink or non-directory fails + /// (`ELOOP` or `ENOTDIR`). + pub fn child(&self, name: &OsStr) -> io::Result { + Ok(Self(rfs::openat( + &self.0, + name, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?)) + } + + /// `lstat` of `name` (`.` for the directory itself). + pub fn stat(&self, name: &OsStr) -> io::Result { + Ok(EntryStat::from_stat(&rfs::statat( + &self.0, + name, + AtFlags::SYMLINK_NOFOLLOW, + )?)) + } + + /// `lstat` plus the birth time where the filesystem records one. + pub fn stat_with_birth(&self, name: &OsStr) -> io::Result { + let stat = rfs::statat(&self.0, name, AtFlags::SYMLINK_NOFOLLOW)?; + let mut entry = EntryStat::from_stat(&stat); + entry.birth_ms = birth_ms(&self.0, name, &stat); + Ok(entry) + } + + pub fn read_link(&self, name: &OsStr) -> io::Result { + let target = rfs::readlinkat(&self.0, name, Vec::new())?; + Ok(OsString::from_vec(target.into_bytes())) + } + + /// Open `name` for reading without following a symlink and without + /// blocking on a FIFO or device; callers `fstat` before reading. + pub fn open_read(&self, name: &OsStr) -> io::Result { + Ok(rfs::openat( + &self.0, + name, + OFlags::RDONLY | OFlags::NONBLOCK | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )? + .into()) + } + + /// Create `name` exclusively, never following a symlink. + pub fn create_file(&self, name: &OsStr, mode: u32) -> io::Result { + Ok(rfs::openat( + &self.0, + name, + OFlags::RDWR | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::from_raw_mode(mode as _), + )? + .into()) + } + + pub fn mkdir(&self, name: &OsStr, mode: u32) -> io::Result<()> { + Ok(rfs::mkdirat(&self.0, name, Mode::from_raw_mode(mode as _))?) + } + + pub fn unlink(&self, name: &OsStr) -> io::Result<()> { + Ok(rfs::unlinkat(&self.0, name, AtFlags::empty())?) + } + + pub fn rmdir(&self, name: &OsStr) -> io::Result<()> { + Ok(rfs::unlinkat(&self.0, name, AtFlags::REMOVEDIR)?) + } + + /// Rename `from` in `self` to `to` in `destination`. + pub fn rename( + &self, + from: &OsStr, + destination: &Directory, + to: &OsStr, + mode: RenameMode, + ) -> io::Result<()> { + match mode { + RenameMode::Replace => rfs::renameat(&self.0, from, &destination.0, to)?, + RenameMode::NoReplace => { + rfs::renameat_with(&self.0, from, &destination.0, to, RenameFlags::NOREPLACE)? + } + RenameMode::Exchange => { + rfs::renameat_with(&self.0, from, &destination.0, to, RenameFlags::EXCHANGE)? + } + } + Ok(()) + } + + /// Every name except `.` and `..`, unsorted. A separate open file + /// description is used, so enumerations never share an offset. + pub fn names(&self) -> io::Result> { + let mut names = Vec::new(); + for entry in self.entries()? { + names.push(entry?); + } + Ok(names) + } + + fn entries(&self) -> io::Result { + let fd = rfs::openat( + &self.0, + SELF, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + )?; + Ok(Entries(rfs::Dir::new(fd)?)) + } + + pub fn sync(&self) -> io::Result<()> { + Ok(rfs::fsync(&self.0)?) + } + + /// Remove `name` and everything below it without following symlinks. + /// Streams each directory, so a large tree is never collected at once. + pub fn remove_tree(&self, name: &OsStr) -> io::Result<()> { + struct Frame { + name: OsString, + directory: Directory, + entries: Entries, + } + let directory = match self.child(name) { + Ok(directory) => directory, + Err(error) if is_not_directory(&error) => return self.unlink(name), + Err(error) => return Err(error), + }; + let mut stack = vec![Frame { + name: name.to_owned(), + entries: directory.entries()?, + directory, + }]; + while let Some(frame) = stack.last_mut() { + if let Some(child) = frame.entries.next() { + let child = child?; + match frame.directory.child(&child) { + Ok(directory) => { + if stack.len() >= MAX_REMOVE_DEPTH { + return Err(io::Error::other("directory tree is too deep to remove")); + } + stack.push(Frame { + name: child, + entries: directory.entries()?, + directory, + }); + } + Err(error) if is_not_directory(&error) => frame.directory.unlink(&child)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } else { + let frame = stack.pop().expect("frame exists"); + let parent = stack.last().map_or(self, |frame| &frame.directory); + match parent.rmdir(&frame.name) { + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + result => result?, + } + } + } + Ok(()) + } +} + +const MAX_REMOVE_DEPTH: usize = 4096; + +/// How [`Directory::rename`] treats an existing destination. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RenameMode { + /// Replace a file (`rename(2)`). + Replace, + /// Fail with `EEXIST` when the destination exists. + NoReplace, + /// Swap source and destination atomically. + Exchange, +} + +/// `ELOOP` (a symlink met with `O_NOFOLLOW`) or `ENOTDIR`. +pub(crate) fn is_not_directory(error: &io::Error) -> bool { + matches!( + rustix::io::Errno::from_io_error(error), + Some(rustix::io::Errno::LOOP | rustix::io::Errno::NOTDIR) + ) +} + +struct Entries(rfs::Dir); + +impl Iterator for Entries { + type Item = io::Result; + + fn next(&mut self) -> Option { + self.0.find_map(|entry| match entry { + Ok(entry) => { + let name = entry.file_name().to_bytes(); + (name != b"." && name != b"..").then(|| Ok(OsString::from_vec(name.to_vec()))) + } + Err(error) => Some(Err(error.into())), + }) + } +} + +/// What `lstat` or `fstat` reported about one entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct EntryStat { + pub kind: FileKind, + pub mode: u32, + pub size: u64, + pub dev: u64, + pub ino: u64, + pub modified: (i64, i64), + pub changed: (i64, i64), + pub birth_ms: Option, +} + +impl EntryStat { + #[allow(clippy::unnecessary_cast)] + pub fn from_stat(stat: &rfs::Stat) -> Self { + let kind = match FileType::from_raw_mode(stat.st_mode as _) { + FileType::RegularFile => FileKind::File, + FileType::Directory => FileKind::Directory, + FileType::Symlink => FileKind::Symlink, + _ => FileKind::Other, + }; + Self { + kind, + mode: stat.st_mode as u32 & 0o7777, + size: stat.st_size as u64, + dev: stat.st_dev as u64, + ino: stat.st_ino as u64, + modified: (stat.st_mtime as i64, stat.st_mtime_nsec as i64), + changed: (stat.st_ctime as i64, stat.st_ctime_nsec as i64), + birth_ms: None, + } + } + + pub fn of_file(file: &File) -> io::Result { + Ok(Self::from_stat(&rfs::fstat(file)?)) + } + + pub fn of_directory(directory: &Directory) -> io::Result { + Ok(Self::from_stat(&rfs::fstat(&directory.0)?)) + } + + pub fn is_dir(&self) -> bool { + self.kind == FileKind::Directory + } + + pub fn is_file(&self) -> bool { + self.kind == FileKind::File + } + + pub fn executable(&self) -> bool { + self.mode & 0o111 != 0 + } + + pub fn modified_ms(&self) -> i64 { + to_ms(self.modified) + } + + /// Device and inode. + pub fn identity(&self) -> (u64, u64) { + (self.dev, self.ino) + } + + /// Same inode with the same size, modification and change times: the + /// content observed earlier is still the content. + pub fn unchanged(&self, other: &Self) -> bool { + self.dev == other.dev + && self.ino == other.ino + && self.size == other.size + && self.modified == other.modified + && self.changed == other.changed + } + + /// The opaque `version` token: identity, size and both timestamps. + pub fn version(&self) -> FileVersion { + FileVersion::new(format!( + "v1-{:x}-{:x}-{:x}-{:x}.{:x}-{:x}.{:x}", + self.dev, + self.ino, + self.size, + self.modified.0 as u64, + self.modified.1 as u64, + self.changed.0 as u64, + self.changed.1 as u64, + )) + .expect("version tokens are short and printable") + } +} + +fn to_ms((seconds, nanoseconds): (i64, i64)) -> i64 { + seconds + .saturating_mul(1000) + .saturating_add(nanoseconds / 1_000_000) +} + +#[cfg(target_os = "linux")] +fn birth_ms(directory: &OwnedFd, name: &OsStr, _: &rfs::Stat) -> Option { + let statx = rfs::statx( + directory, + name, + AtFlags::SYMLINK_NOFOLLOW, + rfs::StatxFlags::BTIME, + ) + .ok()?; + rfs::StatxFlags::from_bits_retain(statx.stx_mask) + .contains(rfs::StatxFlags::BTIME) + .then(|| to_ms((statx.stx_btime.tv_sec, statx.stx_btime.tv_nsec as i64))) +} + +#[cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd"))] +#[allow(clippy::unnecessary_cast)] +fn birth_ms(_: &OwnedFd, _: &OsStr, stat: &rfs::Stat) -> Option { + // Filesystems without birth times report zero or a negative value. + (stat.st_birthtime > 0) + .then(|| to_ms((stat.st_birthtime as i64, stat.st_birthtime_nsec as i64))) +} + +#[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "ios", + target_os = "freebsd" +)))] +fn birth_ms(_: &OwnedFd, _: &OsStr, _: &rfs::Stat) -> Option { + None +} + +/// Read at most `max` bytes from the current position. +pub(crate) fn read_up_to(file: &mut File, max: u64) -> io::Result> { + let mut data = Vec::with_capacity(max.min(1024 * 1024) as usize); + file.take(max).read_to_end(&mut data)?; + Ok(data) +} + +/// A name as UTF-8, or `None` when it is not representable on the wire. +pub(crate) fn utf8_name(name: &OsStr) -> Option<&str> { + std::str::from_utf8(name.as_bytes()).ok() +} + +/// Set the permission bits of an open file. +pub(crate) fn set_mode(file: &File, mode: u32) -> io::Result<()> { + Ok(rfs::fchmod(file, Mode::from_raw_mode(mode as _))?) +} + +/// The mode with an execute bit added wherever a read bit is set. +pub(crate) fn with_execute_bits(mode: u32) -> u32 { + mode | ((mode & 0o444) >> 2) +} + +/// Mode of a file the daemon creates. +pub(crate) const DEFAULT_FILE_MODE: u32 = 0o644; +/// Mode of a directory the daemon creates. +pub(crate) const DEFAULT_DIRECTORY_MODE: u32 = 0o755; +/// Mode of the daemon's own staging directories. +pub(crate) const PRIVATE_DIRECTORY_MODE: u32 = 0o700; +/// Mode of a staged file before its final mode is set. +pub(crate) const PRIVATE_FILE_MODE: u32 = 0o600; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn removal_streams_nested_trees_without_following_symlinks() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("keep"), "untouched").unwrap(); + std::fs::create_dir_all(root.path().join("retired/nested")).unwrap(); + std::fs::write(root.path().join("retired/nested/file"), "content").unwrap(); + symlink(outside.path(), root.path().join("retired/link")).unwrap(); + symlink("missing", root.path().join("retired/dangling")).unwrap(); + let directory = Directory::open_host(root.path()).unwrap(); + directory.remove_tree(OsStr::new("retired")).unwrap(); + assert!(!root.path().join("retired").exists()); + assert_eq!( + std::fs::read_to_string(outside.path().join("keep")).unwrap(), + "untouched" + ); + } + + #[test] + fn enumerations_have_independent_offsets() { + let root = tempfile::tempdir().unwrap(); + for name in ["a", "b", "c"] { + std::fs::write(root.path().join(name), name).unwrap(); + } + let directory = Directory::open_host(root.path()).unwrap(); + let mut first = directory.entries().unwrap(); + let mut seen = vec![first.next().unwrap().unwrap()]; + let mut all = directory.names().unwrap(); + all.sort(); + assert_eq!(all, ["a", "b", "c"]); + seen.extend(first.map(Result::unwrap)); + seen.sort(); + assert_eq!(seen, ["a", "b", "c"]); + } + + #[test] + fn rename_modes_refuse_replace_and_exchange() { + let root = tempfile::tempdir().unwrap(); + let directory = Directory::open_host(root.path()).unwrap(); + std::fs::write(root.path().join("a"), "a").unwrap(); + std::fs::write(root.path().join("b"), "b").unwrap(); + let error = directory + .rename( + OsStr::new("a"), + &directory, + OsStr::new("b"), + RenameMode::NoReplace, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + directory + .rename( + OsStr::new("a"), + &directory, + OsStr::new("b"), + RenameMode::Exchange, + ) + .unwrap(); + assert_eq!(std::fs::read(root.path().join("a")).unwrap(), b"b"); + assert_eq!(std::fs::read(root.path().join("b")).unwrap(), b"a"); + } + + #[test] + fn versions_change_with_content_identity() { + let root = tempfile::tempdir().unwrap(); + let directory = Directory::open_host(root.path()).unwrap(); + std::fs::write(root.path().join("file"), "one").unwrap(); + let before = directory.stat(OsStr::new("file")).unwrap(); + assert_eq!( + before.version(), + directory.stat(OsStr::new("file")).unwrap().version() + ); + std::fs::write(root.path().join("staged"), "two!").unwrap(); + std::fs::rename(root.path().join("staged"), root.path().join("file")).unwrap(); + assert_ne!( + before.version(), + directory.stat(OsStr::new("file")).unwrap().version() + ); + } +} diff --git a/crates/environment-daemon/src/filesystem/scan.rs b/crates/environment-daemon/src/filesystem/scan.rs index 7d060f256..48eecb872 100644 --- a/crates/environment-daemon/src/filesystem/scan.rs +++ b/crates/environment-daemon/src/filesystem/scan.rs @@ -1,312 +1,546 @@ -//! Bounded endpoint-local observations through the shared scoped filesystem backend. -use super::backend::{ - self, Directory, Result, conflict, digest, error, invalid, io, relative, valid_path, +//! `fs/scan`: a bounded observation of several roots, each reported on its +//! own. A skipped entry never fails its root, and a failed root never hides +//! the others. + +use std::{ + sync::Arc, + time::{Duration, Instant}, }; -use environment_protocol::shared::EnvironmentPath; + use environment_protocol::{ - data::inventory::*, error::EnvironmentProtocolErrorCode as Code, shared::ByteChunk, + bytes::ByteChunk, + data::{ + fs::TreeContent, + handshake::ScanCeilings, + scan::{ + ScanEntry, ScanFingerprint, ScanParams, ScanResponse, ScanRoot, ScanRootResult, + ScanSkip, ScanSkipReason, + }, + }, + error::{ErrorData, PreconditionReason, ProtocolError}, + limits::{MAX_LISTED_SKIPS, MAX_RESULT_BYTES}, + paths::{ContentDigest, TreePath}, }; use sha2::{Digest, Sha256}; -use std::{collections::BTreeSet, path::PathBuf}; -use std::{ - io::Read, - path::Path, - time::{Duration, Instant}, +use tokio_util::sync::CancellationToken; + +use super::{ + confinement::{Confinement, Lookup}, + json_string_len, + os::{self, EntryStat}, + search::check_quota, + walk::{CompiledFilter, Event, LinkMode, Skip, Walker}, }; -pub fn scan(root: &Path, params: ScanParams) -> Result { - let ceiling = InventoryLimits::default(); - let limits = params.limits; - if limits.max_entries == 0 - || limits.max_entries > ceiling.max_entries - || limits.max_depth > ceiling.max_depth - || limits.max_file_bytes > ceiling.max_file_bytes - || limits.max_total_bytes > ceiling.max_total_bytes - || limits.max_manifest_bytes > ceiling.max_manifest_bytes - || limits.max_duration_ms == 0 - || limits.max_duration_ms > ceiling.max_duration_ms - || params.roots.len() > 32 - || params.include_patterns.len() > 32 - || params - .include_patterns - .iter() - .any(|p| p.len() > MAX_INVENTORY_PATH_BYTES) - { - return Err(invalid("scan quotas exceed daemon ceilings")); +/// Encoded overhead of one entry or skip besides its path and content. +const ENTRY_OVERHEAD: u64 = 96; + +pub(crate) fn check_scan( + params: &ScanParams, + ceilings: &ScanCeilings, +) -> Result<(), ProtocolError> { + let roots = params.roots.len() as u64; + if roots == 0 { + return Err(ProtocolError::invalid_field( + "roots", + "scan needs at least one root", + )); } - let patterns = params - .include_patterns - .iter() - .map(|p| glob::Pattern::new(p).map_err(|e| invalid(&e.to_string()))) - .collect::>>()?; - // Filename-only patterns select direct children, so unrelated subtrees do - // not consume scan budgets or cause failures for a nonrecursive query. - let direct_only = !params.include_patterns.is_empty() - && params - .include_patterns - .iter() - .all(|pattern| !pattern.contains('/') && !pattern.contains("**")); - let canonical_root = root.canonicalize().map_err(io)?; - let anchor = Directory::anchor(&canonical_root).map_err(io)?; - let mut response = ScanResponse { - fingerprint: None, - unchanged: false, - complete: true, - entries: vec![], - diagnostics: vec![], + if roots > u64::from(ceilings.max_roots) { + return Err(ProtocolError::above_ceiling( + "roots", + roots, + ceilings.max_roots.into(), + )); + } + let patterns = (params.filter.include.len() + params.filter.exclude.len()) as u64; + if patterns > u64::from(ceilings.max_patterns) { + return Err(ProtocolError::above_ceiling( + "filter", + patterns, + ceilings.max_patterns.into(), + )); + } + let limits = ¶ms.limits; + check_quota( + "limits.maxEntries", + limits.max_entries.into(), + ceilings.max_entries.into(), + )?; + if limits.max_file_bytes > ceilings.max_file_bytes { + return Err(ProtocolError::above_ceiling( + "limits.maxFileBytes", + limits.max_file_bytes, + ceilings.max_file_bytes, + )); + } + if limits.max_total_bytes > ceilings.max_total_bytes { + return Err(ProtocolError::above_ceiling( + "limits.maxTotalBytes", + limits.max_total_bytes, + ceilings.max_total_bytes, + )); + } + check_quota( + "limits.maxDurationMs", + limits.max_duration_ms, + ceilings.max_duration_ms, + ) +} + +/// Budgets shared by every root of one scan. +struct Totals { + content_bytes: u64, + result_bytes: u64, +} + +pub(crate) fn scan( + confinement: &Arc, + params: ScanParams, + cancel: &CancellationToken, +) -> Result { + let filter = CompiledFilter::compile(¶ms.filter, "filter")?; + let mut totals = Totals { + content_bytes: 0, + result_bytes: 0, }; - let start = Instant::now(); - let mut visited = 0u32; - let mut inspected = 0u64; - let mut response_bytes = 0u64; - let mut observations = Vec::new(); - let mut observation_bytes = 0u64; - for selected in ¶ms.roots { - let result = (|| -> Result<()> { - let selected_relative = relative(root, selected)?; - let selected_path = canonical_root.join(selected_relative); - // A missing configured root is a complete empty observation. A dangling - // link or a permission error is an incomplete observation. - match std::fs::symlink_metadata(&selected_path) { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - let mut ancestor = selected_path.parent(); - while let Some(path) = ancestor { - match std::fs::symlink_metadata(path) { - Ok(_) => { - let resolved = path.canonicalize().map_err(io)?; - if !resolved.starts_with(&canonical_root) { - return Err(error( - Code::Forbidden, - "root outside access scope", - )); - } - break; - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - ancestor = path.parent() - } - Err(e) => return Err(io(e)), - } - } - return Ok(()); - } - Err(e) => return Err(io(e)), - Ok(_) => {} + let mut roots = Vec::with_capacity(params.roots.len()); + for root in ¶ms.roots { + let result = match scan_root(confinement, ¶ms, root, &filter, &mut totals, cancel) { + Ok(result) => result, + Err(RootError::Cancelled) => return Err(ProtocolError::cancelled()), + Err(RootError::Failed(error)) => ScanRootResult::Failed { + path: root.path.clone(), + error, + }, + }; + roots.push(result); + } + Ok(ScanResponse { roots }) +} + +enum RootError { + Cancelled, + Failed(ProtocolError), +} + +impl From for RootError { + fn from(error: ProtocolError) -> Self { + Self::Failed(error) + } +} + +fn scan_root( + confinement: &Arc, + params: &ScanParams, + root: &ScanRoot, + filter: &CompiledFilter, + totals: &mut Totals, + cancel: &CancellationToken, +) -> Result { + let started = Instant::now(); + let deadline = Duration::from_millis(params.limits.max_duration_ms); + let resolved = match confinement.lookup(&root.path, Lookup::FOLLOW) { + Ok(resolved) => resolved, + Err(error) if matches!(error.data, ErrorData::NotFound { .. }) => { + return Ok(ScanRootResult::Missing { + path: root.path.clone(), + }); + } + Err(error) => return Err(error.into()), + }; + if !resolved.stat.is_some_and(|stat| stat.is_dir()) { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{} is not a directory", root.path), + ) + .into()); + } + let canonical_path = confinement + .wire_path(&resolved.segments) + .filter(|canonical| canonical != &root.path); + let links = if params.follow_symlinks { + LinkMode::Follow + } else { + LinkMode::Report + }; + let mut walker = Walker::new( + confinement.clone(), + &resolved, + filter.clone(), + links, + root.max_depth, + &root.path, + )?; + let limits = params.limits; + let read_files = params.read_content || params.include_digests; + let mut entries = Vec::new(); + let mut skipped = Vec::new(); + let mut skipped_entry_count = 0u32; + let mut content_bytes = 0u64; + let mut result_bytes = 0u64; + let mut fingerprint = Sha256::new(); + fingerprint.update([ + u8::from(params.read_content), + u8::from(params.include_digests), + ]); + let mut skip = |path: Option, reason: ScanSkipReason, result_bytes: &mut u64| { + skipped_entry_count = skipped_entry_count.saturating_add(1); + if (skipped.len() as u32) < MAX_LISTED_SKIPS { + *result_bytes += path + .as_ref() + .map_or(0, |path| json_string_len(path.as_str())) + + ENTRY_OVERHEAD; + skipped.push(ScanSkip { path, reason }); + } + }; + while let Some(event) = walker.next_event() { + if cancel.is_cancelled() { + return Err(RootError::Cancelled); + } + if started.elapsed() >= deadline { + return Err(ProtocolError::deadline_exceeded(format!( + "scanning {} took longer than {} ms", + root.path, limits.max_duration_ms + )) + .into()); + } + let entry = match event { + Event::Yield + | Event::Skip { + selected: false, .. + } => continue, + Event::Skip { path, reason, .. } => { + let reason = match reason { + Skip::NonUtf8Name => ScanSkipReason::NonUtf8Name, + Skip::Unreadable(_) | Skip::DanglingLink => ScanSkipReason::Unreadable, + Skip::LinkOutsideRoot(_) => ScanSkipReason::SymlinkOutsideRoot, + }; + skip(path, reason, &mut result_bytes); + continue; } - let mut pending = vec![(selected_path, String::new(), BTreeSet::::new())]; - while let Some((requested, path, mut ancestors)) = pending.pop() { - visited += 1; - if visited > limits.max_entries { - return Err(invalid("scan visited entry limit exceeded")); - } - if start.elapsed() > Duration::from_millis(limits.max_duration_ms.min(30_000)) { - return Err(error(Code::Timeout, "scan deadline exceeded")); - } - if !valid_path(&path) - || (!path.is_empty() && path.split('/').count() > limits.max_depth as usize) - { - return Err(invalid("scan depth/path limit exceeded")); - } - let canonical = requested.canonicalize().map_err(io)?; - let scoped = canonical - .strip_prefix(&canonical_root) - .map_err(|_| error(Code::Forbidden, "symlink target outside access scope"))?; - if !params.follow_symlinks && requested != canonical { - return Err(error(Code::Forbidden, "scan symlink following disabled")); - } - let scoped = scoped.to_str().ok_or_else(|| invalid("non-UTF-8 path"))?; - // The canonical spelling is still opened beneath an anchored directory - // without following symlinks, preventing a retarget race from escaping. - let mut file = if !params.read_content && params.digest_algorithm.is_none() { - anchor.metadata(scoped) - } else { - anchor.open(scoped) + Event::Entry(entry) => entry, + }; + if !entry.selected { + continue; + } + let (content, data) = match entry.stat.kind { + environment_protocol::data::fs::FileKind::Directory => { + (TreeContent::Directory {}, None) + } + environment_protocol::data::fs::FileKind::Symlink => { + skip(Some(entry.path), ScanSkipReason::Symlink, &mut result_bytes); + continue; + } + environment_protocol::data::fs::FileKind::Other => { + skip( + Some(entry.path), + ScanSkipReason::SpecialFile, + &mut result_bytes, + ); + continue; + } + environment_protocol::data::fs::FileKind::File => { + if entry.stat.size > limits.max_file_bytes { + skip( + Some(entry.path), + ScanSkipReason::FileTooLarge, + &mut result_bytes, + ); + continue; } - .map_err(io)?; - let observed = backend::observe(&file).map_err(io)?; - let matches = patterns.is_empty() || patterns.iter().any(|p| p.matches(&path)); - let mut data = None; - let content = if observed.is_dir() { - if direct_only && !path.is_empty() { - continue; - } - if !ancestors.insert(canonical.clone()) { - return Err(error(Code::Conflict, "scan symlink loop")); - } - let names = Directory::names(&file, limits.max_entries as usize).map_err(io)?; - if visited as usize + pending.len() + names.len() > limits.max_entries as usize - { - return Err(invalid("scan traversal limit exceeded")); - } - let pending_bytes: usize = pending - .iter() - .map(|(requested, path, ancestors)| { - requested.as_os_str().len() - + path.len() - + ancestors.iter().map(|p| p.as_os_str().len()).sum::() - }) - .sum(); - let added_bytes: usize = names - .iter() - .map(|name| { - requested.as_os_str().len() - + path.len() - + name.len() * 2 - + ancestors.iter().map(|p| p.as_os_str().len()).sum::() - }) - .sum(); - if (pending_bytes + added_bytes) as u64 > limits.max_manifest_bytes { - return Err(invalid("scan traversal byte limit exceeded")); - } - for name in names.into_iter().rev() { - if !valid_path(&name) { - return Err(invalid("unsupported filename")); - } - let child = if path.is_empty() { - name.clone() - } else { - format!("{path}/{name}") - }; - pending.push((requested.join(name), child, ancestors.clone())); - } - ScanContent::Directory + if !read_files { + ( + TreeContent::File { + size_bytes: entry.stat.size, + executable: entry.stat.executable(), + digest: None, + }, + None, + ) } else { - let digest = - if matches && (params.read_content || params.digest_algorithm.is_some()) { - if observed.size() > limits.max_file_bytes { - return Err(invalid("scan per-file byte limit exceeded")); - } - inspected = inspected - .checked_add(observed.size()) - .ok_or_else(|| invalid("byte overflow"))?; - if inspected > limits.max_total_bytes { - return Err(invalid("scan total byte limit exceeded")); - } - if params.read_content && observed.size() > MAX_CONTENT_CHUNK as u64 { - return Err(invalid("scan response file limit exceeded")); - } - let mut hash = Sha256::new(); - let mut bytes = Vec::new(); - let mut read = 0u64; - let mut buffer = [0u8; 65536]; - loop { - let count = file.read(&mut buffer).map_err(io)?; - if count == 0 { - break; - } - read += count as u64; - if read > observed.size() { - return Err(conflict()); - } - if start.elapsed() - > Duration::from_millis(limits.max_duration_ms.min(30_000)) - { - return Err(error(Code::Timeout, "scan deadline exceeded")); + match read_file( + &entry.parent, + &entry.name, + &entry.stat, + limits.max_file_bytes, + ) { + Ok(Some(bytes)) => { + let digest = ContentDigest::from_sha256(Sha256::digest(&bytes).into()); + fingerprint.update(digest.sha256()); + let data = if params.read_content { + content_bytes += bytes.len() as u64; + if totals.content_bytes + content_bytes > limits.max_total_bytes { + return Err(ProtocolError::payload_too_large( + limits.max_total_bytes, + format!( + "the content of {} exceeds the scan's remaining maxTotalBytes", + root.path + ), + ) + .into()); } - hash.update(&buffer[..count]); - if params.read_content { - bytes.extend_from_slice(&buffer[..count]); - } - } - if read != observed.size() - || !observed.matches(&backend::observe(&file).map_err(io)?) - { - return Err(conflict()); - } - if params.read_content { - data = Some(ByteChunk::from(bytes)); - } - params.digest_algorithm.map(|_| digest(hash)) - } else { - None - }; - ScanContent::File { - size_bytes: observed.size(), - executable: observed.executable(), - digest, - } - }; - observation_bytes += - (requested.as_os_str().len() + canonical.as_os_str().len() + 128) as u64; - if observation_bytes > limits.max_manifest_bytes { - return Err(invalid("scan observation memory limit exceeded")); - } - observations.push((requested, canonical.clone(), observed)); - if matches { - let row = ScanEntry { - root: selected.clone(), - path, - canonical_path: EnvironmentPath::new(canonical.to_string_lossy()) - .map_err(|e| invalid(&e.to_string()))?, - content, - data, - }; - response_bytes += serde_json::to_vec(&row) - .map_err(|e| invalid(&e.to_string()))? - .len() as u64; - if response_bytes > limits.max_manifest_bytes.min(4 * 1024 * 1024) - || response.entries.len() >= 10_000 - { - return Err(invalid("scan response byte limit exceeded")); + Some(ByteChunk::from(bytes)) + } else { + None + }; + ( + TreeContent::File { + size_bytes: entry.stat.size, + executable: entry.stat.executable(), + digest: params.include_digests.then_some(digest), + }, + data, + ) + } + Ok(None) => { + skip( + Some(entry.path), + ScanSkipReason::FileTooLarge, + &mut result_bytes, + ); + continue; + } + Err(()) => { + skip( + Some(entry.path), + ScanSkipReason::Unreadable, + &mut result_bytes, + ); + continue; + } } - response.entries.push(row); } } - Ok(()) - })(); - if let Err(error) = result { - response.complete = false; - response.diagnostics.push(ScanDiagnostic { - root: selected.clone(), - error, - }); + }; + if entries.len() as u32 >= limits.max_entries { + return Err(ProtocolError::too_many_entries( + limits.max_entries, + format!( + "{} selects more than {} entries", + root.path, limits.max_entries + ), + ) + .into()); } - } - // Validate identities and observations again, including nonmatching directories. - for (requested, canonical, observed) in observations { - let result = (|| -> Result<()> { - if start.elapsed() > Duration::from_millis(limits.max_duration_ms.min(30_000)) { - return Err(error(Code::Timeout, "scan deadline exceeded")); - } - if requested.canonicalize().map_err(io)? != canonical { - return Err(conflict()); - } - let path = canonical - .strip_prefix(&canonical_root) - .unwrap() - .to_str() - .ok_or_else(|| invalid("non-UTF-8 path"))?; - if !observed - .matches(&backend::observe(&anchor.metadata(path).map_err(io)?).map_err(io)?) - { - return Err(conflict()); + fingerprint.update(entry.path.as_str().as_bytes()); + fingerprint.update([0]); + match &content { + TreeContent::Directory {} => fingerprint.update(b"d"), + TreeContent::File { + size_bytes, + executable, + .. + } => { + fingerprint.update([b'f', u8::from(*executable)]); + fingerprint.update(size_bytes.to_le_bytes()); } - Ok(()) - })(); - if let Err(error) = result { - response.complete = false; - response.diagnostics.push(ScanDiagnostic { - root: EnvironmentPath::new(requested.to_string_lossy()) - .map_err(|e| invalid(&e.to_string()))?, - error, + } + result_bytes += json_string_len(entry.path.as_str()) + + ENTRY_OVERHEAD + + data.as_ref().map_or(0, |data: &ByteChunk| { + (data.as_slice().len() as u64).div_ceil(3) * 4 }); + if totals.result_bytes + result_bytes > MAX_RESULT_BYTES { + return Err(ProtocolError::payload_too_large( + MAX_RESULT_BYTES, + format!( + "the result of {} exceeds the scan's response budget", + root.path + ), + ) + .into()); } + entries.push(ScanEntry { + path: entry.path, + content, + data, + }); } - if response.complete { - response - .entries - .sort_by(|a, b| (a.root.as_str(), &a.path).cmp(&(b.root.as_str(), &b.path))); - let mut query = params.clone(); - query.if_none_match = None; - let fingerprint = digest(Sha256::new_with_prefix( - serde_json::to_vec(&(&canonical_root, query, &response.entries)) - .map_err(|e| invalid(&e.to_string()))?, - )); - response.unchanged = params.if_none_match.as_ref() == Some(&fingerprint); - response.fingerprint = Some(fingerprint); - if response.unchanged { - response.entries.clear(); + let fingerprint = ScanFingerprint::new(format!( + "{}{}", + ScanFingerprint::PREFIX, + hex::encode(fingerprint.finalize()) + )) + .expect("a hex digest is a valid fingerprint"); + if root.if_none_match.as_ref() == Some(&fingerprint) { + return Ok(ScanRootResult::Unchanged { + path: root.path.clone(), + fingerprint, + }); + } + totals.content_bytes += content_bytes; + totals.result_bytes += result_bytes; + Ok(ScanRootResult::Complete { + path: root.path.clone(), + canonical_path, + fingerprint, + entries, + skipped, + skipped_entry_count, + }) +} + +/// Read a selected file whole: `Ok(None)` when it grew past `max`, `Err` +/// when it vanished, changed or could not be read. +fn read_file( + parent: &os::Directory, + name: &std::ffi::OsStr, + observed: &EntryStat, + max: u64, +) -> Result>, ()> { + let mut file = parent.open_read(name).map_err(drop)?; + let opened = EntryStat::of_file(&file).map_err(drop)?; + if !opened.is_file() || !opened.unchanged(observed) { + return Err(()); + } + let bytes = os::read_up_to(&mut file, max + 1).map_err(drop)?; + if bytes.len() as u64 > max { + return Ok(None); + } + Ok(Some(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + use environment_protocol::{ + data::{fs::TreeFilter, scan::ScanLimits}, + error::ProtocolErrorKind, + paths::EnvironmentPath, + }; + + fn limits() -> ScanLimits { + ScanLimits { + max_entries: 100, + max_file_bytes: 1024, + max_total_bytes: 4096, + max_duration_ms: 10_000, } } - Ok(response) + + fn root(path: &std::path::Path) -> ScanRoot { + ScanRoot { + path: EnvironmentPath::new(path.to_str().unwrap()).unwrap(), + if_none_match: None, + max_depth: None, + } + } + + fn fixture() -> (tempfile::TempDir, std::path::PathBuf, Arc) { + let temp = tempfile::tempdir().unwrap(); + let host = temp.path().canonicalize().unwrap(); + let confinement = Arc::new(Confinement::new(host.clone(), host.join(".state")).unwrap()); + (temp, host, confinement) + } + + #[test] + fn one_bad_entry_is_skipped_and_one_bad_root_fails_alone() { + let (_temp, host, confinement) = fixture(); + std::fs::create_dir_all(host.join("skills/a")).unwrap(); + std::fs::create_dir_all(host.join("skills/b")).unwrap(); + std::fs::write(host.join("skills/a/SKILL.md"), "a").unwrap(); + std::fs::write(host.join("skills/b/SKILL.md"), vec![b'b'; 2048]).unwrap(); + std::fs::write(host.join("file"), "not a directory").unwrap(); + let params = ScanParams { + roots: vec![ + root(&host.join("skills")), + root(&host.join("missing")), + root(&host.join("file")), + root(&host.join(".state")), + ], + filter: TreeFilter { + include: vec!["**/SKILL.md".into()], + ..TreeFilter::default() + }, + read_content: true, + include_digests: false, + follow_symlinks: false, + limits: limits(), + }; + let response = scan(&confinement, params, &CancellationToken::new()).unwrap(); + let ScanRootResult::Complete { + entries, + skipped, + skipped_entry_count, + .. + } = &response.roots[0] + else { + panic!("{:?}", response.roots[0]); + }; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].path.as_str(), "a/SKILL.md"); + assert_eq!(entries[0].data.as_ref().unwrap().as_slice(), b"a"); + assert_eq!( + skipped, + &[ScanSkip { + path: Some(TreePath::new("b/SKILL.md").unwrap()), + reason: ScanSkipReason::FileTooLarge, + }] + ); + assert_eq!(*skipped_entry_count, 1); + assert!(matches!(response.roots[1], ScanRootResult::Missing { .. })); + let ScanRootResult::Failed { error, .. } = &response.roots[2] else { + panic!("{:?}", response.roots[2]); + }; + assert!(matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::NotDirectory + } + )); + let ScanRootResult::Failed { error, .. } = &response.roots[3] else { + panic!("{:?}", response.roots[3]); + }; + assert_eq!(error.kind(), ProtocolErrorKind::PermissionDenied); + } + + #[test] + fn depth_zero_probes_a_directory_and_fingerprints_detect_changes() { + let (_temp, host, confinement) = fixture(); + std::fs::create_dir_all(host.join("work")).unwrap(); + std::fs::write(host.join("work/a"), "one").unwrap(); + let probe = ScanRoot { + max_depth: Some(0), + ..root(&host.join("work")) + }; + let params = |roots: Vec| ScanParams { + roots, + filter: TreeFilter::default(), + read_content: false, + include_digests: true, + follow_symlinks: false, + limits: limits(), + }; + let response = scan( + &confinement, + params(vec![probe, root(&host.join("work"))]), + &CancellationToken::new(), + ) + .unwrap(); + let ScanRootResult::Complete { entries, .. } = &response.roots[0] else { + panic!("{:?}", response.roots[0]); + }; + assert!(entries.is_empty()); + let ScanRootResult::Complete { fingerprint, .. } = &response.roots[1] else { + panic!("{:?}", response.roots[1]); + }; + let again = ScanRoot { + if_none_match: Some(fingerprint.clone()), + ..root(&host.join("work")) + }; + let response = scan( + &confinement, + params(vec![again.clone()]), + &CancellationToken::new(), + ) + .unwrap(); + assert!(matches!( + response.roots[0], + ScanRootResult::Unchanged { .. } + )); + std::fs::write(host.join("work/a"), "two").unwrap(); + let response = scan(&confinement, params(vec![again]), &CancellationToken::new()).unwrap(); + assert!(matches!(response.roots[0], ScanRootResult::Complete { .. })); + } } diff --git a/crates/environment-daemon/src/filesystem/search.rs b/crates/environment-daemon/src/filesystem/search.rs new file mode 100644 index 000000000..49729fddc --- /dev/null +++ b/crates/environment-daemon/src/filesystem/search.rs @@ -0,0 +1,546 @@ +//! `fs/searchText` and `fs/globFiles`: bounded walks run on a blocking +//! thread that stop at their duration ceiling, on `$/cancelRequest` and +//! when their connection closes. + +use std::{ + io::{self, Read}, + sync::Arc, + time::{Duration, Instant}, +}; + +use environment_protocol::{ + data::{ + fs::{ + FileKind, GlobFilesParams, GlobFilesResponse, GlobFilesStop, SearchTextMatch, + SearchTextParams, SearchTextResponse, SearchTextStop, + }, + handshake::{GlobCeilings, SearchCeilings}, + }, + error::{PreconditionReason, ProtocolError}, + limits::MAX_RESULT_BYTES, + paths::EnvironmentPath, +}; +use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkMatch}; +use tokio_util::sync::CancellationToken; + +use super::{ + confinement::{Confinement, Lookup}, + json_string_len, + os::EntryStat, + walk::{CompiledFilter, Event, LinkMode, Walker}, +}; + +/// Encoded overhead of one search match besides its encoded path and line. +const SEARCH_MATCH_OVERHEAD: u64 = 64; +/// Encoded overhead of one glob match besides its encoded path. +const GLOB_MATCH_OVERHEAD: u64 = 16; +/// Longest line the searcher buffers; a file with a longer line is skipped. +const MAX_SEARCH_LINE_BUFFER: usize = 16 * 1024 * 1024; + +pub(crate) fn check_search( + params: &SearchTextParams, + ceilings: &SearchCeilings, +) -> Result<(), ProtocolError> { + if params.pattern.is_empty() { + return Err(ProtocolError::invalid_field( + "pattern", + "search pattern must not be empty", + )); + } + let limits = ¶ms.limits; + check_quota( + "limits.maxMatches", + limits.max_matches.into(), + ceilings.max_matches.into(), + )?; + check_quota( + "limits.maxFiles", + limits.max_files.into(), + ceilings.max_files.into(), + )?; + check_quota("limits.maxBytes", limits.max_bytes, ceilings.max_bytes)?; + check_quota( + "limits.maxDurationMs", + limits.max_duration_ms, + ceilings.max_duration_ms, + )?; + check_quota( + "limits.maxLineBytes", + limits.max_line_bytes.into(), + ceilings.max_line_bytes.into(), + ) +} + +pub(crate) fn check_glob( + params: &GlobFilesParams, + ceilings: &GlobCeilings, +) -> Result<(), ProtocolError> { + if params.filter.include.is_empty() { + return Err(ProtocolError::invalid_field( + "filter.include", + "globFiles needs at least one include pattern", + )); + } + let limits = ¶ms.limits; + check_quota( + "limits.maxMatches", + limits.max_matches.into(), + ceilings.max_matches.into(), + )?; + check_quota( + "limits.maxFiles", + limits.max_files.into(), + ceilings.max_files.into(), + )?; + check_quota( + "limits.maxDurationMs", + limits.max_duration_ms, + ceilings.max_duration_ms, + ) +} + +/// A caller quota: at least 1 and at most its ceiling. +pub(crate) fn check_quota(field: &str, value: u64, ceiling: u64) -> Result<(), ProtocolError> { + if value == 0 { + return Err(ProtocolError::invalid_field( + field, + format!("{field} must be at least 1"), + )); + } + if value > ceiling { + return Err(ProtocolError::above_ceiling(field, value, ceiling)); + } + Ok(()) +} + +/// Resolve a walk root: an existing directory under the confinement rule. +fn walker( + confinement: &Arc, + root: &EnvironmentPath, + filter: CompiledFilter, + links: LinkMode, +) -> Result { + let resolved = confinement.lookup(root, Lookup::FOLLOW)?; + if !resolved.stat.is_some_and(|stat| stat.is_dir()) { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotDirectory, + format!("{root} is not a directory"), + )); + } + Walker::new(confinement.clone(), &resolved, filter, links, None, root) +} + +pub(crate) fn search_text( + confinement: &Arc, + params: SearchTextParams, + cancel: &CancellationToken, +) -> Result { + let started = Instant::now(); + let limits = params.limits; + let matcher = grep_regex::RegexMatcherBuilder::new() + .case_insensitive(params.pattern_ignore_case) + .build(¶ms.pattern) + .map_err(|error| { + ProtocolError::invalid_field("pattern", format!("invalid search regex: {error}")) + })?; + let filter = CompiledFilter::compile(¶ms.filter, "filter")?; + let mut walker = walker(confinement, ¶ms.root, filter, LinkMode::Report)?; + let deadline = Duration::from_millis(limits.max_duration_ms); + let mut searcher = SearcherBuilder::new() + .line_number(true) + .binary_detection(BinaryDetection::quit(0)) + .heap_limit(Some(MAX_SEARCH_LINE_BUFFER)) + .build(); + let mut response = SearchTextResponse { + matches: Vec::new(), + searched_file_count: 0, + searched_bytes: 0, + skipped_file_count: 0, + skipped_entry_count: 0, + elapsed_ms: 0, + stopped: None, + }; + let mut result_bytes = 0u64; + while let Some(event) = walker.next_event() { + if cancel.is_cancelled() { + return Err(ProtocolError::cancelled()); + } + if started.elapsed() >= deadline { + response.stopped = Some(SearchTextStop::TimeLimit); + break; + } + let entry = match event { + Event::Yield => continue, + Event::Skip { .. } => { + response.skipped_entry_count = response.skipped_entry_count.saturating_add(1); + continue; + } + Event::Entry(entry) => entry, + }; + if entry.stat.kind != FileKind::File || !entry.selected { + continue; + } + let remaining = limits.max_bytes - response.searched_bytes; + if remaining == 0 { + response.stopped = Some(SearchTextStop::ByteLimit); + break; + } + if response.searched_file_count >= limits.max_files { + response.stopped = Some(SearchTextStop::FileLimit); + break; + } + if entry.stat.size > remaining { + response.skipped_file_count = response.skipped_file_count.saturating_add(1); + continue; + } + let Ok(file) = entry.parent.open_read(&entry.name) else { + response.skipped_file_count = response.skipped_file_count.saturating_add(1); + continue; + }; + if !EntryStat::of_file(&file).is_ok_and(|stat| stat.is_file()) { + response.skipped_file_count = response.skipped_file_count.saturating_add(1); + continue; + } + let path = params + .root + .join(&entry.path) + .map_err(|error| ProtocolError::internal(error.to_string()))?; + let mut sink = MatchSink { + path, + matches: &mut response.matches, + max_matches: limits.max_matches, + max_line_bytes: limits.max_line_bytes as usize, + result_bytes: &mut result_bytes, + stopped: None, + binary: false, + }; + let mut reader = Counting { + inner: file.take(remaining), + read: 0, + deadline: started + deadline, + cancel, + interrupted: None, + }; + let outcome = searcher.search_reader(&matcher, &mut reader, &mut sink); + let (stopped, binary) = (sink.stopped, sink.binary); + response.searched_bytes += reader.read; + match reader.interrupted { + Some(Interrupted::Cancelled) => return Err(ProtocolError::cancelled()), + Some(Interrupted::Deadline) => { + response.stopped = Some(SearchTextStop::TimeLimit); + break; + } + None => {} + } + if outcome.is_err() || binary { + response.skipped_file_count = response.skipped_file_count.saturating_add(1); + } else { + response.searched_file_count += 1; + } + if stopped.is_some() { + response.stopped = stopped; + break; + } + } + response.elapsed_ms = started.elapsed().as_millis() as u64; + Ok(response) +} + +/// Why a file's search stopped before its end. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Interrupted { + Cancelled, + Deadline, +} + +/// Counts the bytes read and ends the read once the walk's deadline passed +/// or its request was cancelled, so one large file cannot outlast either. +struct Counting<'a, R> { + inner: R, + read: u64, + deadline: Instant, + cancel: &'a CancellationToken, + interrupted: Option, +} + +impl Read for Counting<'_, R> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.cancel.is_cancelled() { + self.interrupted = Some(Interrupted::Cancelled); + } else if Instant::now() >= self.deadline { + self.interrupted = Some(Interrupted::Deadline); + } + if self.interrupted.is_some() { + // Not `Interrupted`: readers retry that kind. + return Err(io::Error::other("the search was stopped")); + } + let read = self.inner.read(buffer)?; + self.read += read as u64; + Ok(read) + } +} + +struct MatchSink<'a> { + path: EnvironmentPath, + matches: &'a mut Vec, + max_matches: u32, + max_line_bytes: usize, + result_bytes: &'a mut u64, + stopped: Option, + binary: bool, +} + +impl Sink for MatchSink<'_> { + type Error = io::Error; + + fn matched(&mut self, _: &Searcher, found: &SinkMatch<'_>) -> Result { + if self.matches.len() as u64 >= u64::from(self.max_matches) { + self.stopped = Some(SearchTextStop::MatchLimit); + return Ok(false); + } + let line = String::from_utf8_lossy(found.bytes()); + let line = line.trim_end_matches(['\r', '\n']); + let (line, line_truncated) = cut(line, self.max_line_bytes); + let size = + json_string_len(self.path.as_str()) + json_string_len(line) + SEARCH_MATCH_OVERHEAD; + if *self.result_bytes + size > MAX_RESULT_BYTES { + self.stopped = Some(SearchTextStop::ResponseLimit); + return Ok(false); + } + *self.result_bytes += size; + let line_number = found + .line_number() + .map_or(0, |number| u32::try_from(number).unwrap_or(u32::MAX)); + self.matches.push(SearchTextMatch { + path: self.path.clone(), + line_number, + line: line.to_owned(), + line_truncated, + }); + Ok(true) + } + + fn binary_data(&mut self, _: &Searcher, _: u64) -> Result { + self.binary = true; + Ok(false) + } +} + +/// Cut `line` to at most `max` bytes on a character boundary. +fn cut(line: &str, max: usize) -> (&str, bool) { + if line.len() <= max { + return (line, false); + } + let mut end = max; + while !line.is_char_boundary(end) { + end -= 1; + } + (&line[..end], true) +} + +pub(crate) fn glob_files( + confinement: &Arc, + params: GlobFilesParams, + cancel: &CancellationToken, +) -> Result { + let started = Instant::now(); + let limits = params.limits; + let filter = CompiledFilter::compile(¶ms.filter, "filter")?; + let mut walker = walker(confinement, ¶ms.root, filter, LinkMode::Report)?; + let deadline = Duration::from_millis(limits.max_duration_ms); + let mut response = GlobFilesResponse { + matches: Vec::new(), + visited_file_count: 0, + skipped_entry_count: 0, + elapsed_ms: 0, + stopped: None, + }; + let mut result_bytes = 0u64; + while let Some(event) = walker.next_event() { + if cancel.is_cancelled() { + return Err(ProtocolError::cancelled()); + } + if started.elapsed() >= deadline { + response.stopped = Some(GlobFilesStop::TimeLimit); + break; + } + let entry = match event { + Event::Yield => continue, + Event::Skip { .. } => { + response.skipped_entry_count = response.skipped_entry_count.saturating_add(1); + continue; + } + Event::Entry(entry) => entry, + }; + if entry.stat.is_dir() { + continue; + } + if response.visited_file_count >= limits.max_files { + response.stopped = Some(GlobFilesStop::FileLimit); + break; + } + response.visited_file_count += 1; + if !entry.selected { + continue; + } + if response.matches.len() as u64 >= u64::from(limits.max_matches) { + response.stopped = Some(GlobFilesStop::MatchLimit); + break; + } + let path = params + .root + .join(&entry.path) + .map_err(|error| ProtocolError::internal(error.to_string()))?; + let size = json_string_len(path.as_str()) + GLOB_MATCH_OVERHEAD; + if result_bytes + size > MAX_RESULT_BYTES { + response.stopped = Some(GlobFilesStop::ResponseLimit); + break; + } + result_bytes += size; + response.matches.push(path); + } + response.elapsed_ms = started.elapsed().as_millis() as u64; + Ok(response) +} + +#[cfg(test)] +mod tests { + use super::*; + use environment_protocol::data::fs::{GlobFilesLimits, SearchTextLimits, TreeFilter}; + + struct Fixture { + _temp: tempfile::TempDir, + root: EnvironmentPath, + host: std::path::PathBuf, + confinement: Arc, + } + + fn fixture() -> Fixture { + let temp = tempfile::tempdir().unwrap(); + let host = temp.path().canonicalize().unwrap(); + std::fs::create_dir_all(host.join("src/deep")).unwrap(); + std::fs::create_dir_all(host.join(".git/objects")).unwrap(); + std::fs::write( + host.join("src/lib.rs"), + "pub fn target() {}\nfn other() {}\n", + ) + .unwrap(); + std::fs::write(host.join("src/deep/nested.rs"), "target nested\n").unwrap(); + std::fs::write(host.join("readme.md"), "target readme\n").unwrap(); + std::fs::write(host.join("binary.bin"), b"tar\x00get\n").unwrap(); + // A pack file larger than any test budget; VCS metadata is pruned + // before the budget is charged. + std::fs::write(host.join(".git/objects/pack"), vec![b't'; 64 * 1024]).unwrap(); + let confinement = Arc::new(Confinement::new(host.clone(), host.join(".state")).unwrap()); + Fixture { + _temp: temp, + root: EnvironmentPath::new(host.to_str().unwrap()).unwrap(), + host, + confinement, + } + } + + fn search(root: &EnvironmentPath) -> SearchTextParams { + SearchTextParams { + root: root.clone(), + pattern: "target".into(), + pattern_ignore_case: false, + filter: TreeFilter::default(), + limits: SearchTextLimits { + max_matches: 100, + max_files: 100, + max_bytes: 1024, + max_duration_ms: 10_000, + max_line_bytes: 4096, + }, + } + } + + #[test] + fn search_skips_vcs_metadata_and_oversized_files_without_stopping() { + let fixture = fixture(); + std::fs::write(fixture.host.join("big.txt"), vec![b'x'; 2048]).unwrap(); + let response = search_text( + &fixture.confinement, + search(&fixture.root), + &CancellationToken::new(), + ) + .unwrap(); + let paths: Vec<_> = response + .matches + .iter() + .map(|found| { + found + .path + .as_str() + .trim_start_matches(fixture.root.as_str()) + }) + .collect(); + assert_eq!(paths, ["/readme.md", "/src/deep/nested.rs", "/src/lib.rs"]); + assert_eq!(response.stopped, None); + // big.txt exceeds the remaining budget and binary.bin is binary: + // both are skipped and counted, and the walk continues. + assert_eq!(response.skipped_file_count, 2); + assert!(response.searched_bytes < 1024); + } + + #[test] + fn search_stops_on_limits_and_cancellation() { + let fixture = fixture(); + let mut params = search(&fixture.root); + params.limits.max_matches = 1; + let response = + search_text(&fixture.confinement, params, &CancellationToken::new()).unwrap(); + assert_eq!(response.matches.len(), 1); + assert_eq!(response.stopped, Some(SearchTextStop::MatchLimit)); + + let mut params = search(&fixture.root); + params.limits.max_line_bytes = 3; + let response = + search_text(&fixture.confinement, params, &CancellationToken::new()).unwrap(); + assert!(response.matches.iter().all(|found| found.line_truncated)); + + let cancel = CancellationToken::new(); + cancel.cancel(); + let error = search_text(&fixture.confinement, search(&fixture.root), &cancel).unwrap_err(); + assert_eq!(error, ProtocolError::cancelled()); + } + + #[test] + fn glob_uses_the_tree_grammar_and_counts_only_files() { + let fixture = fixture(); + let params = |patterns: &[&str]| GlobFilesParams { + root: fixture.root.clone(), + filter: TreeFilter { + include: patterns.iter().map(|p| (*p).to_owned()).collect(), + ..TreeFilter::default() + }, + limits: GlobFilesLimits { + max_matches: 100, + max_files: 100, + max_duration_ms: 10_000, + }, + }; + let token = CancellationToken::new(); + let response = glob_files(&fixture.confinement, params(&["src/*.rs"]), &token).unwrap(); + assert_eq!(response.matches.len(), 1); + assert!(response.matches[0].as_str().ends_with("/src/lib.rs")); + let response = glob_files(&fixture.confinement, params(&["*.rs"]), &token).unwrap(); + assert_eq!(response.matches.len(), 2); + // Directories are free; the four visible files are counted. + assert_eq!(response.visited_file_count, 4); + let error = check_glob( + ¶ms(&[]), + &GlobCeilings { + max_matches: 10, + max_files: 10, + max_duration_ms: 10, + }, + ) + .unwrap_err(); + assert!(matches!( + error.data, + environment_protocol::error::ErrorData::InvalidParams { .. } + )); + } +} diff --git a/crates/environment-daemon/src/filesystem/transfer/capture.rs b/crates/environment-daemon/src/filesystem/transfer/capture.rs new file mode 100644 index 000000000..b7474a22d --- /dev/null +++ b/crates/environment-daemon/src/filesystem/transfer/capture.rs @@ -0,0 +1,416 @@ +//! Capture: walk a source under the shared traversal, hash every selected +//! file in bounded steps, and serve the sealed inventory's content by +//! digest. A file that changes between hashing and reading fails the +//! operation (`aborted`, `sourceChanged`). + +use std::{ + collections::BTreeMap, + ffi::OsString, + fs::File, + io::{Read, Seek, SeekFrom}, + sync::Arc, +}; + +use environment_protocol::{ + data::{ + fs::{FileKind, TreeContent, TreeFilter}, + transfer::{ + InventoryEntry, SpecialFilePolicy, SymlinkPolicy, TransferLimits, TransferSkip, + TransferSkipReason, + }, + }, + error::{AbortReason, PreconditionReason, ProtocolError}, + limits::MAX_LISTED_SKIPS, + paths::{ContentDigest, EnvironmentPath, TreePath}, +}; +use sha2::{Digest, Sha256}; + +use crate::filesystem::{ + confinement::{Confinement, Lookup}, + io_error, + os::{self, EntryStat}, + walk::{CompiledFilter, Event, LinkMode, Skip, Walker}, +}; + +use super::{STEP_BYTES, STEP_ENTRIES, check_entry_quota, entry_bytes}; + +/// Where a captured digest's bytes live. +#[derive(Clone, Debug)] +pub(super) struct Source { + pub canonical: Vec, + pub stat: EntryStat, +} + +struct Hashing { + path: TreePath, + source: Source, + file: File, + hash: Sha256, + read: u64, +} + +pub(super) struct Capture { + /// Canonical segments of the source; expanded links must stay inside. + selection: Vec, + walker: Option, + hashing: Option, + symlinks: SymlinkPolicy, + special_files: SpecialFilePolicy, + limits: TransferLimits, + pub entries: Vec, + pub sources: BTreeMap, + pub total_bytes: u64, + manifest_bytes: u64, + pub skipped: Vec, + pub skipped_count: u32, +} + +impl Capture { + pub fn begin( + confinement: Arc, + source: &EnvironmentPath, + filter: &TreeFilter, + symlinks: SymlinkPolicy, + special_files: SpecialFilePolicy, + limits: TransferLimits, + ) -> Result { + let compiled = CompiledFilter::compile(filter, "selection.filter")?; + let resolved = confinement.lookup(source, Lookup::FOLLOW)?; + confinement.refuse_containing_state(&resolved, source)?; + let stat = resolved.stat.expect("a followed lookup reports its target"); + let mut capture = Self { + selection: resolved.segments.clone(), + walker: None, + hashing: None, + symlinks, + special_files, + limits, + entries: Vec::new(), + sources: BTreeMap::new(), + total_bytes: 0, + manifest_bytes: 0, + skipped: Vec::new(), + skipped_count: 0, + }; + match stat.kind { + FileKind::Directory => { + capture.record(InventoryEntry { + path: TreePath::root(), + content: TreeContent::Directory {}, + })?; + let links = if symlinks == SymlinkPolicy::ExpandInside { + LinkMode::Follow + } else { + LinkMode::Report + }; + capture.walker = Some(Walker::new( + confinement, + &resolved, + compiled, + links, + None, + source, + )?); + } + FileKind::File => { + let file = resolved + .parent + .open_read(&resolved.name) + .map_err(|error| io_error(error, source))?; + capture.start_hashing( + TreePath::root(), + Source { + canonical: resolved.segments, + stat, + }, + file, + )?; + } + FileKind::Symlink | FileKind::Other => { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotRegularFile, + format!("{source} is not a regular file or directory"), + )); + } + } + Ok(capture) + } + + /// One bounded step; `true` once the inventory is sealed. + pub fn advance(&mut self) -> Result { + let mut allowance = STEP_BYTES; + for _ in 0..STEP_ENTRIES { + if self.hashing.is_some() { + if !self.hash_step(&mut allowance)? { + return Ok(false); + } + continue; + } + let Some(walker) = self.walker.as_mut() else { + return Ok(true); + }; + let Some(event) = walker.next_event() else { + self.walker = None; + return Ok(true); + }; + self.visit(event)?; + } + Ok(self.walker.is_none() && self.hashing.is_none()) + } + + /// Selection comes first: an entry or skip the filter does not select + /// is left out silently, before any policy, depth or readability check + /// could fail the capture. Directories are always walked. + fn visit(&mut self, event: Event) -> Result<(), ProtocolError> { + let entry = match event { + Event::Yield + | Event::Skip { + selected: false, .. + } => return Ok(()), + Event::Skip { + path, + reason: Skip::NonUtf8Name, + .. + } => { + self.skip(path, TransferSkipReason::NonUtf8Name); + return Ok(()); + } + Event::Skip { + path, + reason: Skip::DanglingLink, + .. + } => { + self.skip(path, TransferSkipReason::Symlink); + return Ok(()); + } + Event::Skip { + reason: Skip::Unreadable(error), + .. + } => return Err(error), + Event::Skip { + path, + reason: Skip::LinkOutsideRoot(error), + .. + } => { + return Err(symlink_rejected( + path.as_ref().map_or(".", TreePath::as_str), + &error.message, + )); + } + Event::Entry(entry) => entry, + }; + if !entry.selected && !entry.stat.is_dir() { + return Ok(()); + } + if entry.via_link && !entry.canonical.starts_with(&self.selection) { + return Err(symlink_rejected( + entry.path.as_str(), + "the link leaves the captured selection", + )); + } + if entry.depth > self.limits.max_depth { + return Err(ProtocolError::invalid_field( + "limits.maxDepth", + format!("{} is deeper than limits.maxDepth", entry.path), + )); + } + match entry.stat.kind { + FileKind::Directory => self.record(InventoryEntry { + path: entry.path, + content: TreeContent::Directory {}, + }), + FileKind::Symlink => match self.symlinks { + SymlinkPolicy::Skip => { + self.skip(Some(entry.path), TransferSkipReason::Symlink); + Ok(()) + } + SymlinkPolicy::Reject | SymlinkPolicy::ExpandInside => Err(symlink_rejected( + entry.path.as_str(), + "the capture refuses symlinks", + )), + }, + FileKind::Other => match self.special_files { + SpecialFilePolicy::Skip => { + self.skip(Some(entry.path), TransferSkipReason::SpecialFile); + Ok(()) + } + SpecialFilePolicy::Reject => Err(ProtocolError::failed_precondition( + PreconditionReason::NotRegularFile, + format!("{} is a special file", entry.path), + )), + }, + FileKind::File => { + let file = entry + .parent + .open_read(&entry.name) + .map_err(|error| io_error(error, &entry.path))?; + self.start_hashing( + entry.path, + Source { + canonical: entry.canonical, + stat: entry.stat, + }, + file, + ) + } + } + } + + fn start_hashing( + &mut self, + path: TreePath, + source: Source, + file: File, + ) -> Result<(), ProtocolError> { + let opened = EntryStat::of_file(&file).map_err(|error| io_error(error, &path))?; + if !opened.is_file() || !opened.unchanged(&source.stat) { + return Err(source_changed(&path)); + } + if source.stat.size > self.limits.max_file_bytes { + return Err(ProtocolError::payload_too_large( + self.limits.max_file_bytes, + format!("{path} is larger than limits.maxFileBytes"), + )); + } + self.total_bytes = self.total_bytes.saturating_add(source.stat.size); + if self.total_bytes > self.limits.max_total_bytes { + return Err(ProtocolError::payload_too_large( + self.limits.max_total_bytes, + "the capture exceeds limits.maxTotalBytes", + )); + } + self.hashing = Some(Hashing { + path, + source, + file, + hash: Sha256::new(), + read: 0, + }); + Ok(()) + } + + /// Hash within `allowance`; `true` when the file is done. + fn hash_step(&mut self, allowance: &mut u64) -> Result { + let task = self.hashing.as_mut().expect("caller checked"); + let mut buffer = vec![0u8; 64 * 1024]; + loop { + if *allowance == 0 { + return Ok(false); + } + let limit = (buffer.len() as u64).min(*allowance) as usize; + let read = task + .file + .read(&mut buffer[..limit]) + .map_err(|error| io_error(error, &task.path))?; + if read == 0 { + break; + } + task.read += read as u64; + if task.read > task.source.stat.size { + return Err(source_changed(&task.path)); + } + task.hash.update(&buffer[..read]); + *allowance -= read as u64; + } + let task = self.hashing.take().expect("hashing above"); + let now = EntryStat::of_file(&task.file).map_err(|error| io_error(error, &task.path))?; + if task.read != task.source.stat.size || !now.unchanged(&task.source.stat) { + return Err(source_changed(&task.path)); + } + let digest = ContentDigest::from_sha256(task.hash.finalize().into()); + let executable = task.source.stat.executable(); + let size_bytes = task.source.stat.size; + self.sources.entry(digest).or_insert(task.source); + self.record(InventoryEntry { + path: task.path, + content: TreeContent::File { + size_bytes, + executable, + digest: Some(digest), + }, + })?; + Ok(true) + } + + fn record(&mut self, entry: InventoryEntry) -> Result<(), ProtocolError> { + check_entry_quota(self.entries.len() + 1, &self.limits)?; + self.manifest_bytes += entry_bytes(&entry); + if self.manifest_bytes > self.limits.max_manifest_bytes { + return Err(ProtocolError::payload_too_large( + self.limits.max_manifest_bytes, + "the inventory exceeds limits.maxManifestBytes", + )); + } + self.entries.push(entry); + Ok(()) + } + + fn skip(&mut self, path: Option, reason: TransferSkipReason) { + self.skipped_count = self.skipped_count.saturating_add(1); + if (self.skipped.len() as u32) < MAX_LISTED_SKIPS { + self.skipped.push(TransferSkip { path, reason }); + } + } +} + +/// Why reading captured content failed. +pub(super) enum ReadFailure { + /// The bytes are not the bytes that were hashed. + Changed(ProtocolError), + Io(ProtocolError), +} + +/// Read `max` bytes of a captured source from `offset`: `(data, truncated)`. +pub(super) fn read_content( + confinement: &Confinement, + source: &Source, + offset: u64, + max: u64, +) -> Result<(Vec, bool), ReadFailure> { + let subject = format!("captured {}", source_label(source)); + let resolved = confinement + .reopen(&source.canonical, &subject) + .map_err(|_| ReadFailure::Changed(source_changed(&subject)))?; + let mut file = resolved + .parent + .open_read(&resolved.name) + .map_err(|error| ReadFailure::Io(io_error(error, &subject)))?; + let unchanged = |file: &File| { + EntryStat::of_file(file).is_ok_and(|stat| stat.is_file() && stat.unchanged(&source.stat)) + }; + if !unchanged(&file) { + return Err(ReadFailure::Changed(source_changed(&subject))); + } + file.seek(SeekFrom::Start(offset)) + .map_err(|error| ReadFailure::Io(io_error(error, &subject)))?; + let take = max.min(source.stat.size - offset); + let data = os::read_up_to(&mut file, take) + .map_err(|error| ReadFailure::Io(io_error(error, &subject)))?; + if data.len() as u64 != take || !unchanged(&file) { + return Err(ReadFailure::Changed(source_changed(&subject))); + } + let truncated = offset + take < source.stat.size; + Ok((data, truncated)) +} + +fn source_label(source: &Source) -> String { + source + .canonical + .last() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| ".".into()) +} + +pub(super) fn source_changed(subject: &dyn std::fmt::Display) -> ProtocolError { + ProtocolError::aborted( + Some(AbortReason::SourceChanged), + format!("{subject} changed during the transfer"), + ) +} + +fn symlink_rejected(path: &str, why: &str) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::SymlinkRejected, + format!("{path}: {why}"), + ) +} diff --git a/crates/environment-daemon/src/filesystem/transfer/journal.rs b/crates/environment-daemon/src/filesystem/transfer/journal.rs index 20d958f83..5dcb66c89 100644 --- a/crates/environment-daemon/src/filesystem/transfer/journal.rs +++ b/crates/environment-daemon/src/filesystem/transfer/journal.rs @@ -1,39 +1,247 @@ -//! Durable publication receipts. Interrupted in-flight operations fail closed after -//! restart; they are never silently re-executed against a possibly edited target. -use super::*; +//! Durable transfer records under `/transfers`. +//! +//! An operation is recorded at begin, before its staging directory exists, +//! so a restart can discard its staging directory and spool; when its commit +//! starts; and when it finishes, as its receipt. A receipt keeps the staging +//! and spool records until a sweep confirmed their removal. A completed +//! capture also keeps its sealed inventory beside its receipt, one entry +//! per line, so a retried capture can page it again. The +//! record is this daemon's own file: it is never read by another version, +//! and an undecodable record is renamed aside instead of failing startup. + +use std::{ + ffi::OsString, + fs::File, + io::{BufRead, BufReader, BufWriter, Read, Write}, + os::unix::ffi::{OsStrExt, OsStringExt}, + path::{Path, PathBuf}, +}; + +use environment_protocol::{ + data::transfer::{BeginTransferParams, InventoryEntry, TransferStatus}, + error::ProtocolError, + ids::TransferOperationId, +}; use serde::{Deserialize, Serialize}; -use std::path::PathBuf; -#[derive(Serialize, Deserialize)] -pub struct Record { - pub root: PathBuf, - pub request: TransferRequest, + +use crate::filesystem::io_error; + +/// Largest record read back; records hold no inventories. +const MAX_RECORD_BYTES: u64 = 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct Record { + pub params: BeginTransferParams, pub status: TransferStatus, - pub expires_at_ms: u64, - pub stage_name: Option, - #[serde(default)] - pub spool_directory: Option, + /// The staging directory of an active publish. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stage: Option, + /// The upload spool of an active publish. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub spool: Option, + /// A publish's commit started: after a restart, whether it published + /// is unknown. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub committing: bool, } -pub fn now_ms() -> u64 { + +/// Where a staging directory lives: the canonical segments of its parent +/// below the filesystem root (raw bytes, names need not be UTF-8) and its +/// name. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(super) struct StageRecord { + pub parent: Vec>, + pub name: String, +} + +impl StageRecord { + pub fn new(parent: &[OsString], name: &str) -> Self { + Self { + parent: parent + .iter() + .map(|segment| segment.as_bytes().to_vec()) + .collect(), + name: name.to_owned(), + } + } + + pub fn parent_segments(&self) -> Vec { + self.parent + .iter() + .map(|segment| OsString::from_vec(segment.clone())) + .collect() + } +} + +pub(super) fn now_ms() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis().min(i64::MAX as u128) as i64) .unwrap_or_default() - .as_millis() as u64 } -pub fn write(directory: &Path, id: &str, record: &Record) -> Result<()> { + +fn record_path(directory: &Path, id: &TransferOperationId) -> PathBuf { + directory.join(format!("{}.json", id.as_str())) +} + +const INVENTORY_EXTENSION: &str = "inventory"; + +fn inventory_path(directory: &Path, id: &TransferOperationId) -> PathBuf { + directory.join(format!("{}.{INVENTORY_EXTENSION}", id.as_str())) +} + +/// Write a completed capture's inventory atomically and durably, before +/// its receipt. +pub(super) fn write_inventory( + directory: &Path, + id: &TransferOperationId, + entries: &[InventoryEntry], +) -> Result<(), ProtocolError> { + let io = |error| io_error(error, format!("transfer journal {}", directory.display())); + let temporary = tempfile::NamedTempFile::new_in(directory).map_err(io)?; + let mut writer = BufWriter::new(temporary); + for entry in entries { + serde_json::to_writer(&mut writer, entry).map_err(|error| { + ProtocolError::internal(format!("encode transfer inventory: {error}")) + })?; + writer.write_all(b"\n").map_err(io)?; + } + let temporary = writer + .into_inner() + .map_err(|error| io(error.into_error()))?; + temporary.as_file().sync_all().map_err(io)?; + temporary + .persist(inventory_path(directory, id)) + .map_err(|error| io(error.error))?; + File::open(directory) + .and_then(|file| file.sync_all()) + .map_err(io) +} + +/// Up to `limit` entries of a retained inventory from `index`, and whether +/// more follow; `Ok(None)` when none is retained. +pub(super) fn read_inventory( + directory: &Path, + id: &TransferOperationId, + index: usize, + limit: usize, +) -> Result, bool)>, ProtocolError> { + let path = inventory_path(directory, id); + let file = match File::open(&path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(io_error(error, path.display())), + }; + let mut lines = BufReader::new(file).lines().skip(index); + let mut entries = Vec::new(); + for line in lines.by_ref().take(limit) { + let line = line.map_err(|error| io_error(error, path.display()))?; + entries.push(serde_json::from_str(&line).map_err(|error| { + ProtocolError::internal(format!("decode transfer inventory: {error}")) + })?); + } + Ok(Some((entries, lines.next().is_some()))) +} + +/// Write a record atomically and durably. +pub(super) fn write( + directory: &Path, + id: &TransferOperationId, + record: &Record, +) -> Result<(), ProtocolError> { + let io = |error| io_error(error, format!("transfer journal {}", directory.display())); let mut temporary = tempfile::NamedTempFile::new_in(directory).map_err(io)?; - serde_json::to_writer(&mut temporary, record).map_err(|e| invalid(&e.to_string()))?; + serde_json::to_writer(&mut temporary, record) + .map_err(|error| ProtocolError::internal(format!("encode transfer record: {error}")))?; temporary.as_file().sync_all().map_err(io)?; temporary - .persist(directory.join(format!("{id}.json"))) - .map_err(|e| io(e.error))?; - backend::sync_directory(directory).map_err(io) -} -pub fn read(directory: &Path, id: &str) -> Result> { - match File::open(directory.join(format!("{id}.json"))) { - Ok(file) => Ok(Some( - serde_json::from_reader(file.take(64 * 1024)).map_err(|e| invalid(&e.to_string()))?, - )), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(io(e)), + .persist(record_path(directory, id)) + .map_err(|error| io(error.error))?; + File::open(directory) + .and_then(|file| file.sync_all()) + .map_err(io) +} + +/// Read a record; `Ok(None)` when there is none or it was quarantined. +pub(super) fn read( + directory: &Path, + id: &TransferOperationId, +) -> Result, ProtocolError> { + read_path(&record_path(directory, id)) +} + +fn read_path(path: &Path) -> Result, ProtocolError> { + let file = match File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(io_error(error, path.display())), + }; + match serde_json::from_reader(file.take(MAX_RECORD_BYTES)) { + Ok(record) => Ok(Some(record)), + Err(error) => { + quarantine(path, &error.to_string()); + Ok(None) + } + } +} + +/// Remove a record and any inventory retained with it. +pub(super) fn remove(directory: &Path, id: &TransferOperationId) { + let _ = std::fs::remove_file(record_path(directory, id)); + let _ = std::fs::remove_file(inventory_path(directory, id)); +} + +/// Retained inventories whose record is gone (a crash between writing the +/// inventory and its receipt, or between removing the two). +pub(super) fn remove_orphan_inventories(directory: &Path) { + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path + .extension() + .is_some_and(|extension| extension == INVENTORY_EXTENSION) + && !path.with_extension("json").exists() + { + let _ = std::fs::remove_file(path); + } + } +} + +fn quarantine(path: &Path, reason: &str) { + let aside = path.with_extension("quarantined"); + eprintln!( + "lightspeed-envd: unreadable transfer record {} moved to {}: {reason}", + path.display(), + aside.display() + ); + let _ = std::fs::rename(path, aside); +} + +/// Every readable record with its id, in no particular order. +pub(super) fn all(directory: &Path) -> Vec<(TransferOperationId, Record)> { + let Ok(entries) = std::fs::read_dir(directory) else { + return Vec::new(); + }; + let mut records = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().is_none_or(|extension| extension != "json") { + continue; + } + let Some(id) = path + .file_stem() + .and_then(|stem| stem.to_str()) + .and_then(|stem| TransferOperationId::new(stem).ok()) + else { + continue; + }; + if let Ok(Some(record)) = read_path(&path) { + records.push((id, record)); + } } + records } diff --git a/crates/environment-daemon/src/filesystem/transfer/mod.rs b/crates/environment-daemon/src/filesystem/transfer/mod.rs index 712cf9f78..7bfb321c4 100644 --- a/crates/environment-daemon/src/filesystem/transfer/mod.rs +++ b/crates/environment-daemon/src/filesystem/transfer/mod.rs @@ -1,252 +1,1177 @@ -//! Compatibility adapters for small inline copies. All confinement, hashing and -//! publication use the same backend as paged transfers on Linux and macOS. -#[cfg(any(target_os = "linux", target_os = "macos"))] -pub(super) mod session; -use super::*; -use environment_protocol::data::{inventory::*, transfer_session::*}; -use sha2::{Digest, Sha256}; -fn invalid(message: &str) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::InvalidRequest, message) -} -fn limits(value: TransferLimits) -> Result { - if value.max_entries == 0 - || value.max_entries > MAX_TRANSFER_ENTRIES - || value.max_depth > MAX_TRANSFER_DEPTH - || value.max_file_bytes > MAX_TRANSFER_BYTES - || value.max_total_bytes > MAX_TRANSFER_BYTES - || value.max_duration_ms == 0 - || value.max_duration_ms > MAX_TRANSFER_DURATION_MS - { - return Err(invalid("inline transfer limits exceed protocol ceilings")); - } - Ok(InventoryLimits { - max_entries: value.max_entries, - max_depth: value.max_depth, - max_file_bytes: value.max_file_bytes, - max_total_bytes: value.max_total_bytes, - max_duration_ms: value.max_duration_ms, - ..Default::default() - }) -} -async fn advance( - fs: &LocalFileSystem, - id: &str, - phase: TransferPhase, -) -> Result<(), EnvironmentProtocolError> { - loop { - if let TransferResponse::Status(status) = fs - .transfer(TransferRequest::Advance { - operation_id: id.into(), - }) - .await? - && status.phase == phase +//! `transfer/*`: one logical capture, materialize or file write as many +//! bounded calls. +//! +//! Each operation has its own lock; uploads and captured reads of +//! different digests proceed concurrently. Only active operations are held +//! in memory and count against `maxActiveOperations`: a finished operation +//! (complete, aborted or failed) is journaled as its receipt, evicted, and +//! answered from the journal for `receiptRetentionMs` after it finished. +//! A completed capture's sealed inventory is retained with its receipt and +//! stays pageable, so a caller retrying a capture whose answer it lost +//! gets the tree it first captured without revisiting the live source. +//! +//! Errors of the operation's own work (walking, hashing, staging, +//! publishing, a source that changed, a digest mismatch, the duration +//! running out) move it to `failed`; every later call except +//! `transfer/getStatus` and `transfer/abort` then answers that error. A +//! request the daemon refuses without acting on it (invalid params, a call +//! in the wrong phase, an unexpected offset, an unknown digest) leaves the +//! operation as it was. + +mod capture; +mod journal; +mod publish; +#[cfg(test)] +mod tests; + +use std::{ + collections::BTreeMap, + path::PathBuf, + sync::{ + Arc, Mutex, MutexGuard, Weak, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use environment_protocol::{ + bytes::ByteChunk, + data::{ + handshake::TransferCeilings, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, InventoryEntry, + ListTransferMissingParams, ListTransferMissingResponse, ReadTransferContentParams, + ReadTransferContentResponse, ReadTransferInventoryParams, + ReadTransferInventoryResponse, TransferLimits, TransferOperationParams, TransferPhase, + TransferSelection, TransferStatus, WriteTransferContentParams, + WriteTransferContentResponse, + }, + }, + error::{BusyReason, Capability, NotFoundReason, PreconditionReason, ProtocolError}, + ids::TransferOperationId, +}; + +use super::{confinement::Confinement, io_error, search::check_quota}; +use capture::{Capture, ReadFailure}; +use journal::Record; +use publish::{ChunkFailure, Kind, Publish}; + +/// Content hashed or copied by one `transfer/advance`. +const STEP_BYTES: u64 = 4 * 1024 * 1024; +/// Entries visited by one `transfer/advance`. +const STEP_ENTRIES: usize = 1024; +/// How often finished receipts and overdue operations are swept. +const SWEEP_INTERVAL: Duration = Duration::from_secs(30); + +fn check_entry_quota(count: usize, limits: &TransferLimits) -> Result<(), ProtocolError> { + if count as u64 > u64::from(limits.max_entries) { + return Err(ProtocolError::too_many_entries( + limits.max_entries, + "the inventory exceeds limits.maxEntries", + )); + } + Ok(()) +} + +fn entry_bytes(entry: &InventoryEntry) -> u64 { + serde_json::to_vec(entry).map_or(0, |encoded| encoded.len() as u64) +} + +/// The daemon's transfer operations. +#[derive(Clone)] +pub(crate) struct Transfers { + shared: Arc, +} + +struct Shared { + confinement: Arc, + journal: PathBuf, + ceilings: TransferCeilings, + writable: bool, + active: Mutex>>, +} + +struct Operation { + id: TransferOperationId, + params: BeginTransferParams, + /// Canonical destination of a publish, for overlap checks. + destination: Option>, + /// Set (under `state`) once the operation finished; read without it. + finished: AtomicBool, + state: Mutex, +} + +struct State { + status: TransferStatus, + deadline_ms: i64, + work: Work, + /// The receipt of a finished operation whose journal write failed. The + /// operation stays in memory, answering from here, until a retry + /// persists it. + unpersisted: Option, +} + +enum Work { + Capture(Box), + Publish(Box), + Finished, +} + +/// How a finished operation answers each call. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Call { + Begin, + Abort, + Commit, + ReadInventory, + Other, +} + +enum Found { + Active(Arc), + Finished(TransferStatus), +} + +impl Transfers { + /// Journal under `/transfers`. Operations a restart + /// interrupted are recorded as failed (`aborted`) and their staging is + /// discarded. + pub fn new( + confinement: Arc, + ceilings: TransferCeilings, + writable: bool, + ) -> Result { + let journal = confinement.state_dir().join("transfers"); { - return Ok(()); + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new() + .recursive(true) + .mode(super::os::PRIVATE_DIRECTORY_MODE) + .create(&journal) + .map_err(|error| io_error(error, journal.display()))?; + } + let shared = Arc::new(Shared { + confinement, + journal, + ceilings, + writable, + active: Mutex::new(BTreeMap::new()), + }); + shared.recover(); + let weak = Arc::downgrade(&shared); + std::thread::Builder::new() + .name("transfer-sweep".into()) + .spawn(move || sweep_loop(weak)) + .map_err(|error| ProtocolError::internal(format!("start transfer sweep: {error}")))?; + Ok(Self { shared }) + } + + pub fn begin(&self, params: BeginTransferParams) -> Result { + self.shared.begin(params) + } + + pub fn advance( + &self, + params: TransferOperationParams, + ) -> Result { + self.shared.advance(¶ms.operation_id) + } + + pub fn get_status( + &self, + params: TransferOperationParams, + ) -> Result { + match self.shared.find(¶ms.operation_id)? { + Found::Finished(status) => Ok(status), + Found::Active(operation) => Ok(operation.lock().status.clone()), } } + + pub fn read_inventory( + &self, + params: ReadTransferInventoryParams, + ) -> Result { + self.shared.read_inventory(params) + } + + pub fn append( + &self, + params: AppendTransferInventoryParams, + ) -> Result { + self.shared.append(params) + } + + pub fn list_missing( + &self, + params: ListTransferMissingParams, + ) -> Result { + self.shared.list_missing(params) + } + + pub fn read( + &self, + params: ReadTransferContentParams, + ) -> Result { + self.shared.read(params) + } + + pub fn write( + &self, + params: WriteTransferContentParams, + ) -> Result { + self.shared.write(params) + } + + pub fn commit(&self, params: TransferOperationParams) -> Result { + self.shared.commit(¶ms.operation_id) + } + + pub fn abort(&self, params: TransferOperationParams) -> Result { + self.shared.abort(¶ms.operation_id) + } +} + +impl Operation { + fn lock(&self) -> MutexGuard<'_, State> { + self.state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } } -pub(super) async fn capture( - fs: &LocalFileSystem, - params: CaptureParams, -) -> Result { - let id = format!("inline-{:032x}", rand::random::()); - let result = async { - fs.transfer(TransferRequest::Begin { + +impl Shared { + fn retention_ms(&self) -> i64 { + i64::try_from(self.ceilings.receipt_retention_ms).unwrap_or(i64::MAX) + } + + fn find(&self, id: &TransferOperationId) -> Result { + if let Some(operation) = self.active_map().get(id).cloned() { + return Ok(Found::Active(operation)); + } + let Some(record) = journal::read(&self.journal, id)? else { + return Err(ProtocolError::not_found(format!("unknown transfer {id}"))); + }; + if !record.status.phase.is_finished() { + // Only a record left by an earlier daemon instance can be + // active without being in memory; recovery normally converts + // it already. + return Ok(Found::Finished(self.interrupted(id, record))); + } + if record + .status + .receipt_expires_at_ms + .is_some_and(|expires| expires <= journal::now_ms()) + { + return Err(ProtocolError::not_found_because( + NotFoundReason::Expired, + format!("the receipt of transfer {id} expired"), + )); + } + Ok(Found::Finished(record.status)) + } + + fn active_map(&self) -> MutexGuard<'_, BTreeMap>> { + self.active + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// Drop a finished operation from memory once its receipt is on disk; + /// retries a receipt write that failed. + fn evict(&self, operation: &Arc) { + if !operation.finished.load(Ordering::Acquire) { + return; + } + { + let mut state = operation.lock(); + if let Some(record) = &state.unpersisted { + if let Err(error) = journal::write(&self.journal, &operation.id, record) { + eprintln!( + "lightspeed-envd: journaling the receipt of transfer {}: {error}", + operation.id + ); + return; + } + state.unpersisted = None; + } + } + let mut active = self.active_map(); + if active + .get(&operation.id) + .is_some_and(|current| Arc::ptr_eq(current, operation)) + { + active.remove(&operation.id); + } + } + + /// Run `body` on an active operation, answering finished ones per + /// `call`, failing an overdue one, and evicting it once finished. + fn with_active( + &self, + id: &TransferOperationId, + call: Call, + body: impl FnOnce(&Self, &Arc, &mut State) -> Result, + finished: impl FnOnce(TransferStatus) -> Result, + ) -> Result { + let operation = match self.find(id)? { + Found::Finished(status) => return answer_finished(status, call).and_then(finished), + Found::Active(operation) => operation, + }; + let result = { + let mut state = operation.lock(); + if state.status.phase.is_finished() { + answer_finished(state.status.clone(), call).and_then(finished) + } else if call != Call::Abort && journal::now_ms() >= state.deadline_ms { + let error = ProtocolError::deadline_exceeded(format!( + "transfer {id} ran past its maxDurationMs" + )); + Err(self.fail(&operation, &mut state, error)) + } else { + body(self, &operation, &mut state) + } + }; + self.evict(&operation); + result + } + + /// Record `phase` (complete, aborted, failed), release the work and + /// journal the receipt. + fn finish( + &self, + operation: &Operation, + state: &mut State, + phase: TransferPhase, + error: Option, + ) { + state.status.phase = phase; + state.status.error = error; + state.status.deadline_at_ms = None; + state.status.receipt_expires_at_ms = + Some(journal::now_ms().saturating_add(self.retention_ms())); + // The staging directory and spool are removed in the background; + // the receipt names them until a sweep confirms they are gone. + let (stage, spool) = match &state.work { + Work::Publish(publish) => (publish.created_stage_record(), publish.spool_path()), + Work::Capture(_) | Work::Finished => (None, None), + }; + state.work = Work::Finished; + let record = Record { + params: operation.params.clone(), + status: state.status.clone(), + stage, + spool, + committing: false, + }; + if let Err(error) = journal::write(&self.journal, &operation.id, &record) { + eprintln!( + "lightspeed-envd: journaling the receipt of transfer {}: {error}", + operation.id + ); + state.unpersisted = Some(record); + } + operation.finished.store(true, Ordering::Release); + } + + fn fail( + &self, + operation: &Operation, + state: &mut State, + error: ProtocolError, + ) -> ProtocolError { + self.finish(operation, state, TransferPhase::Failed, Some(error.clone())); + error + } + + fn begin(&self, params: BeginTransferParams) -> Result { + self.check_limits(¶ms.limits)?; + let publishes = !matches!(params.selection, TransferSelection::Capture { .. }); + if publishes && !self.writable { + return Err(ProtocolError::capability_unavailable( + Capability::FilesystemWrite, + "the filesystem is read-only", + )); + } + let id = params.operation_id.clone(); + if let Some(existing) = self.existing(¶ms)? { + return Ok(existing); + } + // Checked again at insert; this avoids staging work that would be + // refused anyway. + self.check_capacity(&self.active_map())?; + let (mut work, destination) = self.start(¶ms)?; + let now = journal::now_ms(); + let deadline_ms = + now.saturating_add(i64::try_from(params.limits.max_duration_ms).unwrap_or(i64::MAX)); + let (phase, stage, spool) = match &work { + Work::Capture(_) => (TransferPhase::Scanning, None, None), + Work::Publish(publish) => ( + TransferPhase::Inventory, + Some(publish.stage_record()), + publish.spool_path(), + ), + Work::Finished => unreachable!("start returns work"), + }; + let status = TransferStatus { operation_id: id.clone(), - selection: TransferSelection::Capture { - source: params.source.clone(), - }, - limits: limits(params.limits)?, - }) - .await?; - advance(fs, &id, TransferPhase::Ready).await?; - let mut entries = Vec::new(); - let mut offset = 0; - let mut bytes = 0; - loop { - let TransferResponse::Inventory { - entries: page, - next_offset, - } = fs - .transfer(TransferRequest::Inventory { - operation_id: id.clone(), - offset, + phase, + entry_count: 0, + total_bytes: 0, + transferred_bytes: 0, + reused_bytes: 0, + skipped: Vec::new(), + skipped_entry_count: 0, + error: None, + deadline_at_ms: Some(deadline_ms), + receipt_expires_at_ms: None, + }; + let mut active = self.active_map(); + if active.contains_key(&id) { + drop(active); + discard(work); + return self.existing(¶ms)?.ok_or_else(|| { + ProtocolError::internal(format!("transfer {id} vanished during begin")) + }); + } + let admitted = self.check_capacity(&active).and_then(|()| { + let overlaps = destination.as_ref().is_some_and(|destination| { + active.values().any(|other| { + !other.finished.load(Ordering::Acquire) + && other.destination.as_ref().is_some_and(|other| { + other.starts_with(destination) || destination.starts_with(other) + }) }) - .await? - else { - return Err(invalid("unexpected inventory")); + }); + if overlaps { + return Err(ProtocolError::busy( + Some(BusyReason::TransferCapacity), + None, + "the destination overlaps an active transfer", + )); + } + Ok(()) + }); + // The record names the staging directory before it exists, so an + // exit at any later point leaves nothing a restart cannot find. + let recorded = admitted.and_then(|()| { + journal::write( + &self.journal, + &id, + &Record { + params: params.clone(), + status: status.clone(), + stage, + spool, + committing: false, + }, + ) + }); + if let Err(error) = recorded { + drop(active); + discard(work); + return Err(error); + } + if let Work::Publish(publish) = &mut work + && let Err(error) = publish.create_stage() + { + journal::remove(&self.journal, &id); + drop(active); + discard(work); + return Err(error); + } + active.insert( + id.clone(), + Arc::new(Operation { + id, + params, + destination, + finished: AtomicBool::new(false), + state: Mutex::new(State { + status: status.clone(), + deadline_ms, + work, + unpersisted: None, + }), + }), + ); + Ok(status) + } + + fn check_capacity( + &self, + active: &BTreeMap>, + ) -> Result<(), ProtocolError> { + let running = active + .values() + .filter(|operation| !operation.finished.load(Ordering::Acquire)) + .count(); + if running >= self.ceilings.max_active_operations as usize { + return Err(ProtocolError::busy( + Some(BusyReason::TransferCapacity), + None, + format!( + "{running} transfers are active; the ceiling is {}", + self.ceilings.max_active_operations + ), + )); + } + Ok(()) + } + + /// The answer to a repeated begin, or `None` for a new id. + fn existing( + &self, + params: &BeginTransferParams, + ) -> Result, ProtocolError> { + let id = ¶ms.operation_id; + let in_use = || { + ProtocolError::failed_precondition( + PreconditionReason::OperationIdInUse, + format!("transfer {id} was begun with different input"), + ) + }; + let active = self.active_map().get(id).cloned(); + if let Some(operation) = active { + if operation.params != *params { + return Err(in_use()); + } + let status = operation.lock().status.clone(); + return answer_finished_or_active(status, Call::Begin).map(Some); + } + match journal::read(&self.journal, id)? { + None => Ok(None), + Some(record) if record.params != *params => Err(in_use()), + Some(_) => match self.find(id)? { + Found::Finished(status) => answer_finished(status, Call::Begin).map(Some), + Found::Active(operation) => { + let status = operation.lock().status.clone(); + answer_finished_or_active(status, Call::Begin).map(Some) + } + }, + } + } + + fn check_limits(&self, limits: &TransferLimits) -> Result<(), ProtocolError> { + let ceilings = &self.ceilings; + check_quota( + "limits.maxEntries", + limits.max_entries.into(), + ceilings.max_entries.into(), + )?; + let above = |field: &str, value: u64, ceiling: u64| { + if value > ceiling { + Err(ProtocolError::above_ceiling(field, value, ceiling)) + } else { + Ok(()) + } + }; + above( + "limits.maxDepth", + limits.max_depth.into(), + ceilings.max_depth.into(), + )?; + above( + "limits.maxFileBytes", + limits.max_file_bytes, + ceilings.max_file_bytes, + )?; + above( + "limits.maxTotalBytes", + limits.max_total_bytes, + ceilings.max_total_bytes, + )?; + check_quota( + "limits.maxManifestBytes", + limits.max_manifest_bytes, + ceilings.max_manifest_bytes, + )?; + check_quota( + "limits.maxDurationMs", + limits.max_duration_ms, + ceilings.max_duration_ms, + ) + } + + fn start( + &self, + params: &BeginTransferParams, + ) -> Result<(Work, Option>), ProtocolError> { + let confinement = self.confinement.clone(); + let limits = params.limits; + let (destination, kind) = match ¶ms.selection { + TransferSelection::Capture { + source, + filter, + symlinks, + special_files, + } => { + let capture = Capture::begin( + confinement, + source, + filter, + *symlinks, + *special_files, + limits, + )?; + return Ok((Work::Capture(Box::new(capture)), None)); + } + TransferSelection::Materialize { + destination, + on_existing, + } => ( + destination, + Kind::Materialize { + on_existing: *on_existing, + }, + ), + TransferSelection::WriteFile { + destination, + create, + if_match, + executable, + } => ( + destination, + Kind::WriteFile { + create: *create, + if_match: if_match.clone(), + executable: *executable, + }, + ), + }; + let spool = tempfile::Builder::new() + .prefix(&format!("{}.spool-", params.operation_id)) + .tempdir_in(&self.journal) + .map_err(|error| io_error(error, self.journal.display()))?; + let publish = Publish::begin(confinement, destination, kind, limits, spool)?; + let segments = publish.destination_segments.clone(); + Ok((Work::Publish(Box::new(publish)), Some(segments))) + } + + fn advance(&self, id: &TransferOperationId) -> Result { + self.with_active( + id, + Call::Other, + |shared, operation, state| { + let step = match &mut state.work { + Work::Capture(capture) => advance_capture(capture, &mut state.status), + Work::Publish(publish) => advance_publish(publish, &mut state.status), + Work::Finished => Ok(()), + }; + match step { + Ok(()) => Ok(state.status.clone()), + Err(error) => Err(shared.fail(operation, state, error)), + } + }, + Ok, + ) + } + + fn read_inventory( + &self, + params: ReadTransferInventoryParams, + ) -> Result { + check_quota( + "limit", + params.limit.into(), + self.ceilings.max_inventory_page_entries.into(), + )?; + let retained = |status: TransferStatus| { + let index = params.index as usize; + let page = journal::read_inventory( + &self.journal, + ¶ms.operation_id, + index, + params.limit as usize, + )?; + let Some((entries, more)) = page else { + return Err(phase_error("the transfer is finished")); }; - for entry in page { - let content = match entry.content { - InventoryContent::Directory => TransferContent::Directory, - InventoryContent::File { - digest, - size_bytes, - executable, - } => { - let mut data = Vec::with_capacity(size_bytes as usize); - loop { - let TransferResponse::Chunk { data: chunk, eof } = fs - .transfer(TransferRequest::Read { - operation_id: id.clone(), - digest: digest.clone(), - offset: data.len() as u64, - }) - .await? - else { - return Err(invalid("unexpected chunk")); - }; - data.extend(chunk.into_inner()); - if eof { - break; - } - } - bytes += size_bytes; - TransferContent::File { - data: ByteChunk::from(data), - executable, - } + if entries.is_empty() && index > status.entry_count as usize { + return Err(ProtocolError::invalid_field( + "index", + format!("the inventory has {} entries", status.entry_count), + )); + } + let next_index = more.then_some(params.index + entries.len() as u32); + Ok(ReadTransferInventoryResponse { + entries, + next_index, + }) + }; + self.with_active( + ¶ms.operation_id, + Call::ReadInventory, + |_, _, state| { + let Work::Capture(capture) = &state.work else { + return Err(phase_error("only a capture has an inventory to read")); + }; + if state.status.phase != TransferPhase::Inventory { + return Err(phase_error("the capture's inventory is not sealed yet")); + } + let count = capture.entries.len(); + let start = params.index as usize; + if start > count { + return Err(ProtocolError::invalid_field( + "index", + format!("the inventory has {count} entries"), + )); + } + let end = (start + params.limit as usize).min(count); + Ok(ReadTransferInventoryResponse { + entries: capture.entries[start..end].to_vec(), + next_index: (end < count).then_some(end as u32), + }) + }, + retained, + ) + } + + fn append( + &self, + params: AppendTransferInventoryParams, + ) -> Result { + let page = self.ceilings.max_inventory_page_entries; + if params.entries.len() as u64 > u64::from(page) { + return Err(ProtocolError::above_ceiling( + "entries", + params.entries.len() as u64, + page.into(), + )); + } + self.with_active( + ¶ms.operation_id, + Call::Other, + |shared, operation, state| { + let Work::Publish(publish) = &mut state.work else { + return Err(phase_error("a capture takes no inventory")); + }; + // A retried page, the sealing one included, answers the + // current status in any phase. + if publish.is_repeat(params.index, ¶ms.entries, params.last) { + return Ok(state.status.clone()); + } + if state.status.phase != TransferPhase::Inventory { + return Err(phase_error("the inventory is already sealed")); + } + let sealed = publish.append(params.index, params.entries, params.last)?; + state.status.entry_count = publish.entries.len() as u32; + state.status.total_bytes = publish.total_bytes; + if sealed { + if let Err(error) = publish.start_scan() { + return Err(shared.fail(operation, state, error)); } + state.status.phase = TransferPhase::Scanning; + } + Ok(state.status.clone()) + }, + |_| Err(phase_error("the transfer is finished")), + ) + } + + fn list_missing( + &self, + params: ListTransferMissingParams, + ) -> Result { + check_quota( + "limit", + params.limit.into(), + self.ceilings.max_inventory_page_entries.into(), + )?; + self.with_active( + ¶ms.operation_id, + Call::Other, + |_, _, state| { + let Work::Publish(publish) = &state.work else { + return Err(phase_error("a capture needs no content")); }; - entries.push(TransferEntry { - path: entry.path, - content, - }); + if !matches!( + state.status.phase, + TransferPhase::Content | TransferPhase::Staging | TransferPhase::Ready + ) { + return Err(phase_error("the destination scan has not finished")); + } + let missing = publish.missing_digests(); + let limit = params.limit as usize; + Ok(ListTransferMissingResponse { + remaining_count: missing.len().saturating_sub(limit) as u32, + digests: missing.into_iter().take(limit).collect(), + }) + }, + |_| Err(phase_error("the transfer is finished")), + ) + } + + fn read( + &self, + params: ReadTransferContentParams, + ) -> Result { + check_quota( + "maxBytes", + params.max_bytes, + self.ceilings.max_content_chunk_bytes, + )?; + let id = params.operation_id.clone(); + let (operation, source) = self.with_active( + &id, + Call::Other, + |_, operation, state| { + let Work::Capture(capture) = &state.work else { + return Err(phase_error("only a capture serves content")); + }; + if state.status.phase != TransferPhase::Inventory { + return Err(phase_error("the capture's inventory is not sealed yet")); + } + let source = capture + .sources + .get(¶ms.digest) + .cloned() + .ok_or_else(|| { + ProtocolError::not_found(format!("{} is not in the capture", params.digest)) + })?; + if params.offset > source.stat.size { + return Err(ProtocolError::invalid_field( + "offset", + format!("the content has {} bytes", source.stat.size), + )); + } + Ok((operation.clone(), source)) + }, + |_| Err(phase_error("the transfer is finished")), + )?; + let read = + capture::read_content(&self.confinement, &source, params.offset, params.max_bytes); + let mut state = operation.lock(); + let result = match read { + Ok((data, truncated)) => { + if state.status.phase == TransferPhase::Inventory { + state.status.transferred_bytes += data.len() as u64; + } + Ok(ReadTransferContentResponse { + data: ByteChunk::from(data), + truncated, + }) } - if let Some(next) = next_offset { - offset = next; - } else { - break; + Err(ReadFailure::Changed(error) | ReadFailure::Io(error)) => { + if state.status.phase.is_finished() { + Err(error) + } else { + Err(self.fail(&operation, &mut state, error)) + } } + }; + drop(state); + self.evict(&operation); + result + } + + fn write( + &self, + params: WriteTransferContentParams, + ) -> Result { + let chunk = self.ceilings.max_content_chunk_bytes; + if params.data.len() as u64 > chunk { + return Err(ProtocolError::above_ceiling( + "data", + params.data.len() as u64, + chunk, + )); } - fs.transfer(TransferRequest::Commit { - operation_id: id.clone(), - }) - .await?; - Ok(CaptureResponse { - source: params.source, - entries, - bytes, - }) - } - .await; - if result.is_err() { - let _ = fs - .transfer(TransferRequest::Abort { operation_id: id }) - .await; - } - result -} -pub(super) async fn materialize( - fs: &LocalFileSystem, - params: MaterializeParams, -) -> Result { - let quota = limits(params.limits)?; - if params.entries.len() > quota.max_entries as usize { - return Err(invalid("inline entry quota exceeded")); - } - let id = format!("inline-{:032x}", rand::random::()); - let result = async { - let mut content = std::collections::BTreeMap::new(); - let mut entries = Vec::new(); - let mut bytes = 0u64; - for entry in ¶ms.entries { - let kind = match &entry.content { - TransferContent::Directory => InventoryContent::Directory, - TransferContent::File { data, executable } => { - bytes += data.as_slice().len() as u64; - if bytes > quota.max_total_bytes { - return Err(invalid("inline byte quota exceeded")); + let id = params.operation_id.clone(); + let (operation, upload) = self.with_active( + &id, + Call::Other, + |shared, operation, state| { + let Work::Publish(publish) = &mut state.work else { + return Err(phase_error("a capture takes no content")); + }; + if state.status.phase != TransferPhase::Content { + return Err(phase_error("the operation is not collecting content")); + } + let upload = match publish.upload(¶ms.digest) { + Ok(Some(upload)) => upload, + Ok(None) => { + return Err(ProtocolError::not_found(format!( + "{} is not in the inventory", + params.digest + ))); + } + Err(error) => return Err(shared.fail(operation, state, error)), + }; + Ok((operation.clone(), upload)) + }, + |_| Err(phase_error("the transfer is finished")), + )?; + let written = { + let mut upload = upload + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let before = upload.length(); + upload + .write(¶ms.digest, params.offset, params.data.as_slice()) + .map(|next| (next, next.saturating_sub(before))) + }; + let mut state = operation.lock(); + let result = match written { + Ok((next_offset, added)) => { + state.status.transferred_bytes += added; + Ok(WriteTransferContentResponse { next_offset }) + } + Err(ChunkFailure::Offset(error)) => Err(error), + Err(ChunkFailure::Mismatch(error) | ChunkFailure::Io(error)) => { + if state.status.phase.is_finished() { + Err(error) + } else { + Err(self.fail(&operation, &mut state, error)) + } + } + }; + drop(state); + self.evict(&operation); + result + } + + fn commit(&self, id: &TransferOperationId) -> Result { + self.with_active( + id, + Call::Commit, + |shared, operation, state| { + let outcome = match &mut state.work { + // The inventory is retained before the receipt that + // answers for it. + Work::Capture(capture) if state.status.phase == TransferPhase::Inventory => { + journal::write_inventory(&shared.journal, &operation.id, &capture.entries) } - let digest = format!("sha256:{}", hex::encode(Sha256::digest(data.as_slice()))); - content.insert(digest.clone(), data.as_slice()); - InventoryContent::File { - digest, - size_bytes: data.as_slice().len() as u64, - executable: *executable, + Work::Publish(publish) if state.status.phase == TransferPhase::Ready => { + // Recorded first: a restart during the commit must + // not claim that nothing was published. + journal::write( + &shared.journal, + &operation.id, + &Record { + params: operation.params.clone(), + status: state.status.clone(), + stage: publish.created_stage_record(), + spool: publish.spool_path(), + committing: true, + }, + )?; + publish.commit() } + _ => return Err(phase_error("the operation is not ready to commit")), + }; + match outcome { + Ok(()) => { + shared.finish(operation, state, TransferPhase::Complete, None); + Ok(state.status.clone()) + } + Err(error) => Err(shared.fail(operation, state, error)), } - }; - entries.push(InventoryEntry { - path: entry.path.clone(), - content: kind, - }); + }, + Ok, + ) + } + + fn abort(&self, id: &TransferOperationId) -> Result { + self.with_active( + id, + Call::Abort, + |shared, operation, state| { + shared.finish(operation, state, TransferPhase::Aborted, None); + Ok(state.status.clone()) + }, + Ok, + ) + } + + /// Convert operations left active by an earlier daemon instance and + /// drop expired receipts. + fn recover(&self) { + for (id, record) in journal::all(&self.journal) { + if !record.status.phase.is_finished() { + self.interrupted(&id, record); + } else { + self.settle_receipt(&id, record, journal::now_ms()); + } } - if entries.is_empty() { - return Err(invalid("empty inventory")); + journal::remove_orphan_inventories(&self.journal); + // Spools of operations without a record (a crash between creating + // the spool and the first journal write). + if let Ok(entries) = std::fs::read_dir(&self.journal) { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some((id, _)) = name.split_once(".spool-") + && let Ok(id) = TransferOperationId::new(id) + && journal::read(&self.journal, &id).ok().flatten().is_none() + { + let _ = std::fs::remove_dir_all(entry.path()); + } + } } - fs.transfer(TransferRequest::Begin { - operation_id: id.clone(), - selection: TransferSelection::Materialize { - destination: params.destination.clone(), - on_existing: params.on_existing, + } + + /// Remove the staging directory and spool a record names. + fn release(&self, record: &mut Record) { + if let Some(stage) = record.stage.take() { + publish::discard_stage(&self.confinement, &stage); + } + if let Some(spool) = record.spool.take() + && spool.parent() == Some(self.journal.as_path()) + { + let _ = std::fs::remove_dir_all(spool); + } + } + + /// A journaled receipt: removed once expired; otherwise its staging + /// and spool, which the finishing instance removes in the background, + /// are made sure of and dropped from the record. + fn settle_receipt(&self, id: &TransferOperationId, mut record: Record, now: i64) { + let expired = record + .status + .receipt_expires_at_ms + .is_none_or(|expires| expires <= now); + if expired { + self.release(&mut record); + journal::remove(&self.journal, id); + } else if record.stage.is_some() || record.spool.is_some() { + self.release(&mut record); + if let Err(error) = journal::write(&self.journal, id, &record) { + eprintln!("lightspeed-envd: settling the receipt of transfer {id}: {error}"); + } + } + } + + /// Record an operation a restart interrupted as failed (`aborted`) and + /// discard its staging and spool. + fn interrupted(&self, id: &TransferOperationId, mut record: Record) -> TransferStatus { + self.release(&mut record); + record.status.phase = TransferPhase::Failed; + record.status.error = Some(ProtocolError::aborted( + None, + if record.committing { + "the daemon restarted while the transfer was committing; the destination may \ + hold the published tree" + } else { + "the daemon restarted while the transfer was active; nothing was committed" }, - limits: quota, - }) - .await?; - advance(fs, &id, TransferPhase::Inventory).await?; - for (index, page) in entries.chunks(MAX_INVENTORY_PAGE).enumerate() { - fs.transfer(TransferRequest::Append { - operation_id: id.clone(), - offset: (index * MAX_INVENTORY_PAGE) as u32, - entries: page.to_vec(), - last: (index + 1) * MAX_INVENTORY_PAGE >= entries.len(), - }) - .await?; + )); + record.committing = false; + record.status.deadline_at_ms = None; + record.status.receipt_expires_at_ms = + Some(journal::now_ms().saturating_add(self.retention_ms())); + if let Err(error) = journal::write(&self.journal, id, &record) { + eprintln!("lightspeed-envd: recording interrupted transfer {id}: {error}"); } - loop { - let TransferResponse::Missing { digests, .. } = fs - .transfer(TransferRequest::Missing { - operation_id: id.clone(), - offset: 0, - }) - .await? - else { - return Err(invalid("unexpected missing response")); - }; - if digests.is_empty() { - break; - } - for digest in digests { - let data = content[&digest]; - if data.is_empty() { - fs.transfer(TransferRequest::Write { - operation_id: id.clone(), - digest, - offset: 0, - data: ByteChunk::from(vec![]), - }) - .await?; - } else { - for (index, chunk) in data.chunks(MAX_CONTENT_CHUNK).enumerate() { - fs.transfer(TransferRequest::Write { - operation_id: id.clone(), - digest: digest.clone(), - offset: (index * MAX_CONTENT_CHUNK) as u64, - data: ByteChunk::from(chunk), - }) - .await?; - } + record.status + } + + /// Fail overdue operations and remove expired receipts. + fn sweep(&self) { + let operations: Vec<_> = self.active_map().values().cloned().collect(); + for operation in operations { + { + let mut state = operation.lock(); + if !state.status.phase.is_finished() && journal::now_ms() >= state.deadline_ms { + let error = ProtocolError::deadline_exceeded(format!( + "transfer {} ran past its maxDurationMs", + operation.id + )); + self.fail(&operation, &mut state, error); } } + self.evict(&operation); } - advance(fs, &id, TransferPhase::Ready).await?; - fs.transfer(TransferRequest::Commit { - operation_id: id.clone(), - }) - .await?; - Ok(MaterializeResponse { - destination: params.destination, - entries: entries.len() as u32, - bytes, - retired_directory: None, - }) - } - .await; - if result.is_err() { - let _ = fs - .transfer(TransferRequest::Abort { operation_id: id }) - .await; - } - result + let now = journal::now_ms(); + // Receipts that finished at least a sweep ago: the background + // removal of their staging had its time. + let finished_before = now.saturating_sub(SWEEP_INTERVAL.as_millis() as i64); + let retention = self.retention_ms(); + for (id, record) in journal::all(&self.journal) { + let due = record.status.receipt_expires_at_ms.is_none_or(|expires| { + expires <= now || expires.saturating_sub(retention) <= finished_before + }); + if record.status.phase.is_finished() && due && !self.active_map().contains_key(&id) { + self.settle_receipt(&id, record, now); + } + } + } +} + +fn sweep_loop(shared: Weak) { + loop { + std::thread::sleep(SWEEP_INTERVAL); + let Some(shared) = shared.upgrade() else { + return; + }; + shared.sweep(); + } +} + +/// Undo a begin refused before its operation was recorded. +fn discard(work: Work) { + if let Work::Publish(publish) = work { + publish.discard(); + } +} + +fn phase_error(message: &str) -> ProtocolError { + ProtocolError::failed_precondition(PreconditionReason::TransferPhase, message) +} + +/// A finished operation's answer: its status for abort, an identical begin +/// and (when complete) commit and inventory reads; its error when it failed; otherwise a phase +/// error. `transfer/getStatus` always answers the status. +fn answer_finished(status: TransferStatus, call: Call) -> Result { + match (call, status.phase) { + (Call::Abort, _) => Ok(status), + (_, TransferPhase::Failed) => Err(status + .error + .clone() + .unwrap_or_else(|| ProtocolError::internal("the transfer failed"))), + (Call::Begin, _) | (Call::Commit | Call::ReadInventory, TransferPhase::Complete) => { + Ok(status) + } + _ => Err(phase_error("the transfer is finished")), + } +} + +fn answer_finished_or_active( + status: TransferStatus, + call: Call, +) -> Result { + if status.phase.is_finished() { + answer_finished(status, call) + } else { + Ok(status) + } +} + +fn advance_capture( + capture: &mut Capture, + status: &mut TransferStatus, +) -> Result<(), ProtocolError> { + if status.phase == TransferPhase::Scanning && capture.advance()? { + status.phase = TransferPhase::Inventory; + } + status.entry_count = capture.entries.len() as u32; + status.total_bytes = capture.total_bytes; + status.skipped = capture.skipped.clone(); + status.skipped_entry_count = capture.skipped_count; + Ok(()) +} + +fn advance_publish( + publish: &mut Publish, + status: &mut TransferStatus, +) -> Result<(), ProtocolError> { + if status.phase == TransferPhase::Scanning && publish.scan_step()? { + status.phase = TransferPhase::Content; + } + if status.phase == TransferPhase::Content && publish.missing_digests().is_empty() { + status.phase = TransferPhase::Staging; + } + if status.phase == TransferPhase::Staging && publish.stage_step()? { + status.phase = TransferPhase::Ready; + } + status.reused_bytes = publish.reused_bytes; + Ok(()) } diff --git a/crates/environment-daemon/src/filesystem/transfer/publish.rs b/crates/environment-daemon/src/filesystem/transfer/publish.rs new file mode 100644 index 000000000..1435f6bbe --- /dev/null +++ b/crates/environment-daemon/src/filesystem/transfer/publish.rs @@ -0,0 +1,970 @@ +//! Materialize and writeFile: append the incoming inventory, scan the +//! destination (hashing only files whose size matches an incoming file), +//! collect missing content, stage the tree next to the destination and +//! publish it with one rename at commit. + +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + ffi::{OsStr, OsString}, + fs::File, + io::{Read, Seek, SeekFrom, Write}, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use environment_protocol::{ + data::{ + fs::{FileKind, FileVersion, TreeContent, WriteCondition}, + transfer::{InventoryEntry, TransferLimits, TransferOnExisting}, + }, + error::{PreconditionReason, ProtocolError}, + paths::{ContentDigest, EnvironmentPath, TreePath}, +}; +use sha2::{Digest, Sha256}; + +use crate::filesystem::{ + confinement::{Confinement, Lookup, Resolved}, + io_error, + os::{self, DEFAULT_DIRECTORY_MODE, DEFAULT_FILE_MODE, Directory, EntryStat, RenameMode}, + walk::{CompiledFilter, Event, LinkMode, Walker}, + write_preconditions, +}; + +use super::{ + STEP_BYTES, STEP_ENTRIES, + capture::{Source, source_changed}, + check_entry_quota, entry_bytes, + journal::StageRecord, +}; + +/// Prefix of staging directories next to a destination. +pub(super) const STAGE_PREFIX: &str = ".lightspeed-transfer-"; +/// The staged tree inside a staging directory. +const STAGED_TREE: &str = "tree"; + +/// What a publish writes. +#[derive(Clone, Debug)] +pub(super) enum Kind { + Materialize { + on_existing: TransferOnExisting, + }, + WriteFile { + create: WriteCondition, + if_match: Option, + executable: bool, + }, +} + +struct Stage { + parent: Arc, + name: OsString, + directory: Directory, +} + +/// Content uploaded for one digest, spooled under the journal directory. +pub(super) struct Upload { + file: File, + path: PathBuf, + hash: Sha256, + length: u64, + size: u64, + complete: bool, +} + +/// Why one uploaded chunk was refused. +pub(super) enum ChunkFailure { + /// The offset is not the digest's next offset and the chunk is not an + /// identical repeat; nothing changed. + Offset(ProtocolError), + /// The completed content does not hash to its digest. + Mismatch(ProtocolError), + Io(ProtocolError), +} + +impl Upload { + pub fn is_complete(&self) -> bool { + self.complete + } + + /// Bytes written so far: the next expected offset. + pub fn length(&self) -> u64 { + self.length + } + + /// Append a chunk at `offset`; returns the next expected offset. + pub fn write( + &mut self, + digest: &ContentDigest, + offset: u64, + data: &[u8], + ) -> Result { + let io = |error| ChunkFailure::Io(io_error(error, digest)); + let end = offset.saturating_add(data.len() as u64); + if offset < self.length { + // An identical repeat of bytes already written is accepted. + if end > self.length { + return Err(ChunkFailure::Offset(offset_error(digest, self.length))); + } + let mut existing = vec![0; data.len()]; + self.file.seek(SeekFrom::Start(offset)).map_err(io)?; + self.file.read_exact(&mut existing).map_err(io)?; + if existing != data { + return Err(ChunkFailure::Offset(offset_error(digest, self.length))); + } + return Ok(self.length); + } + if offset != self.length || self.complete || end > self.size { + return Err(ChunkFailure::Offset(offset_error(digest, self.length))); + } + self.file.seek(SeekFrom::Start(offset)).map_err(io)?; + self.file.write_all(data).map_err(io)?; + self.hash.update(data); + self.length = end; + if self.length == self.size { + let actual = ContentDigest::from_sha256(self.hash.clone().finalize().into()); + if actual != *digest { + return Err(ChunkFailure::Mismatch(ProtocolError::failed_precondition( + PreconditionReason::DigestMismatch, + format!("uploaded content hashes to {actual}, not {digest}"), + ))); + } + self.file.sync_all().map_err(io)?; + self.complete = true; + } + Ok(self.length) + } +} + +fn offset_error(digest: &ContentDigest, next: u64) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::TransferOffset, + format!("the next offset of {digest} is {next}"), + ) +} + +struct Copying { + source: File, + destination: File, + hash: Sha256, + expected: ContentDigest, + copied: u64, + size: u64, + mode: u32, + changed: Option, +} + +struct DestinationScan { + walker: Option, + single: Option, + hashing: Option<(Source, File, Sha256, u64)>, +} + +pub(super) struct Publish { + confinement: Arc, + pub kind: Kind, + destination: EnvironmentPath, + /// The destination's parent directory and its name there. + parent: Arc, + name: OsString, + parent_segments: Vec, + pub destination_segments: Vec, + /// Directories `begin` created for the destination; removed again + /// unless a commit published into them. + created_parents: Vec>, + /// Name of the staging directory, chosen at begin and created by + /// [`Publish::create_stage`] once it is journaled. + stage_name: OsString, + stage: Option, + limits: TransferLimits, + pub entries: Vec, + pages: BTreeMap, bool)>, + paths: HashMap, + sizes: BTreeMap, + pub total_bytes: u64, + manifest_bytes: u64, + scan: Option, + reusable: BTreeMap, + pub uploads: BTreeMap>>, + spool: Option, + staging_index: usize, + copying: Option, + pub reused_bytes: u64, +} + +impl Publish { + pub fn begin( + confinement: Arc, + destination: &EnvironmentPath, + kind: Kind, + limits: TransferLimits, + spool: tempfile::TempDir, + ) -> Result { + let resolved = match &kind { + Kind::Materialize { on_existing } => { + let resolved = confinement.lookup( + destination, + Lookup::NO_FOLLOW.allowing_missing().creating_parents(), + )?; + if let Some(stat) = &resolved.stat { + if stat.kind == FileKind::Symlink { + return Err(ProtocolError::failed_precondition( + PreconditionReason::SymlinkRejected, + format!("{destination} is a symlink"), + )); + } + if stat.kind == FileKind::Other { + return Err(ProtocolError::failed_precondition( + PreconditionReason::NotRegularFile, + format!("{destination} is a special file"), + )); + } + if *on_existing == TransferOnExisting::Error { + return Err(ProtocolError::already_exists(format!( + "{destination} exists" + ))); + } + } + resolved + } + Kind::WriteFile { + create, if_match, .. + } => { + let resolved = + confinement.lookup(destination, Lookup::FOLLOW.allowing_missing())?; + write_preconditions( + destination, + resolved.stat.as_ref(), + *create, + if_match.as_ref(), + )?; + resolved + } + }; + if resolved.is_root() { + return Err(ProtocolError::failed_precondition( + PreconditionReason::FilesystemRoot, + "the filesystem root cannot be replaced", + )); + } + if let Err(error) = confinement.refuse_containing_state(&resolved, destination) { + confinement.remove_created(&resolved.created); + return Err(error); + } + let Resolved { + parent, + name, + segments, + created, + .. + } = resolved; + Ok(Self { + confinement, + kind, + destination: destination.clone(), + parent_segments: segments[..segments.len() - 1].to_vec(), + destination_segments: segments, + created_parents: created, + parent: Arc::new(parent), + name, + stage_name: OsString::from(format!("{STAGE_PREFIX}{:032x}", rand::random::())), + stage: None, + limits, + entries: Vec::new(), + pages: BTreeMap::new(), + paths: HashMap::new(), + sizes: BTreeMap::new(), + total_bytes: 0, + manifest_bytes: 0, + scan: None, + reusable: BTreeMap::new(), + uploads: BTreeMap::new(), + spool: Some(spool), + staging_index: 0, + copying: None, + reused_bytes: 0, + }) + } + + /// Create the staging directory next to the destination. Called once + /// its record is journaled, so a restart always finds and removes it. + pub fn create_stage(&mut self) -> Result<(), ProtocolError> { + let io = |error| io_error(error, &self.destination); + self.parent + .mkdir(&self.stage_name, os::PRIVATE_DIRECTORY_MODE) + .map_err(io)?; + let directory = self.parent.child(&self.stage_name).map_err(io)?; + self.stage = Some(Stage { + parent: self.parent.clone(), + name: self.stage_name.clone(), + directory, + }); + Ok(()) + } + + /// Remove everything `begin` created, at once: a begin refused before + /// the operation was recorded leaves nothing behind. + pub fn discard(mut self) { + if let Some(stage) = self.stage.take() { + remove_stage(stage); + } + self.confinement + .remove_created(&std::mem::take(&mut self.created_parents)); + drop(self.spool.take()); + } + + /// Where the staging directory is (or is about to be) created. + pub fn stage_record(&self) -> StageRecord { + StageRecord::new( + &self.parent_segments, + self.stage_name.to_str().expect("stage names are ASCII"), + ) + } + + /// [`Self::stage_record`] once the staging directory exists. + pub fn created_stage_record(&self) -> Option { + self.stage.as_ref().map(|_| self.stage_record()) + } + + /// The page at `index` was already appended with these entries. + pub fn is_repeat(&self, index: u32, entries: &[InventoryEntry], last: bool) -> bool { + self.pages + .get(&index) + .is_some_and(|(previous, sealed)| previous == entries && *sealed == last) + } + + pub fn spool_path(&self) -> Option { + self.spool.as_ref().map(|spool| spool.path().to_owned()) + } + + /// Validate and add one inventory page; `Ok(true)` when it sealed the + /// inventory. A repeat of an identical page changes nothing. + pub fn append( + &mut self, + index: u32, + entries: Vec, + last: bool, + ) -> Result { + if let Some((previous, sealed)) = self.pages.get(&index) { + if *previous == entries && *sealed == last { + return Ok(false); + } + return Err(ProtocolError::failed_precondition( + PreconditionReason::TransferOffset, + format!("page {index} was appended with different entries"), + )); + } + if index as usize != self.entries.len() { + return Err(ProtocolError::failed_precondition( + PreconditionReason::TransferOffset, + format!("the next inventory index is {}", self.entries.len()), + )); + } + if entries.is_empty() && !(last && !self.entries.is_empty()) { + return Err(ProtocolError::invalid_field( + "entries", + "an inventory page needs at least one entry", + )); + } + let single_file = matches!(self.kind, Kind::WriteFile { .. }); + let mut paths = HashMap::new(); + let mut sizes = BTreeMap::new(); + let mut total = self.total_bytes; + let mut manifest = self.manifest_bytes; + for (offset, entry) in entries.iter().enumerate() { + let position = self.entries.len() + offset; + let field = |member: &str| format!("entries[{offset}]{member}"); + if entry.path.depth() > self.limits.max_depth { + return Err(ProtocolError::invalid_field( + field(".path"), + "deeper than limits.maxDepth", + )); + } + if position == 0 { + if !entry.path.is_root() { + return Err(ProtocolError::invalid_field( + field(".path"), + "the inventory starts with the root entry `.`", + )); + } + } else { + if single_file { + return Err(ProtocolError::invalid_field( + "entries", + "a writeFile inventory holds exactly one file", + )); + } + if self.paths.contains_key(&entry.path) || paths.contains_key(&entry.path) { + return Err(ProtocolError::invalid_field( + field(".path"), + "duplicate path", + )); + } + let parent = parent_of(&entry.path); + let parent_is_directory = self + .paths + .get(&parent) + .or_else(|| paths.get(&parent)) + .copied() + .unwrap_or(false); + if !parent_is_directory { + return Err(ProtocolError::invalid_field( + field(".path"), + "a parent directory must precede its children", + )); + } + } + let is_directory = matches!(entry.content, TreeContent::Directory {}); + if single_file && is_directory { + return Err(ProtocolError::invalid_field( + field(".content"), + "a writeFile inventory holds one file", + )); + } + paths.insert(entry.path.clone(), is_directory); + if let TreeContent::File { + size_bytes, digest, .. + } = &entry.content + { + let Some(digest) = digest else { + return Err(ProtocolError::invalid_field( + field(".content.digest"), + "inventory files carry their digest", + )); + }; + if *size_bytes > self.limits.max_file_bytes { + return Err(ProtocolError::invalid_field( + field(".content.sizeBytes"), + "larger than limits.maxFileBytes", + )); + } + let known = self.sizes.get(digest).or_else(|| sizes.get(digest)); + if known.is_some_and(|known| known != size_bytes) { + return Err(ProtocolError::invalid_field( + field(".content.sizeBytes"), + "one digest with two sizes", + )); + } + sizes.insert(*digest, *size_bytes); + total = total.saturating_add(*size_bytes); + } + manifest += entry_bytes(entry); + } + check_entry_quota(self.entries.len() + entries.len(), &self.limits)?; + if total > self.limits.max_total_bytes { + return Err(ProtocolError::invalid_field( + "entries", + "the inventory exceeds limits.maxTotalBytes", + )); + } + if manifest > self.limits.max_manifest_bytes { + return Err(ProtocolError::invalid_field( + "entries", + "the inventory exceeds limits.maxManifestBytes", + )); + } + if single_file && !last { + return Err(ProtocolError::invalid_field( + "last", + "a writeFile inventory is one page", + )); + } + self.pages.insert(index, (entries.clone(), last)); + self.paths.extend(paths); + self.sizes.extend(sizes); + self.entries.extend(entries); + self.total_bytes = total; + self.manifest_bytes = manifest; + Ok(last) + } + + /// Start the destination scan after the inventory was sealed. + pub fn start_scan(&mut self) -> Result<(), ProtocolError> { + let current = self.parent.stat(&self.name); + let stat = match current { + Ok(stat) => Some(stat), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(io_error(error, &self.destination)), + }; + let mut scan = DestinationScan { + walker: None, + single: None, + hashing: None, + }; + match stat { + Some(stat) if stat.is_dir() && matches!(self.kind, Kind::Materialize { .. }) => { + let resolved = self + .confinement + .reopen(&self.destination_segments, &self.destination)?; + scan.walker = Some(Walker::new( + self.confinement.clone(), + &resolved, + CompiledFilter::everything(), + LinkMode::Report, + None, + &self.destination, + )?); + } + Some(stat) if stat.is_file() => { + scan.single = Some(Source { + canonical: self.destination_segments.clone(), + stat, + }); + } + _ => {} + } + self.scan = Some(scan); + Ok(()) + } + + /// One bounded step of the destination scan; `true` when it is done. + /// Reuse is an optimisation: unreadable destination entries are left + /// out, never fatal. + pub fn scan_step(&mut self) -> Result { + let wanted: HashSet = self + .missing_digests() + .iter() + .map(|digest| self.sizes[digest]) + .collect(); + let mut allowance = STEP_BYTES; + let Some(scan) = self.scan.as_mut() else { + return Ok(true); + }; + for _ in 0..STEP_ENTRIES { + if let Some((_, file, hash, read)) = scan.hashing.as_mut() { + let mut buffer = vec![0u8; 64 * 1024]; + let mut finished = false; + while allowance > 0 { + let limit = (buffer.len() as u64).min(allowance) as usize; + match file.read(&mut buffer[..limit]) { + Ok(0) => { + finished = true; + break; + } + Ok(count) => { + hash.update(&buffer[..count]); + *read += count as u64; + allowance -= count as u64; + } + Err(_) => { + scan.hashing = None; + break; + } + } + } + if !finished { + if scan.hashing.is_some() { + return Ok(false); + } + continue; + } + let (source, file, hash, read) = scan.hashing.take().expect("hashing above"); + let digest = ContentDigest::from_sha256(hash.finalize().into()); + let unchanged = EntryStat::of_file(&file) + .is_ok_and(|stat| read == source.stat.size && stat.unchanged(&source.stat)); + if unchanged && self.sizes.contains_key(&digest) { + self.reusable.entry(digest).or_insert(source); + } + continue; + } + let candidate = if let Some(single) = scan.single.take() { + Some(( + single.clone(), + self.confinement + .reopen(&single.canonical, &self.destination), + )) + } else if let Some(walker) = scan.walker.as_mut() { + match walker.next_event() { + Some(Event::Entry(entry)) + if entry.stat.is_file() && wanted.contains(&entry.stat.size) => + { + let source = Source { + canonical: entry.canonical.clone(), + stat: entry.stat, + }; + let file = entry.parent.open_read(&entry.name); + if let Ok(file) = file { + scan.hashing = Some((source, file, Sha256::new(), 0)); + } + continue; + } + Some(_) => continue, + None => { + scan.walker = None; + None + } + } + } else { + None + }; + match candidate { + Some((source, Ok(resolved))) if wanted.contains(&source.stat.size) => { + if let Ok(file) = resolved.parent.open_read(&resolved.name) { + scan.hashing = Some((source, file, Sha256::new(), 0)); + } + } + Some(_) => {} + None => { + self.scan = None; + return Ok(true); + } + } + } + Ok(false) + } + + /// Digests whose content is neither empty, reusable nor uploaded. + pub fn missing_digests(&self) -> Vec { + self.sizes + .iter() + .filter(|(digest, size)| { + **size > 0 + && !self.reusable.contains_key(*digest) + && !self.uploads.get(*digest).is_some_and(|upload| { + upload.lock().is_ok_and(|upload| upload.is_complete()) + }) + }) + .map(|(digest, _)| *digest) + .collect() + } + + /// The upload of `digest`, created on first use. `None` when the digest + /// is not in the inventory. + pub fn upload( + &mut self, + digest: &ContentDigest, + ) -> Result>>, ProtocolError> { + let Some(size) = self.sizes.get(digest).copied() else { + return Ok(None); + }; + if let Some(upload) = self.uploads.get(digest) { + return Ok(Some(upload.clone())); + } + let spool = self + .spool + .as_ref() + .ok_or_else(|| ProtocolError::internal("the upload spool is gone"))?; + let path = spool.path().join(digest.hex()); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .map_err(|error| io_error(error, "the upload spool"))?; + let upload = Arc::new(Mutex::new(Upload { + file, + path, + hash: Sha256::new(), + length: 0, + size, + complete: false, + })); + self.uploads.insert(*digest, upload.clone()); + Ok(Some(upload)) + } + + /// One bounded staging step; `true` when the tree is staged. + pub fn stage_step(&mut self) -> Result { + let mut allowance = STEP_BYTES; + for _ in 0..STEP_ENTRIES { + if self.copying.is_some() { + if !self.copy_step(&mut allowance)? { + return Ok(false); + } + continue; + } + let Some(entry) = self.entries.get(self.staging_index).cloned() else { + return Ok(true); + }; + let stage = self.stage.as_ref().expect("staging before commit"); + let (directory, name) = staged_parent(&stage.directory, &entry.path) + .map_err(|error| io_error(error, &entry.path))?; + match entry.content { + TreeContent::Directory {} => { + directory + .mkdir(&name, DEFAULT_DIRECTORY_MODE) + .map_err(|error| io_error(error, &entry.path))?; + directory + .sync() + .map_err(|error| io_error(error, &entry.path))?; + self.staging_index += 1; + } + TreeContent::File { + size_bytes, + executable, + digest, + } => { + let digest = digest.expect("appended files carry digests"); + let mode = if executable { + os::with_execute_bits(DEFAULT_FILE_MODE) + } else { + DEFAULT_FILE_MODE + }; + let destination = directory + .create_file(&name, os::PRIVATE_FILE_MODE) + .map_err(|error| io_error(error, &entry.path))?; + let (source, changed) = self.content_source(&digest, size_bytes)?; + self.copying = Some(Copying { + source, + destination, + hash: Sha256::new(), + expected: digest, + copied: 0, + size: size_bytes, + mode, + changed, + }); + } + } + } + Ok(false) + } + + /// The bytes of `digest`: an upload, a reusable destination file or + /// nothing for empty content (an empty spool file stands in). + fn content_source( + &mut self, + digest: &ContentDigest, + size: u64, + ) -> Result<(File, Option), ProtocolError> { + if let Some(upload) = self.uploads.get(digest) { + let upload = upload + .lock() + .map_err(|_| ProtocolError::internal("upload lock poisoned"))?; + if upload.complete { + let file = File::open(&upload.path) + .map_err(|error| io_error(error, "the upload spool"))?; + return Ok((file, None)); + } + } + if let Some(source) = self.reusable.get(digest).cloned() { + let resolved = self + .confinement + .reopen(&source.canonical, &self.destination) + .map_err(|_| source_changed(&self.destination))?; + let file = resolved + .parent + .open_read(&resolved.name) + .map_err(|error| io_error(error, &self.destination))?; + if !EntryStat::of_file(&file).is_ok_and(|stat| stat.unchanged(&source.stat)) { + return Err(source_changed(&self.destination)); + } + self.reused_bytes += size; + return Ok((file, Some(source))); + } + if size == 0 { + let spool = self + .spool + .as_ref() + .ok_or_else(|| ProtocolError::internal("the upload spool is gone"))?; + let path = spool.path().join("empty"); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(path) + .map_err(|error| io_error(error, "the upload spool"))?; + return Ok((file, None)); + } + Err(ProtocolError::internal(format!( + "content of {digest} is missing at staging" + ))) + } + + fn copy_step(&mut self, allowance: &mut u64) -> Result { + let copy = self.copying.as_mut().expect("caller checked"); + let mut buffer = vec![0u8; 64 * 1024]; + loop { + if *allowance == 0 { + return Ok(false); + } + let limit = (buffer.len() as u64).min(*allowance) as usize; + let read = copy + .source + .read(&mut buffer[..limit]) + .map_err(|error| io_error(error, &self.destination))?; + if read == 0 { + break; + } + copy.copied += read as u64; + if copy.copied > copy.size { + return Err(copy_mismatch(copy, &self.destination)); + } + copy.hash.update(&buffer[..read]); + copy.destination + .write_all(&buffer[..read]) + .map_err(|error| io_error(error, &self.destination))?; + *allowance -= read as u64; + } + let copy = self.copying.take().expect("copying above"); + let digest = ContentDigest::from_sha256(copy.hash.clone().finalize().into()); + if copy.copied != copy.size || digest != copy.expected { + return Err(copy_mismatch(©, &self.destination)); + } + os::set_mode(©.destination, copy.mode) + .and_then(|()| copy.destination.sync_all()) + .map_err(|error| io_error(error, &self.destination))?; + self.staging_index += 1; + Ok(true) + } + + /// Publish the staged tree with one rename. + pub fn commit(&mut self) -> Result<(), ProtocolError> { + let stage = self.stage.as_ref().expect("commit before release"); + let staged = OsStr::new(STAGED_TREE); + let io = |error| io_error(error, &self.destination); + match &self.kind { + Kind::Materialize { on_existing } => { + let published = + stage + .directory + .rename(staged, &self.parent, &self.name, RenameMode::NoReplace); + match published { + Ok(()) => {} + Err(error) + if error.kind() == std::io::ErrorKind::AlreadyExists + && *on_existing == TransferOnExisting::Replace => + { + // The previous tree lands in the staging directory, + // which is removed with the operation. + stage + .directory + .rename(staged, &self.parent, &self.name, RenameMode::Exchange) + .map_err(io)?; + } + Err(error) => return Err(io(error)), + } + } + Kind::WriteFile { + create, + if_match, + executable, + } => { + let current = match self.parent.stat(&self.name) { + Ok(stat) => Some(stat), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(io(error)), + }; + write_preconditions( + &self.destination, + current.as_ref(), + *create, + if_match.as_ref(), + )?; + let base = current.map_or(DEFAULT_FILE_MODE, |stat| stat.mode); + let mode = if *executable { + os::with_execute_bits(base) + } else { + base + }; + let file = stage.directory.open_read(staged).map_err(io)?; + os::set_mode(&file, mode) + .and_then(|()| file.sync_all()) + .map_err(io)?; + let rename = if *create == WriteCondition::IfAbsent { + RenameMode::NoReplace + } else { + RenameMode::Replace + }; + stage + .directory + .rename(staged, &self.parent, &self.name, rename) + .map_err(io)?; + } + } + self.parent.sync().map_err(io)?; + // The published tree lives in the directories begin created. + self.created_parents.clear(); + Ok(()) + } +} + +impl Drop for Publish { + fn drop(&mut self) { + let stage = self.stage.take(); + let created = std::mem::take(&mut self.created_parents); + if stage.is_some() || !created.is_empty() { + // A replaced tree may be large; removal never delays a receipt. + // The receipt keeps the stage's record until a sweep confirms + // it is gone, so an exit before this thread finishes leaks + // nothing. + let confinement = self.confinement.clone(); + let _ = std::thread::Builder::new() + .name("transfer-stage-cleanup".into()) + .spawn(move || { + if let Some(stage) = stage { + remove_stage(stage); + } + confinement.remove_created(&created); + }); + } + if let Some(spool) = self.spool.take() { + let _ = std::thread::Builder::new() + .name("transfer-spool-cleanup".into()) + .spawn(move || drop(spool)); + } + } +} + +fn remove_stage(stage: Stage) { + drop(stage.directory); + if let Err(error) = stage.parent.remove_tree(&stage.name) + && error.kind() != std::io::ErrorKind::NotFound + { + eprintln!( + "lightspeed-envd: removing transfer staging {}: {error}", + stage.name.to_string_lossy() + ); + } +} + +fn copy_mismatch(copy: &Copying, destination: &EnvironmentPath) -> ProtocolError { + match ©.changed { + Some(_) => source_changed(destination), + None => ProtocolError::failed_precondition( + PreconditionReason::DigestMismatch, + format!( + "staged content of {} does not match its digest", + copy.expected + ), + ), + } +} + +fn parent_of(path: &TreePath) -> TreePath { + match path.as_str().rsplit_once('/') { + Some((parent, _)) => TreePath::new(parent).expect("a parent of a tree path is one"), + None => TreePath::root(), + } +} + +/// The directory that holds `path` in the staged tree, and its name there. +fn staged_parent(stage: &Directory, path: &TreePath) -> std::io::Result<(Directory, OsString)> { + if path.is_root() { + return Ok((stage.try_clone()?, OsString::from(STAGED_TREE))); + } + let mut directory = stage.child(OsStr::new(STAGED_TREE))?; + let segments: Vec<&str> = path.segments().collect(); + let (name, parents) = segments + .split_last() + .expect("a non-root path has a segment"); + for segment in parents { + directory = directory.child(OsStr::new(segment))?; + } + Ok((directory, OsString::from(name))) +} + +/// Remove a staging directory recorded by an operation a restart +/// interrupted. +pub(super) fn discard_stage(confinement: &Confinement, record: &StageRecord) { + if !record.name.starts_with(STAGE_PREFIX) || record.name.contains('/') { + return; + } + let segments = record.parent_segments(); + let Ok(resolved) = confinement.reopen(&segments, &"transfer staging") else { + return; + }; + if let Ok(parent) = resolved.open_directory() { + let _ = parent.remove_tree(OsStr::new(&record.name)); + } +} diff --git a/crates/environment-daemon/src/filesystem/transfer/session.rs b/crates/environment-daemon/src/filesystem/transfer/session.rs deleted file mode 100644 index e09db6002..000000000 --- a/crates/environment-daemon/src/filesystem/transfer/session.rs +++ /dev/null @@ -1,1531 +0,0 @@ -//! Bounded transfer sessions retain inventories and publish through the filesystem backend. -use crate::filesystem::backend::{ - self, Result, conflict, digest, error, invalid, io, relative, valid_digest, valid_path, -}; -use environment_protocol::{ - data::{inventory::*, transfer::TransferOnExisting, transfer_session::*}, - error::EnvironmentProtocolErrorCode as Code, - shared::ByteChunk, -}; -use sha2::{Digest, Sha256}; -use std::{ - collections::BTreeMap, - fs::File, - io::{Read, Seek, SeekFrom, Write}, - path::Path, - time::{Duration, Instant}, -}; -#[path = "journal.rs"] -mod journal; -use backend::{Directory, Observation}; -#[derive(Clone)] -struct Source { - path: String, - observed: Observation, -} -struct Hashing { - source: Source, - file: File, - hash: Sha256, - read: u64, -} -struct Scanner { - anchor: Option, - pending: Vec, - pending_bytes: usize, - hashing: Option, - entries: Vec, - files: BTreeMap, - observed: Vec, - limits: InventoryLimits, - bytes: u64, - manifest_bytes: u64, - selected: String, - ignore_errors: bool, - patterns: Vec, - visited: u32, - metadata_only: bool, - skipped: bool, -} -impl Scanner { - fn new( - anchor: Directory, - selected: String, - limits: InventoryLimits, - ignore_errors: bool, - ) -> Self { - Self { - anchor: Some(anchor), - pending_bytes: selected.len(), - pending: vec![selected.clone()], - hashing: None, - entries: vec![], - files: BTreeMap::new(), - observed: vec![], - limits, - bytes: 0, - manifest_bytes: 0, - selected, - ignore_errors, - patterns: vec![], - visited: 0, - metadata_only: false, - skipped: false, - } - } - fn record(&mut self, source: Source, content: InventoryContent) -> Result<()> { - let relative = source - .path - .strip_prefix(&self.selected) - .unwrap() - .trim_start_matches('/') - .to_owned(); - if !valid_path(&relative) { - return Err(invalid("invalid inventory path")); - } - let entry = InventoryEntry { - path: relative, - content, - }; - self.manifest_bytes += serde_json::to_vec(&entry) - .map_err(|e| invalid(&e.to_string()))? - .len() as u64; - if self.manifest_bytes > self.limits.max_manifest_bytes { - return Err(invalid("manifest memory quota exceeded")); - } - self.entries.push(entry); - self.observed.push(source); - Ok(()) - } - /// At most 4 MiB of hashing and 128 visited nodes per exchange. Directory - /// enumeration is additionally bounded by the inventory entry/manifest quotas. - fn advance(&mut self) -> Result { - let mut allowance = 4 * 1024 * 1024; - for _ in 0..MAX_INVENTORY_PAGE { - if let Some(mut task) = self.hashing.take() { - let mut buffer = [0u8; 64 * 1024]; - while allowance > 0 { - let limit = buffer.len().min(allowance); - let read = task.file.read(&mut buffer[..limit]).map_err(io)?; - if read == 0 { - if task.read != task.source.observed.size() - || !task - .source - .observed - .matches(&backend::observe(&task.file).map_err(io)?) - { - return Err(conflict()); - } - let hash = digest(task.hash); - self.files - .entry(hash.clone()) - .or_insert_with(|| task.source.clone()); - let content = InventoryContent::File { - size_bytes: task.read, - executable: task.source.observed.executable(), - digest: hash, - }; - self.record(task.source, content)?; - break; - } - task.read += read as u64; - if task.read > task.source.observed.size() { - return Err(conflict()); - } - task.hash.update(&buffer[..read]); - allowance -= read; - if allowance == 0 { - self.hashing = Some(task); - return Ok(false); - } - } - continue; - } - let Some(path) = self.pending.pop() else { - return Ok(true); - }; - self.pending_bytes -= path.len(); - let anchor = self.anchor.as_ref().unwrap(); - let file = match if self.metadata_only { - anchor.metadata(&path) - } else { - anchor.open(&path) - } { - Ok(file) => file, - Err(_) if self.ignore_errors => { - self.skipped = true; - continue; - } - Err(e) => return Err(io(e)), - }; - let observed = backend::observe(&file).map_err(io)?; - self.visited += 1; - if self.visited > self.limits.max_entries { - return Err(invalid("entry quota exceeded")); - } - let rel = path - .strip_prefix(&self.selected) - .unwrap() - .trim_start_matches('/'); - if !valid_path(rel) - || (!rel.is_empty() && rel.split('/').count() > self.limits.max_depth as usize) - { - return Err(invalid("path/depth quota exceeded")); - } - let source = Source { - path: path.clone(), - observed, - }; - if source.observed.is_dir() { - let names = - Directory::names(&file, self.limits.max_entries as usize).map_err(io)?; - if self.pending.len() + self.entries.len() + names.len() - > self.limits.max_entries as usize - { - return Err(invalid("entry quota exceeded")); - } - // Bound the pending traversal too, before any names enter the queue. - let pending_bytes = self.pending_bytes; - let added_bytes: usize = names - .iter() - .map(|n| path.len() + usize::from(!path.is_empty()) + n.len()) - .sum(); - if (pending_bytes + added_bytes) as u64 > self.limits.max_manifest_bytes { - return Err(invalid("traversal memory quota exceeded")); - } - self.pending_bytes += added_bytes; - for name in names.into_iter().rev() { - if !valid_path(&name) { - return Err(invalid("unsupported filename")); - } - self.pending.push(if path.is_empty() { - name - } else { - format!("{path}/{name}") - }); - } - self.record(source, InventoryContent::Directory)?; - } else { - if !self.patterns.is_empty() && !self.patterns.iter().any(|p| p.matches(rel)) { - continue; - } - self.bytes = self - .bytes - .checked_add(source.observed.size()) - .ok_or_else(|| invalid("size overflow"))?; - if source.observed.size() > self.limits.max_file_bytes - || self.bytes > self.limits.max_total_bytes - { - return Err(invalid("content quota exceeded")); - } - if self.metadata_only { - let content = InventoryContent::File { - size_bytes: source.observed.size(), - executable: source.observed.executable(), - digest: String::new(), - }; - self.record(source, content)?; - continue; - } - self.hashing = Some(Hashing { - source, - file, - hash: Sha256::new(), - read: 0, - }); - } - } - Ok(false) - } - fn verify(&self) -> Result<()> { - for source in &self.observed { - let file = self - .anchor - .as_ref() - .unwrap() - .open(&source.path) - .map_err(io)?; - if !source - .observed - .matches(&backend::observe(&file).map_err(io)?) - { - return Err(conflict()); - } - } - Ok(()) - } -} -struct Stage { - parent: Directory, - directory: Directory, - name: String, - target: String, -} -impl Drop for Stage { - fn drop(&mut self) { - if let Ok(parent) = self.parent.clone_dir() { - let name = self.name.clone(); - // Retired trees may be large. Cleanup cannot delay the publication receipt. - let _ = std::thread::Builder::new() - .name("transfer-cleanup".into()) - .spawn(move || { - if let Err(error) = parent.remove_tree(&name) - && error.kind() != std::io::ErrorKind::NotFound - { - eprintln!("lightspeed-envd retirement cleanup {name}: {error}"); - } - }); - } - } -} -struct Upload { - path: std::path::PathBuf, - hash: Sha256, - length: u64, - complete: bool, -} -struct Copying { - source: File, - destination: File, - hash: Sha256, - expected: String, - bytes: u64, - size: u64, - executable: bool, -} -struct Operation { - request: TransferRequest, - status: TransferStatus, - started: Instant, - limits: InventoryLimits, - scanner: Scanner, - stage: Option, - entries: Vec, - manifest_bytes: u64, - paths: BTreeMap, - sizes: BTreeMap, - pages: BTreeMap, bool)>, - uploads: BTreeMap, - spool: Option, - retained_inventory: Option, - staging_index: usize, - copying: Option, - unchanged: bool, - staging_name: Option, -} -#[derive(Default)] -pub struct TransferManager { - operations: BTreeMap, - journal: Option, -} -impl TransferManager { - pub fn with_journal(directory: std::path::PathBuf) -> Result { - backend::create_private_directory(&directory).map_err(io)?; - Ok(Self { - operations: BTreeMap::new(), - journal: Some(directory), - }) - } - pub fn expire(&mut self) -> Option { - self.operations.retain(|_, op| { - op.started.elapsed() < Duration::from_millis(op.limits.max_duration_ms) - }); - self.journal.clone() - } - pub fn cleanup_expired(directory: &Path) { - if let Ok(entries) = std::fs::read_dir(directory) { - for entry in entries.flatten().take(8192) { - if entry.path().extension().is_some_and(|s| s == "json") { - let id = entry - .file_name() - .to_string_lossy() - .trim_end_matches(".json") - .to_owned(); - if let Ok(Some(record)) = journal::read(directory, &id) - && record.expires_at_ms < journal::now_ms() - { - if let ( - Some(name), - TransferRequest::Begin { - selection: - TransferSelection::Materialize { destination, .. } - | TransferSelection::WriteFile { destination }, - .. - }, - ) = (&record.stage_name, &record.request) - && name.starts_with(".env-transfer-") - && valid_path(name) - && !name.contains('/') - && let Ok(anchor) = Directory::anchor(&record.root) - && let Ok(relative) = relative(&record.root, destination) - && let Ok((parent, _)) = anchor.parent(&relative) - && let Err(error) = parent.remove_tree(name) - && error.kind() != std::io::ErrorKind::NotFound - { - continue; - } - if let Some(spool) = &record.spool_directory - && spool.parent() == Some(directory) - && spool.file_name().is_some_and(|name| { - name.to_string_lossy() - .starts_with(&format!("{id}.content-")) - }) - { - let _ = std::fs::remove_dir_all(spool); - } - let _ = std::fs::remove_file(directory.join(format!("{id}.inventory"))); - let _ = std::fs::remove_file(entry.path()); - } - } - } - } - } - fn save(&self, root: &Path, id: &str) -> Result<()> { - let Some(directory) = &self.journal else { - return Ok(()); - }; - let op = &self.operations[id]; - if let Some(path) = &op.retained_inventory { - let mut copy = tempfile::NamedTempFile::new_in(directory).map_err(io)?; - std::io::copy(&mut File::open(path).map_err(io)?, &mut copy).map_err(io)?; - copy.as_file().sync_all().map_err(io)?; - copy.persist(directory.join(format!("{id}.inventory"))) - .map_err(|e| io(e.error))?; - } - journal::write( - directory, - id, - &journal::Record { - root: root.to_path_buf(), - request: op.request.clone(), - status: op.status.clone(), - expires_at_ms: journal::now_ms() - + op.limits - .max_duration_ms - .saturating_sub(op.started.elapsed().as_millis() as u64), - stage_name: op.staging_name.clone(), - spool_directory: op.spool.as_ref().map(|s| s.path().to_path_buf()), - }, - ) - } - pub fn execute(&mut self, root: &Path, request: TransferRequest) -> Result { - let id = request.operation_id().to_owned(); - if id.is_empty() - || id.len() > 128 - || !id - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b)) - { - return Err(invalid("invalid operation id")); - } - if !self.operations.contains_key(&id) - && let Some(directory) = &self.journal - && let Some(record) = journal::read(directory, &id)? - { - if record.root != root { - return Err(error( - Code::Forbidden, - "receipt belongs to a different filesystem scope", - )); - } - if record.expires_at_ms < journal::now_ms() { - return Err(error(Code::Timeout, "transfer receipt expired")); - } - if matches!(&request, TransferRequest::Begin { .. }) && request != record.request { - return Err(error( - Code::Conflict, - "operation id already used for different input", - )); - } - if matches!( - request, - TransferRequest::Begin { .. } - | TransferRequest::Status { .. } - | TransferRequest::Commit { .. } - | TransferRequest::Abort { .. } - ) && record.status.phase == TransferPhase::Complete - { - return Ok(TransferResponse::Status(record.status)); - } - if let TransferRequest::Inventory { offset, .. } = &request - && record.status.phase == TransferPhase::Complete - { - return retained_page( - &directory.join(format!("{id}.inventory")), - record.status.entries, - *offset, - ); - } - if matches!(request, TransferRequest::Abort { .. }) { - if let ( - Some(name), - TransferRequest::Begin { - selection: - TransferSelection::Materialize { destination, .. } - | TransferSelection::WriteFile { destination }, - .. - }, - ) = (&record.stage_name, &record.request) - && name.starts_with(".env-transfer-") - && valid_path(name) - && !name.contains('/') - { - let anchor = Directory::anchor(root).map_err(io)?; - let (parent, _) = anchor.parent(&relative(root, destination)?).map_err(io)?; - let _ = parent.remove_tree(name); - } - let mut record = record; - record.status.phase = TransferPhase::Aborted; - record.stage_name = None; - if let Some(spool) = record.spool_directory.take() - && spool.parent() == Some(directory.as_path()) - && spool.file_name().is_some_and(|name| { - name.to_string_lossy() - .starts_with(&format!("{id}.content-")) - }) - { - let _ = std::fs::remove_dir_all(spool); - } - journal::write(directory, &id, &record)?; - return Ok(TransferResponse::Status(record.status)); - } - return Err(error( - Code::Conflict, - "daemon restarted during transfer; inspect the destination and abort the old operation before starting a new one", - )); - } - if let Some(op) = self.operations.get(&id) - && op.started.elapsed() >= Duration::from_millis(op.limits.max_duration_ms) - { - return Err(error(Code::Timeout, "transfer expired")); - } - if let TransferRequest::Begin { - selection, limits, .. - } = &request - { - if let Some(op) = self.operations.get(&id) { - return if op.request == request { - Ok(TransferResponse::Status(op.status.clone())) - } else { - Err(error( - Code::Conflict, - "operation id already used for different input", - )) - }; - } - self.operations.retain(|_, op| { - op.started.elapsed() < Duration::from_millis(op.limits.max_duration_ms) - }); - if self.operations.len() >= 4096 - || self - .operations - .values() - .filter(|op| { - !matches!( - op.status.phase, - TransferPhase::Complete | TransferPhase::Aborted - ) - }) - .count() - >= 8 - { - return Err(error(Code::Conflict, "transfer capacity reached")); - } - let ceiling = InventoryLimits::default(); - if limits.max_entries == 0 - || limits.max_entries > ceiling.max_entries - || limits.max_depth > ceiling.max_depth - || limits.max_file_bytes > ceiling.max_file_bytes - || limits.max_total_bytes > ceiling.max_total_bytes - || limits.max_manifest_bytes > ceiling.max_manifest_bytes - || limits.max_duration_ms == 0 - || limits.max_duration_ms > ceiling.max_duration_ms - { - return Err(invalid("operation quotas exceed daemon ceilings")); - } - let anchor = Directory::anchor(root).map_err(io)?; - let (selected, stage, ignore_errors) = match selection { - TransferSelection::Capture { source } => (relative(root, source)?, None, false), - TransferSelection::Materialize { destination, .. } - | TransferSelection::WriteFile { destination } => { - let path = relative(root, destination)?; - if path.is_empty() { - return Err(invalid("cannot replace the filesystem root")); - } - for operation in self.operations.values().filter(|op| op.stage.is_some()) { - let other = &operation.scanner.selected; - if path == *other - || path - .strip_prefix(other) - .is_some_and(|tail| tail.starts_with('/')) - || other - .strip_prefix(&path) - .is_some_and(|tail| tail.starts_with('/')) - { - return Err(error( - Code::Conflict, - "destination overlaps an active materialization", - )); - } - } - let (parent, target) = anchor.ensure_parent(&path).map_err(io)?; - if parent.target_exists(&target).map_err(io)? { - if matches!( - selection, - TransferSelection::Materialize { - on_existing: TransferOnExisting::Error, - .. - } - ) { - return Err(error(Code::Conflict, "destination exists")); - } - if matches!(selection, TransferSelection::WriteFile { .. }) { - parent.writable_file(&target).map_err(io)?; - } - } - let name = format!(".env-transfer-{:032x}", rand::random::()); - let directory = parent.mkdir(&name, true).map_err(io)?; - ( - path, - Some(Stage { - parent, - directory, - name, - target, - }), - true, - ) - } - }; - let op = Operation { - request: request.clone(), - status: TransferStatus { - operation_id: id.clone(), - phase: TransferPhase::Scanning, - entries: 0, - bytes: 0, - transferred_bytes: 0, - reused_bytes: 0, - }, - started: Instant::now(), - limits: *limits, - scanner: Scanner::new(anchor, selected, *limits, ignore_errors), - staging_name: stage.as_ref().map(|stage| stage.name.clone()), - stage, - entries: vec![], - manifest_bytes: 0, - paths: BTreeMap::new(), - sizes: BTreeMap::new(), - pages: BTreeMap::new(), - uploads: BTreeMap::new(), - spool: Some(if let Some(directory) = &self.journal { - tempfile::Builder::new() - .prefix(&format!("{id}.content-")) - .tempdir_in(directory) - .map_err(io)? - } else { - tempfile::tempdir().map_err(io)? - }), - retained_inventory: None, - staging_index: 0, - copying: None, - unchanged: false, - }; - let response = TransferResponse::Status(op.status.clone()); - self.operations.insert(id.clone(), op); - if let Err(error) = self.save(root, &id) { - self.operations.remove(&id); - return Err(error); - } - return Ok(response); - } - let op = self.operations.get_mut(&id).ok_or_else(|| { - error( - Code::NotFound, - "unknown transfer; operation receipts may have expired", - ) - })?; - if matches!(request, TransferRequest::Status { .. }) { - return Ok(TransferResponse::Status(op.status.clone())); - } - if matches!(request, TransferRequest::Abort { .. }) { - if op.status.phase != TransferPhase::Complete { - op.status.phase = TransferPhase::Aborted; - op.release(); - } - let response = TransferResponse::Status(op.status.clone()); - self.save(root, &id)?; - return Ok(response); - } - if op.started.elapsed() >= Duration::from_millis(op.limits.max_duration_ms) { - return Err(error(Code::Timeout, "transfer expired")); - } - if op.status.phase == TransferPhase::Aborted { - return Err(error(Code::Conflict, "transfer aborted")); - } - let terminal = matches!(request, TransferRequest::Commit { .. }); - let response = op.execute(request)?; - if terminal { - self.save(root, &id)?; - } - Ok(response) - } -} -impl Drop for Operation { - fn drop(&mut self) { - self.release(); - } -} -impl Operation { - fn retain_inventory(&mut self) -> Result<()> { - let mut spool = tempfile::NamedTempFile::new().map_err(io)?; - let count = self.entries.len(); - spool - .seek(SeekFrom::Start((8 * (count + 1)) as u64)) - .map_err(io)?; - let mut positions = Vec::with_capacity(count + 1); - for entry in &self.entries { - positions.push(spool.stream_position().map_err(io)?); - serde_json::to_writer(&mut spool, entry).map_err(|e| invalid(&e.to_string()))?; - } - positions.push(spool.stream_position().map_err(io)?); - spool.seek(SeekFrom::Start(0)).map_err(io)?; - for position in positions { - spool.write_all(&position.to_le_bytes()).map_err(io)?; - } - self.retained_inventory = Some(spool.into_temp_path()); - Ok(()) - } - fn retained_page(&self, offset: u32) -> Result { - let path = self.retained_inventory.as_ref().ok_or_else(|| { - error( - Code::Conflict, - "completed materialization retains only its receipt", - ) - })?; - retained_page(path, self.status.entries, offset) - } - fn release(&mut self) { - self.stage.take(); - self.uploads.clear(); - self.copying.take(); - if let Some(spool) = self.spool.take() { - let _ = std::thread::Builder::new() - .name("transfer-spool-cleanup".into()) - .spawn(move || drop(spool)); - } - self.scanner.anchor.take(); - self.scanner.pending = Vec::new(); - self.scanner.hashing.take(); - self.scanner.files.clear(); - self.scanner.observed = Vec::new(); - self.scanner.entries = Vec::new(); - self.paths.clear(); - self.sizes.clear(); - self.pages.clear(); - self.entries = Vec::new(); - } - fn status(&self) -> TransferResponse { - TransferResponse::Status(self.status.clone()) - } - fn execute(&mut self, request: TransferRequest) -> Result { - match request { - TransferRequest::Advance { .. } => { - if self.status.phase == TransferPhase::Scanning - && match self.scanner.advance() { - Ok(done) => done, - Err(_) if self.stage.is_some() => { - self.scanner.skipped = true; - self.scanner.pending.clear(); - self.scanner.hashing.take(); - true - } - Err(error) => return Err(error), - } - { - if self.stage.is_some() { - self.status.phase = TransferPhase::Inventory; - } else { - self.scanner.verify()?; - self.entries = self.scanner.entries.clone(); - self.status.entries = self.entries.len() as u32; - self.status.bytes = self.scanner.bytes; - self.status.phase = TransferPhase::Ready; - } - } - if self.status.phase == TransferPhase::Content && self.missing().is_empty() { - if !self.scanner.skipped && self.entries == self.scanner.entries { - self.scanner.verify()?; - self.unchanged = true; - self.status.reused_bytes = self.status.bytes; - self.status.phase = TransferPhase::Ready; - } else { - self.status.phase = TransferPhase::Staging; - } - } - if self.status.phase == TransferPhase::Staging { - self.advance_staging()?; - } - Ok(self.status()) - } - TransferRequest::Inventory { offset, .. } => { - if self.status.phase == TransferPhase::Complete { - return self.retained_page(offset); - } - if matches!( - self.status.phase, - TransferPhase::Scanning | TransferPhase::Inventory - ) { - return Err(error(Code::Conflict, "inventory not complete")); - } - let offset = offset as usize; - if offset > self.entries.len() { - return Err(invalid("inventory offset")); - } - let end = (offset + MAX_INVENTORY_PAGE).min(self.entries.len()); - Ok(TransferResponse::Inventory { - entries: self.entries[offset..end].to_vec(), - next_offset: (end < self.entries.len()).then_some(end as u32), - }) - } - TransferRequest::Append { - offset, - entries, - last, - .. - } => { - if self.stage.is_none() || entries.len() > MAX_INVENTORY_PAGE { - return Err(invalid("invalid inventory page")); - } - if let Some((previous, sealed)) = self.pages.get(&offset) { - if previous != &entries || *sealed != last { - return Err(error(Code::Conflict, "inventory retry differs")); - } - return Ok(self.status()); - } - let offset = offset as usize; - if self.status.phase != TransferPhase::Inventory || offset != self.entries.len() { - return Err(error(Code::Conflict, "inventory offset/phase mismatch")); - } - if entries.is_empty() { - return Err(invalid("empty inventory page")); - } - if self.is_file_write() - && (offset != 0 - || !last - || !matches!(entries.as_slice(), [InventoryEntry { path, content: InventoryContent::File { .. } }] if path.is_empty())) - { - return Err(invalid("file write requires exactly one file")); - } - // Validate the complete page before mutating the operation. - let mut total = self.status.bytes; - let mut meta = self.manifest_bytes; - let mut paths = BTreeMap::new(); - let mut sizes = BTreeMap::new(); - for (index, entry) in entries.iter().enumerate() { - if !valid_path(&entry.path) - || (!entry.path.is_empty() - && entry.path.split('/').count() > self.limits.max_depth as usize) - { - return Err(invalid("invalid inventory path")); - } - if offset + index == 0 { - if !entry.path.is_empty() { - return Err(invalid("inventory must start with root")); - } - } else { - let parent = entry.path.rsplit_once('/').map_or("", |(p, _)| p); - if self.paths.contains_key(&entry.path) || paths.contains_key(&entry.path) { - return Err(invalid("duplicate inventory path")); - } - if !self - .paths - .get(parent) - .or_else(|| paths.get(parent)) - .copied() - .unwrap_or(false) - { - return Err(invalid("parent must precede child")); - } - } - paths.insert( - entry.path.clone(), - entry.content == InventoryContent::Directory, - ); - if let InventoryContent::File { - size_bytes, digest, .. - } = &entry.content - { - if !valid_digest(digest) || *size_bytes > self.limits.max_file_bytes { - return Err(invalid("invalid file identity/size")); - } - if self - .sizes - .get(digest) - .or_else(|| sizes.get(digest)) - .is_some_and(|s| s != size_bytes) - || self - .scanner - .files - .get(digest) - .is_some_and(|f| f.observed.size() != *size_bytes) - { - return Err(invalid("same digest has different sizes")); - } - sizes.insert(digest.clone(), *size_bytes); - total = total - .checked_add(*size_bytes) - .ok_or_else(|| invalid("size overflow"))?; - } - meta += serde_json::to_vec(entry) - .map_err(|e| invalid(&e.to_string()))? - .len() as u64; - } - if self.entries.len() + entries.len() > self.limits.max_entries as usize - || total > self.limits.max_total_bytes - || meta > self.limits.max_manifest_bytes - { - return Err(invalid("inventory quota exceeded")); - } - self.pages.insert(offset as u32, (entries.clone(), last)); - self.paths.extend(paths); - self.sizes.extend(sizes); - self.entries.extend(entries); - self.status.entries = self.entries.len() as u32; - self.status.bytes = total; - self.manifest_bytes = meta; - if last { - if self.entries.is_empty() { - return Err(invalid("empty inventory")); - } - self.status.phase = TransferPhase::Content; - } - Ok(self.status()) - } - TransferRequest::Missing { offset, .. } => { - if !matches!( - self.status.phase, - TransferPhase::Content - | TransferPhase::Staging - | TransferPhase::Ready - | TransferPhase::Complete - ) { - return Err(error(Code::Conflict, "inventory not sealed")); - } - let missing = self.missing(); - let offset = offset as usize; - if offset > missing.len() { - return Err(invalid("missing offset")); - } - let end = (offset + MAX_INVENTORY_PAGE).min(missing.len()); - Ok(TransferResponse::Missing { - digests: missing[offset..end].to_vec(), - next_offset: (end < missing.len()).then_some(end as u32), - }) - } - TransferRequest::Read { digest, offset, .. } => { - if self.stage.is_some() || self.status.phase != TransferPhase::Ready { - return Err(error(Code::Conflict, "capture is not ready")); - } - let source = self - .scanner - .files - .get(&digest) - .ok_or_else(|| invalid("digest outside capture"))?; - let mut file = self - .scanner - .anchor - .as_ref() - .unwrap() - .open(&source.path) - .map_err(io)?; - if !source - .observed - .matches(&backend::observe(&file).map_err(io)?) - || offset > source.observed.size() - { - return Err(conflict()); - } - file.seek(SeekFrom::Start(offset)).map_err(io)?; - let mut data = vec![ - 0; - (source.observed.size() - offset).min(MAX_CONTENT_CHUNK as u64) - as usize - ]; - file.read_exact(&mut data).map_err(io)?; - if !source - .observed - .matches(&backend::observe(&file).map_err(io)?) - { - return Err(conflict()); - } - self.status.transferred_bytes += data.len() as u64; - Ok(TransferResponse::Chunk { - eof: offset + data.len() as u64 == source.observed.size(), - data: ByteChunk::from(data), - }) - } - TransferRequest::Write { - digest: expected, - offset, - data, - .. - } => { - if self.status.phase != TransferPhase::Content - || data.as_slice().len() > MAX_CONTENT_CHUNK - { - return Err(invalid("invalid upload chunk/phase")); - } - let size = self - .file_size(&expected) - .ok_or_else(|| invalid("digest outside inventory"))?; - if !self.uploads.contains_key(&expected) { - if offset != 0 { - return Err(error(Code::Conflict, "upload starts at offset zero")); - } - let path = self.spool.as_ref().unwrap().path().join(&expected[7..]); - let file = std::fs::OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .open(&path) - .map_err(io)?; - drop(file); - self.uploads.insert( - expected.clone(), - Upload { - path, - hash: Sha256::new(), - length: 0, - complete: false, - }, - ); - } - let upload = self.uploads.get_mut(&expected).unwrap(); - let mut file = std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(&upload.path) - .map_err(io)?; - let bytes = data.as_slice(); - let end = offset - .checked_add(bytes.len() as u64) - .ok_or_else(|| invalid("chunk offset overflow"))?; - if offset < upload.length || upload.complete { - if end > upload.length { - return Err(error(Code::Conflict, "overlapping chunk")); - } - file.seek(SeekFrom::Start(offset)).map_err(io)?; - let mut existing = vec![0; bytes.len()]; - file.read_exact(&mut existing).map_err(io)?; - if existing != bytes { - return Err(error(Code::Conflict, "chunk retry differs")); - } - return Ok(TransferResponse::Written { - next_offset: upload.length, - }); - } - if offset != upload.length || end > size || (bytes.is_empty() && size != 0) { - return Err(invalid("chunk offset/size mismatch")); - } - file.seek(SeekFrom::Start(offset)).map_err(io)?; - file.write_all(bytes).map_err(io)?; - upload.hash.update(bytes); - upload.length += bytes.len() as u64; - self.status.transferred_bytes += bytes.len() as u64; - if upload.length == size { - if digest(upload.hash.clone()) != expected { - self.uploads.remove(&expected); - let _ = std::fs::remove_file( - self.spool.as_ref().unwrap().path().join(&expected[7..]), - ); - return Err(error(Code::Conflict, "file digest mismatch")); - } - file.sync_all().map_err(io)?; - upload.complete = true; - } - Ok(TransferResponse::Written { next_offset: end }) - } - TransferRequest::Commit { .. } => { - if self.status.phase == TransferPhase::Complete { - return Ok(self.status()); - } - if self.status.phase != TransferPhase::Ready { - return Err(error(Code::Conflict, "transfer is not ready to commit")); - } - if self.stage.is_none() { - self.retain_inventory()?; - } - if self.unchanged { - self.scanner.verify()?; - } else if let Some(stage) = &self.stage { - let replace = matches!( - &self.request, - TransferRequest::Begin { - selection: TransferSelection::Materialize { - on_existing: TransferOnExisting::Replace, - .. - }, - .. - } - ); - stage.parent.target_exists(&stage.target).map_err(io)?; - if self.is_file_write() { - match stage.parent.writable_file(&stage.target) { - Ok(target) => { - let permissions = target.metadata().map_err(io)?.permissions(); - let staged = stage.directory.open("tree").map_err(io)?; - staged.set_permissions(permissions).map_err(io)?; - staged.sync_all().map_err(io)?; - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), - Err(error) => return Err(io(error)), - } - } - if self.is_file_write() { - stage - .parent - .publish_file(&stage.directory, &stage.target) - .map_err(io)?; - } else { - stage - .parent - .publish(&stage.directory, &stage.target, replace) - .map_err(io)?; - } - // Rename has succeeded. A later durability error must never cause - // a second swap on retry. - self.status.phase = TransferPhase::Complete; - stage.parent.sync().map_err(io)?; - stage.directory.sync().map_err(io)?; - } else { - self.scanner.verify()?; - } - self.status.phase = TransferPhase::Complete; - self.release(); - Ok(self.status()) - } - _ => Err(invalid("unexpected transfer action")), - } - } - fn file_size(&self, digest: &str) -> Option { - self.sizes.get(digest).copied() - } - - fn is_file_write(&self) -> bool { - matches!( - &self.request, - TransferRequest::Begin { - selection: TransferSelection::WriteFile { .. }, - .. - } - ) - } - fn missing(&self) -> Vec { - let mut missing = std::collections::BTreeSet::new(); - for digest in self.sizes.keys() { - if !self.scanner.files.contains_key(digest) - && !self.uploads.get(digest).is_some_and(|u| u.complete) - { - missing.insert(digest.clone()); - } - } - missing.into_iter().collect() - } - fn advance_staging(&mut self) -> Result<()> { - let mut allowance = 4 * 1024 * 1024; - for _ in 0..MAX_INVENTORY_PAGE { - if let Some(mut copy) = self.copying.take() { - let mut buffer = [0u8; 64 * 1024]; - while allowance > 0 { - let limit = buffer.len().min(allowance); - let read = copy.source.read(&mut buffer[..limit]).map_err(io)?; - if read == 0 { - if copy.bytes != copy.size || digest(copy.hash) != copy.expected { - return Err(conflict()); - } - backend::set_executable(©.destination, copy.executable).map_err(io)?; - copy.destination.sync_all().map_err(io)?; - let entry = &self.entries[self.staging_index]; - let path = if entry.path.is_empty() { - "tree".into() - } else { - format!("tree/{}", entry.path) - }; - self.stage - .as_ref() - .unwrap() - .directory - .parent(&path) - .map_err(io)? - .0 - .sync() - .map_err(io)?; - self.staging_index += 1; - break; - } - copy.bytes += read as u64; - if copy.bytes > copy.size { - return Err(conflict()); - } - copy.hash.update(&buffer[..read]); - copy.destination.write_all(&buffer[..read]).map_err(io)?; - allowance -= read; - if allowance == 0 { - self.copying = Some(copy); - return Ok(()); - } - } - continue; - } - let Some(entry) = self.entries.get(self.staging_index) else { - self.status.phase = TransferPhase::Ready; - return Ok(()); - }; - let stage = &self.stage.as_ref().unwrap().directory; - let path = if entry.path.is_empty() { - "tree".into() - } else { - format!("tree/{}", entry.path) - }; - match &entry.content { - InventoryContent::Directory => { - let (parent, name) = stage.parent(&path).map_err(io)?; - parent.mkdir(&name, false).map_err(io)?.sync().map_err(io)?; - parent.sync().map_err(io)?; - self.staging_index += 1; - } - InventoryContent::File { - digest, - size_bytes, - executable, - } => { - let source = - if let Some(upload) = self.uploads.get(digest).filter(|u| u.complete) { - let mut f = File::open(&upload.path).map_err(io)?; - f.seek(SeekFrom::Start(0)).map_err(io)?; - f - } else { - let source = self - .scanner - .files - .get(digest) - .ok_or_else(|| invalid("missing content"))?; - let file = self - .scanner - .anchor - .as_ref() - .unwrap() - .open(&source.path) - .map_err(io)?; - if !source - .observed - .matches(&backend::observe(&file).map_err(io)?) - { - return Err(conflict()); - } - self.status.reused_bytes += *size_bytes; - file - }; - self.copying = Some(Copying { - source, - destination: stage.create(&path).map_err(io)?, - hash: Sha256::new(), - expected: digest.clone(), - bytes: 0, - size: *size_bytes, - executable: *executable, - }); - } - } - } - Ok(()) - } -} - -fn retained_page(path: &Path, count: u32, offset: u32) -> Result { - if offset > count { - return Err(invalid("inventory offset")); - } - let end = (offset + MAX_INVENTORY_PAGE as u32).min(count); - let mut file = File::open(path).map_err(io)?; - file.seek(SeekFrom::Start(offset as u64 * 8)).map_err(io)?; - let mut positions = Vec::new(); - for _ in offset..=end { - let mut bytes = [0; 8]; - file.read_exact(&mut bytes).map_err(io)?; - positions.push(u64::from_le_bytes(bytes)); - } - let mut entries = Vec::new(); - for pair in positions.windows(2) { - let size = pair[1] - pair[0]; - if size > MAX_INVENTORY_PATH_BYTES as u64 + 1024 { - return Err(invalid("corrupt retained inventory")); - } - file.seek(SeekFrom::Start(pair[0])).map_err(io)?; - let mut bytes = vec![0; size as usize]; - file.read_exact(&mut bytes).map_err(io)?; - entries.push(serde_json::from_slice(&bytes).map_err(|e| invalid(&e.to_string()))?); - } - Ok(TransferResponse::Inventory { - entries, - next_offset: (end < count).then_some(end), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use environment_protocol::shared::EnvironmentPath; - - #[test] - fn expired_transfer_rejects_progress_and_publication() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - std::fs::write(root.join("destination"), b"keep").unwrap(); - let mut manager = TransferManager::default(); - manager - .execute( - &root, - TransferRequest::Begin { - operation_id: "expired".into(), - selection: TransferSelection::Materialize { - destination: EnvironmentPath::new( - root.join("destination").to_str().unwrap(), - ) - .unwrap(), - on_existing: TransferOnExisting::Replace, - }, - limits: InventoryLimits { - max_duration_ms: 1000, - ..Default::default() - }, - }, - ) - .unwrap(); - // Exercise expiry without waiting for wall-clock time or depending on - // filesystem throughput and test-runner scheduling. - manager.operations.get_mut("expired").unwrap().started = - Instant::now() - Duration::from_secs(2); - for request in [ - TransferRequest::Advance { - operation_id: "expired".into(), - }, - TransferRequest::Commit { - operation_id: "expired".into(), - }, - ] { - assert_eq!( - manager.execute(&root, request).unwrap_err().code, - Code::Timeout - ); - } - assert_eq!(std::fs::read(root.join("destination")).unwrap(), b"keep"); - } - - #[test] - fn scan_respects_remaining_budget_when_a_large_file_follows_a_small_file() { - let root = tempfile::tempdir().unwrap(); - std::fs::create_dir(root.path().join("source")).unwrap(); - std::fs::write(root.path().join("source/small"), [1; 17]).unwrap(); - std::fs::write( - root.path().join("source/large"), - vec![2; 4 * 1024 * 1024 + 13], - ) - .unwrap(); - let mut scanner = Scanner::new( - Directory::anchor(root.path()).unwrap(), - "source".into(), - InventoryLimits::default(), - false, - ); - // Make traversal order explicit: pending nodes are popped from the end. - scanner.pending = vec!["source/large".into(), "source/small".into()]; - scanner.pending_bytes = scanner.pending.iter().map(String::len).sum(); - let progress = |scanner: &Scanner| { - scanner - .entries - .iter() - .map(|entry| match entry.content { - InventoryContent::File { size_bytes, .. } => size_bytes, - InventoryContent::Directory => 0, - }) - .sum::() - + scanner.hashing.as_ref().map_or(0, |task| task.read) - }; - assert!(!scanner.advance().unwrap()); - assert_eq!(progress(&scanner), 4 * 1024 * 1024); - assert!(scanner.advance().unwrap()); - assert_eq!(progress(&scanner), 4 * 1024 * 1024 + 30); - scanner.verify().unwrap(); - } - - #[test] - fn staging_respects_remaining_budget_when_a_large_file_follows_a_small_file() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - std::fs::create_dir(root.join("target")).unwrap(); - let small = vec![1; 17]; - let large = vec![2; 4 * 1024 * 1024 + 13]; - std::fs::write(root.join("target/small"), &small).unwrap(); - std::fs::write(root.join("target/large"), &large).unwrap(); - let mut manager = TransferManager::default(); - manager - .execute( - &root, - TransferRequest::Begin { - operation_id: "bounded".into(), - selection: TransferSelection::Materialize { - destination: EnvironmentPath::new(root.join("target").to_str().unwrap()) - .unwrap(), - on_existing: TransferOnExisting::Replace, - }, - limits: InventoryLimits::default(), - }, - ) - .unwrap(); - while manager.operations["bounded"].status.phase == TransferPhase::Scanning { - manager - .execute( - &root, - TransferRequest::Advance { - operation_id: "bounded".into(), - }, - ) - .unwrap(); - } - let mut entries = vec![InventoryEntry { - path: String::new(), - content: InventoryContent::Directory, - }]; - for (name, bytes) in [("renamed-small", &small), ("renamed-large", &large)] { - entries.push(InventoryEntry { - path: name.into(), - content: InventoryContent::File { - size_bytes: bytes.len() as u64, - executable: false, - digest: harness::BlobRef::from_bytes(bytes).to_string(), - }, - }); - } - manager - .execute( - &root, - TransferRequest::Append { - operation_id: "bounded".into(), - offset: 0, - entries, - last: true, - }, - ) - .unwrap(); - manager - .execute( - &root, - TransferRequest::Advance { - operation_id: "bounded".into(), - }, - ) - .unwrap(); - let op = &manager.operations["bounded"]; - assert_eq!(op.status.phase, TransferPhase::Staging); - let copied: u64 = op.entries[..op.staging_index] - .iter() - .map(|entry| match entry.content { - InventoryContent::File { size_bytes, .. } => size_bytes, - InventoryContent::Directory => 0, - }) - .sum::() - + op.copying.as_ref().map_or(0, |copy| copy.bytes); - assert_eq!(copied, 4 * 1024 * 1024); - manager - .execute( - &root, - TransferRequest::Advance { - operation_id: "bounded".into(), - }, - ) - .unwrap(); - assert_eq!( - manager.operations["bounded"].status.phase, - TransferPhase::Ready - ); - manager - .execute( - &root, - TransferRequest::Commit { - operation_id: "bounded".into(), - }, - ) - .unwrap(); - assert_eq!( - std::fs::read(root.join("target/renamed-small")).unwrap(), - small - ); - assert_eq!( - std::fs::read(root.join("target/renamed-large")).unwrap(), - large - ); - } - - #[test] - fn expiry_sweep_reclaims_abandoned_state_and_preserves_live_work() { - let root = tempfile::tempdir().unwrap(); - let state = root.path().join("state"); - std::fs::create_dir(&state).unwrap(); - std::fs::write(root.path().join("destination"), b"keep").unwrap(); - for (id, expires_at_ms) in [ - ("expired", journal::now_ms() - 1), - ("live", journal::now_ms() + 60_000), - ] { - let stage = format!(".env-transfer-{id}"); - std::fs::create_dir_all(root.path().join(&stage).join("nested")).unwrap(); - std::fs::write(root.path().join(&stage).join("nested/file"), b"staged").unwrap(); - let spool = state.join(format!("{id}.content-private")); - std::fs::create_dir(&spool).unwrap(); - std::fs::write(spool.join("file"), b"uploaded").unwrap(); - std::fs::write(state.join(format!("{id}.inventory")), b"inventory").unwrap(); - journal::write( - &state, - id, - &journal::Record { - root: root.path().into(), - request: TransferRequest::Begin { - operation_id: id.into(), - selection: TransferSelection::Materialize { - destination: EnvironmentPath::new( - root.path().join("destination").to_str().unwrap(), - ) - .unwrap(), - on_existing: TransferOnExisting::Replace, - }, - limits: Default::default(), - }, - status: TransferStatus { - operation_id: id.into(), - phase: TransferPhase::Scanning, - entries: 0, - bytes: 0, - transferred_bytes: 0, - reused_bytes: 0, - }, - expires_at_ms, - stage_name: Some(stage), - spool_directory: Some(spool), - }, - ) - .unwrap(); - } - TransferManager::cleanup_expired(&state); - for suffix in ["json", "inventory", "content-private"] { - assert!(!state.join(format!("expired.{suffix}")).exists()); - assert!(state.join(format!("live.{suffix}")).exists()); - } - assert!(!root.path().join(".env-transfer-expired").exists()); - assert!(root.path().join(".env-transfer-live/nested/file").exists()); - assert_eq!( - std::fs::read(root.path().join("destination")).unwrap(), - b"keep" - ); - } -} diff --git a/crates/environment-daemon/src/filesystem/transfer/tests.rs b/crates/environment-daemon/src/filesystem/transfer/tests.rs index e6c6603d7..fc7813aa6 100644 --- a/crates/environment-daemon/src/filesystem/transfer/tests.rs +++ b/crates/environment-daemon/src/filesystem/transfer/tests.rs @@ -1,1331 +1,964 @@ -use super::*; -use async_trait::async_trait; -use environment_client::{EnvironmentClientResult, EnvironmentDataClient, JsonRpcTransport}; +//! Transfer state machine tests against the handlers, without sockets. + +use std::{os::unix::fs::symlink, path::Path}; + use environment_protocol::{ - data::{inventory::*, transfer_session::*}, - shared::EnvironmentPath, -}; -use harness::{ - BlobRef, - storage::{BlobSource, BlobStore, BlobStoreError}, -}; -use serde_json::json; -use std::path::Path; -use std::{ - os::unix::fs::{PermissionsExt, symlink}, - sync::{ - Arc, - atomic::{AtomicBool, AtomicUsize, Ordering}, + data::{ + fs::{TreeContent, TreeFilter, WriteCondition}, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, InventoryEntry, + ListTransferMissingParams, ReadTransferContentParams, ReadTransferInventoryParams, + SpecialFilePolicy, SymlinkPolicy, TransferLimits, TransferOnExisting, + TransferOperationParams, TransferPhase, TransferSelection, TransferSkipReason, + TransferStatus, WriteTransferContentParams, + }, + }, + error::{ + AbortReason, BusyReason, ErrorData, PreconditionReason, ProtocolError, ProtocolErrorKind, }, + ids::TransferOperationId, + paths::{ContentDigest, EnvironmentPath, TreePath}, }; -use tools::{ - environment_protocol::RemoteEnvironmentConnection, - transfer::{self}, +use sha2::{Digest, Sha256}; + +use crate::filesystem::{ + LocalFileSystem, + tests::{context, filesystem}, }; -struct Transport { - runtime: DaemonRuntime, - pending: Option, - largest: Arc, - lose_commit: Arc, +fn id(value: &str) -> TransferOperationId { + TransferOperationId::new(value).unwrap() } -#[async_trait] -impl JsonRpcTransport for Transport { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()> { - let wire = serde_json::to_vec(&message)?; - self.largest.fetch_max(wire.len(), Ordering::Relaxed); - let message: Value = serde_json::from_slice(&wire)?; - let result = handle_data( - &self.runtime, - message["method"].as_str().unwrap(), - message["params"].clone(), - ) - .await; - if message["params"]["action"] == "commit" - && self.lose_commit.swap(false, Ordering::Relaxed) - { - return Err(environment_client::EnvironmentClientError::TransportClosed); - } - self.pending = Some(match result { - Ok(result) => success_response(message["id"].clone(), result), - Err(error) => error_response(Some(message["id"].clone()), error), - }); - Ok(()) - } - async fn recv(&mut self) -> EnvironmentClientResult> { - let Some(response) = self.pending.take() else { - return Ok(None); - }; - let wire = serde_json::to_vec(&response)?; - self.largest.fetch_max(wire.len(), Ordering::Relaxed); - Ok(Some(serde_json::from_slice(&wire)?)) + +fn path(root: &Path, relative: &str) -> EnvironmentPath { + EnvironmentPath::new(format!("{}/{relative}", root.display())).unwrap() +} + +fn limits() -> TransferLimits { + TransferLimits { + max_entries: 1000, + max_depth: 16, + max_file_bytes: 64 * 1024 * 1024, + max_total_bytes: 256 * 1024 * 1024, + max_manifest_bytes: 1024 * 1024, + max_duration_ms: 600_000, } } -fn runtime(root: &Path, read_only: bool) -> DaemonRuntime { - DaemonRuntime::new(crate::config::DaemonConfig { - listen: None, - cwd: root.into(), - fs_root: root.into(), - state_dir: root.join("state"), - read_only_fs: read_only, - registration: None, - scrubbed_env: vec![], - }) - .unwrap() + +fn digest(bytes: &[u8]) -> ContentDigest { + ContentDigest::from_sha256(Sha256::digest(bytes).into()) } -fn path(value: &str) -> EnvironmentPath { - EnvironmentPath::new(value).unwrap() + +fn operation(value: &str) -> TransferOperationParams { + TransferOperationParams { + operation_id: id(value), + } } -fn remote( - runtime: DaemonRuntime, -) -> ( - RemoteEnvironmentConnection, - Arc, - Arc, -) { - let largest = Arc::new(AtomicUsize::new(0)); - let lose_commit = Arc::new(AtomicBool::new(false)); - let caps = runtime.capabilities(); - ( - RemoteEnvironmentConnection::new( - EnvironmentDataClient::new(Transport { - runtime, - pending: None, - largest: largest.clone(), - lose_commit: lose_commit.clone(), - }), - caps, - ), - largest, - lose_commit, + +fn capture(source: EnvironmentPath, filter: TreeFilter) -> TransferSelection { + TransferSelection::Capture { + source, + filter, + symlinks: SymlinkPolicy::ExpandInside, + special_files: SpecialFilePolicy::Reject, + } +} + +async fn begin( + fs: &LocalFileSystem, + operation_id: &str, + selection: TransferSelection, +) -> Result { + fs.transfer_begin( + &context(), + BeginTransferParams { + operation_id: id(operation_id), + selection, + limits: limits(), + }, ) + .await } -struct Repeated { - remaining: u64, + +async fn advance_until( + fs: &LocalFileSystem, + operation_id: &str, + phase: TransferPhase, +) -> Result { + for _ in 0..10_000 { + let status = fs + .transfer_advance(&context(), operation(operation_id)) + .await?; + if status.phase == phase { + return Ok(status); + } + } + panic!("{operation_id} never reached {phase:?}"); } -#[async_trait] -impl BlobSource for Repeated { - async fn read_chunk(&mut self, max_bytes: usize) -> Result, BlobStoreError> { - let size = self.remaining.min(max_bytes as u64) as usize; - self.remaining -= size as u64; - Ok(vec![0x83; size]) + +async fn inventory(fs: &LocalFileSystem, operation_id: &str) -> Vec { + let mut entries = Vec::new(); + let mut index = 0; + loop { + let page = fs + .transfer_read_inventory( + &context(), + ReadTransferInventoryParams { + operation_id: id(operation_id), + index, + limit: 2, + }, + ) + .await + .unwrap(); + entries.extend(page.entries); + match page.next_index { + Some(next) => index = next, + None => return entries, + } } } -fn repeated_ref(size: u64) -> BlobRef { - use sha2::{Digest, Sha256}; - let mut hash = Sha256::new(); - let chunk = [0x83; 65536]; - let mut remaining = size; - while remaining > 0 { - let n = remaining.min(chunk.len() as u64) as usize; - hash.update(&chunk[..n]); - remaining -= n as u64; + +fn reason(error: &ProtocolError) -> Option { + match error.data { + ErrorData::FailedPrecondition { reason } => Some(reason), + _ => None, } - BlobRef::parse(format!("sha256:{}", hex::encode(hash.finalize()))).unwrap() } #[tokio::test(flavor = "current_thread")] -async fn blob_file_tools_stream_without_vfs_and_keep_receipts_across_retries() { - use tools::builtin::{ - BuiltinTool, BuiltinToolContext, BuiltinToolOperation as Op, BuiltinToolSurface as Surface, +async fn captures_expand_inside_links_skip_ignored_trees_and_serve_ranged_content() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + std::fs::create_dir_all(root.join("project/src")).unwrap(); + std::fs::create_dir_all(root.join("project/node_modules/.bin")).unwrap(); + std::fs::create_dir_all(root.join("project/.git")).unwrap(); + std::fs::write(root.join("project/.gitignore"), "node_modules/\n").unwrap(); + std::fs::write(root.join("project/src/lib.rs"), b"0123456789").unwrap(); + std::fs::write(root.join("project/.git/HEAD"), b"ref").unwrap(); + symlink("src/lib.rs", root.join("project/alias.rs")).unwrap(); + symlink( + "../src/missing", + root.join("project/node_modules/.bin/tool"), + ) + .unwrap(); + + let filter = TreeFilter { + include_hidden: true, + ..TreeFilter::default() }; - for surface in [ - Surface::Canonical, - Surface::CodexLike, - Surface::ClaudeCodeLike, - ] { - let environment = tempfile::tempdir().unwrap(); - let cas = tempfile::tempdir().unwrap(); - let store = Arc::new(store_fs::FsBlobStore::open(cas.path()).await.unwrap()); - let size = 9 * 1024 * 1024 + 7; - let content = repeated_ref(size); - store - .put_stream(&content, size, &mut Repeated { remaining: size }) - .await - .unwrap(); - let (connection, largest, lose_commit) = remote(runtime(environment.path(), false)); - let (_, context) = connection - .with_cwd(tools::fs::FsPath::new(environment.path().to_str().unwrap()).unwrap()) - .into_contexts(store.clone()); - let context = context.with_environment_id("test-env"); - let key = if surface == Surface::ClaudeCodeLike { - "file_path" - } else { - "path" - }; - let write = BuiltinTool::environment(Op::WriteFile, surface); - let capture = BuiltinTool::environment(Op::Reference, surface); - let writer = context.clone().with_operation_id("write-binary"); - let output = write - .invoke_json( - BuiltinToolContext::Environment(&writer), - json!({key:"nested/source.bin","content_ref":content}), - ) - .await - .unwrap(); - assert_eq!(output.output_json["bytes_written"], size); - assert_eq!(output.output_json["receipt"]["phase"], "complete"); - assert_eq!( - std::fs::metadata(environment.path().join("nested/source.bin")) - .unwrap() - .len(), - size - ); - let capture_cas = tempfile::tempdir().unwrap(); - let capture_store = Arc::new( - store_fs::FsBlobStore::open(capture_cas.path()) - .await - .unwrap(), - ); - let mut capture_context = context.clone().with_operation_id("capture-binary"); - capture_context.blobs = capture_store.clone(); - let captured = capture - .invoke_json( - BuiltinToolContext::Environment(&capture_context), - json!({"path":"nested/source.bin"}), - ) - .await - .unwrap(); - assert_eq!(captured.output_json["content_ref"], content.to_string()); - assert_eq!(captured.output_json["byte_len"], size); - assert_eq!(captured.attachments.len(), 1); - assert_eq!(captured.output_json["source"]["id"], "test-env"); - assert_eq!( - capture_store.stat_blob(&content).await.unwrap().byte_len, - size - ); - assert_eq!( - capture_store - .read_blob_range(&content, size - 7, 7) - .await - .unwrap(), - vec![0x83; 7] - ); - std::fs::write( - environment.path().join("nested/source.bin"), - b"changed live file", + begin(&fs, "capture", capture(path(&root, "project"), filter)) + .await + .unwrap(); + let status = advance_until(&fs, "capture", TransferPhase::Inventory) + .await + .unwrap(); + let entries = inventory(&fs, "capture").await; + let paths: Vec<_> = entries.iter().map(|entry| entry.path.as_str()).collect(); + // The .git directory is always pruned and node_modules is ignored. + assert_eq!(paths, [".", ".gitignore", "alias.rs", "src", "src/lib.rs"]); + assert_eq!(status.entry_count, 5); + let content = digest(b"0123456789"); + assert!(entries.iter().any(|entry| matches!( + entry.content, + TreeContent::File { digest: Some(found), .. } if found == content + ))); + let first = fs + .transfer_read( + &context(), + ReadTransferContentParams { + operation_id: id("capture"), + digest: content, + offset: 0, + max_bytes: 4, + }, ) + .await .unwrap(); - let changed_source_retry = capture - .invoke_json( - BuiltinToolContext::Environment(&capture_context), - json!({"path":"nested/source.bin"}), - ) - .await - .unwrap(); - assert_eq!( - changed_source_retry.output_json, captured.output_json, - "same-operation capture retry must keep the original immutable version" - ); - std::fs::remove_file(environment.path().join("nested/source.bin")).unwrap(); - let retry = capture - .invoke_json( - BuiltinToolContext::Environment(&capture_context), - json!({"path":"nested/source.bin"}), - ) - .await - .unwrap(); - assert_eq!( - retry.output_json, captured.output_json, - "completed capture must not revisit the deleted live path" - ); - - // Materialization's receipt must make a lost commit response unambiguous. - std::fs::write(environment.path().join("result.bin"), b"original").unwrap(); - std::fs::set_permissions( - environment.path().join("result.bin"), - std::fs::Permissions::from_mode(0o751), + assert_eq!(first.data.as_slice(), b"0123"); + assert!(first.truncated); + let rest = fs + .transfer_read( + &context(), + ReadTransferContentParams { + operation_id: id("capture"), + digest: content, + offset: 4, + max_bytes: 100, + }, ) + .await .unwrap(); - let writer = context.clone().with_operation_id("lost-write-receipt"); - lose_commit.store(true, Ordering::Relaxed); - assert!( - write - .invoke_json( - BuiltinToolContext::Environment(&writer), - json!({key:"result.bin","content_ref":content}) - ) - .await - .is_err() - ); - assert_eq!( - std::fs::metadata(environment.path().join("result.bin")) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o751 - ); - std::fs::write(environment.path().join("result.bin"), b"later local edit").unwrap(); - let retry = write - .invoke_json( - BuiltinToolContext::Environment(&writer), - json!({key:"result.bin","content_ref":content}), - ) - .await - .unwrap(); - assert_eq!(retry.output_json["receipt"]["phase"], "complete"); - assert_eq!( - std::fs::read(environment.path().join("result.bin")).unwrap(), - b"later local edit" - ); - - std::fs::create_dir(environment.path().join("directory")).unwrap(); - std::fs::write(environment.path().join("directory/keep"), b"keep").unwrap(); - let writer = context.clone().with_operation_id("reject-directory-write"); - assert!( - write - .invoke_json( - BuiltinToolContext::Environment(&writer), - json!({key:"directory","content_ref":content}) - ) - .await - .is_err() - ); - assert_eq!( - std::fs::read(environment.path().join("directory/keep")).unwrap(), - b"keep" - ); - let reader = context - .clone() - .with_operation_id("reject-directory-capture"); - assert!( - capture - .invoke_json( - BuiltinToolContext::Environment(&reader), - json!({"path":"directory"}) - ) - .await - .is_err() - ); - assert!( - largest.load(Ordering::Relaxed) < 400_000, - "large binary transfers must use bounded chunks" - ); - } -} - -#[tokio::test(flavor = "current_thread")] -async fn rpc_vfs_roundtrip_streams_large_files_reuses_bytes_and_preserves_retry_receipt() { - let environment = tempfile::tempdir().unwrap(); - let cas = tempfile::tempdir().unwrap(); - let store = store_fs::FsBlobStore::open(cas.path()).await.unwrap(); - let size = 10 * 1024 * 1024 + 7; - let blob = repeated_ref(size); - store - .put_stream(&blob, size, &mut Repeated { remaining: size }) + assert_eq!(rest.data.as_slice(), b"456789"); + assert!(!rest.truncated); + let committed = fs + .transfer_commit(&context(), operation("capture")) .await .unwrap(); - let mut directory = vfs::VfsDirectory::default(); - directory.entries.insert( - "run".into(), - vfs::VfsEntry::File(vfs::VfsFile { - blob_ref: blob.clone(), - size_bytes: size, - executable: true, - media_type: None, - }), - ); - directory.entries.insert( - "empty".into(), - vfs::VfsEntry::Directory(vfs::VfsDirectory::default()), - ); - let selection = vfs::VfsEntry::Directory(directory.clone()); - let (remote, largest, lose) = remote(runtime(environment.path(), false)); - let first = transfer::materialize( - &remote, - &store, - "first", - &selection, - path("selected"), - TransferOnExisting::Replace, - ) - .await - .unwrap(); - assert_eq!(first.bytes, size); - assert_eq!(first.transferred_bytes, size); - assert_eq!(first.reused_bytes, 0); - assert!(environment.path().join("selected/empty").is_dir()); + assert_eq!(committed.phase, TransferPhase::Complete); + assert!(committed.receipt_expires_at_ms.is_some()); + assert!(committed.deadline_at_ms.is_none()); + // The receipt answers a retried commit and begin from the journal. assert_eq!( - std::fs::metadata(environment.path().join("selected/run")) + fs.transfer_commit(&context(), operation("capture")) + .await .unwrap() - .permissions() - .mode() - & 0o111, - 0o111 + .phase, + TransferPhase::Complete ); - use std::os::unix::fs::MetadataExt; - let inode = std::fs::metadata(environment.path().join("selected/run")) - .unwrap() - .ino(); - let unchanged = transfer::materialize( - &remote, - &store, - "unchanged", - &selection, - path("selected"), - TransferOnExisting::Replace, + let error = begin( + &fs, + "capture", + capture(path(&root, "other"), TreeFilter::default()), ) .await - .unwrap(); - assert_eq!(unchanged.transferred_bytes, 0); - assert_eq!(unchanged.reused_bytes, size); - assert_eq!( - std::fs::metadata(environment.path().join("selected/run")) - .unwrap() - .ino(), - inode, - "identical complete trees need no restaging" - ); - // Rename, mode-only edit and deletion reuse raw bytes while replacing the whole boundary. - let vfs::VfsEntry::File(mut file) = directory.entries.remove("run").unwrap() else { - unreachable!() - }; - file.executable = false; - directory.entries.remove("empty"); - directory - .entries - .insert("renamed".into(), vfs::VfsEntry::File(file)); - let selection = vfs::VfsEntry::Directory(directory); - std::fs::write(environment.path().join("sibling"), b"preserved").unwrap(); - lose.store(true, Ordering::Relaxed); - assert!( - transfer::materialize( - &remote, - &store, - "replace", - &selection, - path("selected"), - TransferOnExisting::Replace + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::OperationIdInUse)); + + // The completed capture's inventory stays pageable with its receipt, + // after the source changed and across a restart; its content does not. + std::fs::remove_file(root.join("project/src/lib.rs")).unwrap(); + assert_eq!(inventory(&fs, "capture").await, entries); + drop(fs); + let fs = filesystem(&root, false); + assert_eq!(inventory(&fs, "capture").await, entries); + let error = fs + .transfer_read( + &context(), + ReadTransferContentParams { + operation_id: id("capture"), + digest: content, + offset: 0, + max_bytes: 4, + }, ) .await - .is_err() - ); - assert!(!environment.path().join("selected/run").exists()); - assert!(!environment.path().join("selected/empty").exists()); - std::fs::write( - environment.path().join("selected/local-edit"), - b"keep on retry", - ) - .unwrap(); - let retry = transfer::materialize( - &remote, - &store, - "replace", - &selection, - path("selected"), - TransferOnExisting::Replace, + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::TransferPhase)); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_failed_capture_answers_its_first_error_and_never_commits() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("root"); + std::fs::create_dir_all(root.join("tree/a")).unwrap(); + std::fs::create_dir_all(base.join("outside")).unwrap(); + std::fs::write(root.join("tree/a/file"), b"a").unwrap(); + symlink(base.join("outside"), root.join("tree/escape")).unwrap(); + std::fs::write(root.join("tree/z"), b"z").unwrap(); + let fs = filesystem(&root, false); + + begin( + &fs, + "escape", + capture(path(&root, "tree"), TreeFilter::default()), ) .await .unwrap(); - assert_eq!(retry.phase, TransferPhase::Complete); - assert_eq!(retry.transferred_bytes, 0); - assert_eq!(retry.reused_bytes, size); - assert!(environment.path().join("selected/local-edit").exists()); - std::fs::remove_file(environment.path().join("selected/local-edit")).unwrap(); - let captured = transfer::capture(&remote, &store, None, "capture-existing", path("selected")) + let first = advance_until(&fs, "escape", TransferPhase::Inventory) + .await + .unwrap_err(); + assert_eq!(reason(&first), Some(PreconditionReason::SymlinkRejected)); + // Retries never continue the walk past the failure. + for _ in 0..3 { + let again = fs + .transfer_advance(&context(), operation("escape")) + .await + .unwrap_err(); + assert_eq!(again, first); + } + let error = fs + .transfer_commit(&context(), operation("escape")) + .await + .unwrap_err(); + assert_eq!(error, first); + let status = fs + .transfer_get_status(&context(), operation("escape")) .await .unwrap(); - assert_eq!(captured.entry, selection); - assert_eq!(captured.status.transferred_bytes, 0); - let empty_cas = tempfile::tempdir().unwrap(); - let empty_store = store_fs::FsBlobStore::open(empty_cas.path()).await.unwrap(); - let captured = transfer::capture( - &remote, - &empty_store, - None, - "capture-missing", - path("selected"), + assert_eq!(status.phase, TransferPhase::Failed); + assert_eq!(status.error.as_ref(), Some(&first)); + let aborted = fs + .transfer_abort(&context(), operation("escape")) + .await + .unwrap(); + assert_eq!(aborted.phase, TransferPhase::Failed); + + // A source that changes after hashing fails the operation at read time. + std::fs::remove_file(root.join("tree/escape")).unwrap(); + begin( + &fs, + "changed", + capture(path(&root, "tree"), TreeFilter::default()), ) .await .unwrap(); - assert_eq!(captured.entry, selection); - assert_eq!(captured.status.transferred_bytes, size); - assert!(empty_store.has_blob(&blob).await.unwrap()); + advance_until(&fs, "changed", TransferPhase::Inventory) + .await + .unwrap(); + std::fs::write(root.join("tree/z"), b"zz").unwrap(); + let error = fs + .transfer_read( + &context(), + ReadTransferContentParams { + operation_id: id("changed"), + digest: digest(b"z"), + offset: 0, + max_bytes: 10, + }, + ) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::Aborted { + reason: Some(AbortReason::SourceChanged) + } + )); + let error = fs + .transfer_commit(&context(), operation("changed")) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::Aborted); +} + +#[tokio::test(flavor = "current_thread")] +async fn entries_outside_the_selection_never_fail_a_capture() { + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("root"); + std::fs::create_dir_all(root.join("tree/src")).unwrap(); + std::fs::create_dir_all(root.join("tree/tmp")).unwrap(); + std::fs::create_dir_all(base.join("outside")).unwrap(); + std::fs::write(root.join("tree/src/lib.rs"), b"lib").unwrap(); + let _socket = std::os::unix::net::UnixListener::bind(root.join("tree/tmp/sock")).unwrap(); + symlink(base.join("outside"), root.join("tree/tmp/escape")).unwrap(); + symlink("missing", root.join("tree/tmp/dangling")).unwrap(); + let fs = filesystem(&root, false); + + let filter = TreeFilter { + include: vec!["src/**".to_owned()], + ..TreeFilter::default() + }; + begin(&fs, "selected", capture(path(&root, "tree"), filter)) + .await + .unwrap(); + let status = advance_until(&fs, "selected", TransferPhase::Inventory) + .await + .unwrap(); + assert_eq!(status.skipped_entry_count, 0, "{:?}", status.skipped); + let entries = inventory(&fs, "selected").await; assert!( - largest.load(Ordering::Relaxed) < 512 * 1024, - "every JSON message remains bounded" + entries + .iter() + .any(|entry| entry.path.as_str() == "src/lib.rs") ); - assert_eq!( - std::fs::read(environment.path().join("sibling")).unwrap(), - b"preserved" + assert!( + !entries + .iter() + .any(|entry| entry.path.as_str().starts_with("tmp/")) ); + + // The same entries fail a capture that selects them. + begin( + &fs, + "everything", + capture(path(&root, "tree"), TreeFilter::default()), + ) + .await + .unwrap(); + let error = advance_until(&fs, "everything", TransferPhase::Inventory) + .await + .unwrap_err(); + assert!(matches!( + reason(&error), + Some(PreconditionReason::SymlinkRejected | PreconditionReason::NotRegularFile) + )); } + #[tokio::test(flavor = "current_thread")] -async fn inventories_are_paged_and_twenty_gib_files_do_not_block_begin_or_one_advance() { +async fn skip_policies_report_what_a_capture_left_out() { let temp = tempfile::tempdir().unwrap(); - std::fs::create_dir(temp.path().join("many")).unwrap(); - for index in 0..300 { - std::fs::write(temp.path().join(format!("many/{index:04}")), b"same").unwrap(); - } - let runtime = runtime(temp.path(), false); - let fs = runtime.filesystem(); - fs.transfer(TransferRequest::Begin { - operation_id: "pages".into(), - selection: TransferSelection::Capture { - source: path("many"), + let root = temp.path().canonicalize().unwrap(); + std::fs::create_dir_all(root.join("tree")).unwrap(); + std::fs::write(root.join("tree/file"), b"f").unwrap(); + symlink("file", root.join("tree/link")).unwrap(); + let fs = filesystem(&root, false); + begin( + &fs, + "skips", + TransferSelection::Capture { + source: path(&root, "tree"), + filter: TreeFilter::default(), + symlinks: SymlinkPolicy::Skip, + special_files: SpecialFilePolicy::Skip, }, - limits: Default::default(), - }) + ) .await .unwrap(); - loop { - if let TransferResponse::Status(status) = fs - .transfer(TransferRequest::Advance { - operation_id: "pages".into(), - }) - .await - .unwrap() - && status.phase == TransferPhase::Ready - { - break; - } - } - let mut offset = 0; - let mut count = 0; - loop { - let TransferResponse::Inventory { - entries, - next_offset, - } = fs - .transfer(TransferRequest::Inventory { - operation_id: "pages".into(), - offset, - }) - .await - .unwrap() - else { - panic!() - }; - assert!(entries.len() <= MAX_INVENTORY_PAGE); - count += entries.len(); - if let Some(next) = next_offset { - offset = next; - } else { - break; - } - } - assert_eq!(count, 301); - let file = std::fs::File::create(temp.path().join("twenty-gib")).unwrap(); - file.set_len(20 * 1024 * 1024 * 1024).unwrap(); - let TransferResponse::Status(start) = fs - .transfer(TransferRequest::Begin { - operation_id: "large".into(), - selection: TransferSelection::Capture { - source: path("twenty-gib"), - }, - limits: Default::default(), - }) + let status = advance_until(&fs, "skips", TransferPhase::Inventory) .await - .unwrap() - else { - panic!() - }; - assert_eq!(start.phase, TransferPhase::Scanning); - let TransferResponse::Status(step) = fs - .transfer(TransferRequest::Advance { - operation_id: "large".into(), - }) - .await - .unwrap() - else { - panic!() - }; - assert_eq!(step.phase, TransferPhase::Scanning); - fs.transfer(TransferRequest::Abort { - operation_id: "large".into(), - }) - .await - .unwrap(); - assert_eq!(file.metadata().unwrap().len(), 20 * 1024 * 1024 * 1024); + .unwrap(); + assert_eq!(status.skipped_entry_count, 1); + assert_eq!(status.skipped[0].reason, TransferSkipReason::Symlink); + assert_eq!( + status.skipped[0].path.as_ref().map(TreePath::as_str), + Some("link") + ); } + #[tokio::test(flavor = "current_thread")] -async fn scan_fingerprints_content_and_filters_large_unrelated_files() { +async fn finished_operations_free_capacity_and_receipts_expire_from_completion() { let temp = tempfile::tempdir().unwrap(); - std::fs::create_dir(temp.path().join("skills")).unwrap(); - std::fs::write(temp.path().join("skills/SKILL.md"), b"first").unwrap(); - std::fs::File::create(temp.path().join("skills/large")) - .unwrap() - .set_len(20 * 1024 * 1024 * 1024) + let root = temp.path().canonicalize().unwrap(); + std::fs::write(root.join("file"), b"x").unwrap(); + let fs = filesystem(&root, false); + // Twice the active ceiling of eight, each finished before the next. + for index in 0..16 { + let name = format!("done-{index}"); + begin( + &fs, + &name, + capture(path(&root, "file"), TreeFilter::default()), + ) + .await .unwrap(); - let runtime = runtime(temp.path(), true); - let fs = runtime.filesystem(); - let mut query = ScanParams { - follow_symlinks: false, - roots: vec![path("skills")], - include_patterns: vec!["**/SKILL.md".into(), "SKILL.md".into()], - read_content: true, - digest_algorithm: None, - limits: Default::default(), - if_none_match: None, - }; - let first = fs.scan(query.clone()).await.unwrap(); - assert!(first.complete); - assert_eq!(first.entries.len(), 1); - query.if_none_match = first.fingerprint; - assert!(fs.scan(query.clone()).await.unwrap().unchanged); - std::fs::write(temp.path().join("skills/SKILL.md"), b"other").unwrap(); - assert!(!fs.scan(query.clone()).await.unwrap().unchanged); - query.roots.push(path("missing")); - let absent = fs.scan(query).await.unwrap(); - assert!(absent.complete); - assert!(!absent.unchanged); - assert!(absent.diagnostics.is_empty()); - let metadata = fs - .scan(ScanParams { - follow_symlinks: false, - roots: vec![path("skills/large")], - include_patterns: vec![], - read_content: false, - digest_algorithm: None, - limits: Default::default(), - if_none_match: None, - }) + advance_until(&fs, &name, TransferPhase::Inventory) + .await + .unwrap(); + fs.transfer_commit(&context(), operation(&name)) + .await + .unwrap(); + } + for index in 0..8 { + begin( + &fs, + &format!("active-{index}"), + capture(path(&root, "file"), TreeFilter::default()), + ) .await .unwrap(); - assert!(metadata.complete); + } + let error = begin( + &fs, + "one-more", + capture(path(&root, "file"), TreeFilter::default()), + ) + .await + .unwrap_err(); assert!(matches!( - metadata.entries[0].content, - ScanContent::File { - digest: None, - size_bytes: 21_474_836_480, + error.data, + ErrorData::Busy { + reason: Some(BusyReason::TransferCapacity), .. } )); - let hashed = fs - .scan(ScanParams { - follow_symlinks: false, - roots: vec![path("skills/SKILL.md")], - include_patterns: vec![], - read_content: false, - digest_algorithm: Some(ScanDigestAlgorithm::Sha256), - limits: Default::default(), - if_none_match: None, - }) + fs.transfer_abort(&context(), operation("active-0")) .await .unwrap(); - assert!(hashed.complete); - assert!( - matches!(&hashed.entries[0].content,ScanContent::File {digest:Some(digest),..} if digest==BlobRef::from_bytes(b"other").as_str()) - ); + begin( + &fs, + "one-more", + capture(path(&root, "file"), TreeFilter::default()), + ) + .await + .unwrap(); } -#[tokio::test(flavor = "current_thread")] -async fn rpc_materialize_creates_missing_parents_for_files_and_trees() { - let environment = tempfile::tempdir().unwrap(); - std::fs::write(environment.path().join("sibling"), b"preserved").unwrap(); - let (remote, _, _) = remote(runtime(environment.path(), false)); - let store = harness::storage::InMemoryBlobStore::new(); - let entry = vfs::VfsEntry::File(vfs::VfsFile { - blob_ref: store.put_bytes(b"content".to_vec()).await.unwrap(), - size_bytes: 7, - executable: true, - media_type: None, - }); - let mut directory = vfs::VfsDirectory::default(); - directory.entries.insert("file".into(), entry.clone()); - for (id, entry, on_existing) in [ - ("file", entry, TransferOnExisting::Error), - ( - "tree", - vfs::VfsEntry::Directory(directory), - TransferOnExisting::Replace, - ), - ] { - let destination = path(&format!("created/{id}/nested/selected")); - let status = transfer::materialize( - &remote, - &store, - id, - &entry, - destination.clone(), - on_existing, + +async fn append_all(fs: &LocalFileSystem, operation_id: &str, entries: Vec) { + let status = fs + .transfer_append( + &context(), + AppendTransferInventoryParams { + operation_id: id(operation_id), + index: 0, + entries, + last: true, + }, ) .await .unwrap(); - assert_eq!(status.phase, TransferPhase::Complete); - let captured = - transfer::capture(&remote, &store, None, &format!("capture-{id}"), destination) - .await - .unwrap(); - assert_eq!(captured.entry, entry); + assert_eq!(status.phase, TransferPhase::Scanning); +} + +fn file_entry(path: &str, bytes: &[u8], executable: bool) -> InventoryEntry { + InventoryEntry { + path: TreePath::new(path).unwrap(), + content: TreeContent::File { + size_bytes: bytes.len() as u64, + executable, + digest: Some(digest(bytes)), + }, + } +} + +fn directory_entry(path: &str) -> InventoryEntry { + InventoryEntry { + path: TreePath::new(path).unwrap(), + content: TreeContent::Directory {}, } - assert_eq!( - std::fs::read(environment.path().join("sibling")).unwrap(), - b"preserved" - ); } #[tokio::test(flavor = "current_thread")] -async fn secure_replacement_rejects_links_and_uses_private_staging_without_reading_write_only_targets() - { +async fn materialize_reuses_matching_destination_files_and_publishes_atomically() { let temp = tempfile::tempdir().unwrap(); - let outside = tempfile::tempdir().unwrap(); - std::fs::write(outside.path().join("secret"), b"untouched").unwrap(); - symlink(outside.path(), temp.path().join("escape")).unwrap(); - symlink("absent", temp.path().join("dangling")).unwrap(); - std::fs::write(temp.path().join("blocked"), b"not a directory").unwrap(); - let read_only = runtime(temp.path(), true); - let runtime = runtime(temp.path(), false); - let fs = runtime.filesystem(); - for (index, destination) in [ - "escape/secret", - "escape/missing/secret", - "dangling/missing/secret", - "blocked/missing/secret", - ] - .into_iter() - .enumerate() - { - assert_eq!( - fs.transfer(TransferRequest::Begin { - operation_id: format!("invalid-parent-{index}"), - selection: TransferSelection::Materialize { - destination: path(destination), - on_existing: TransferOnExisting::Replace - }, - limits: Default::default() - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Forbidden - ); - } - assert!(!outside.path().join("missing").exists()); - assert!(!temp.path().join("absent").exists()); - assert_eq!( - std::fs::read(outside.path().join("secret")).unwrap(), - b"untouched" - ); - assert_eq!( - std::fs::read(temp.path().join("blocked")).unwrap(), - b"not a directory" - ); - assert_eq!( - read_only - .filesystem() - .transfer(TransferRequest::Begin { - operation_id: "read-only-parents".into(), - selection: TransferSelection::Materialize { - destination: path("read-only/missing/target"), - on_existing: TransferOnExisting::Replace - }, - limits: Default::default() - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::CapabilityUnavailable - ); - assert!(!temp.path().join("read-only").exists()); - std::fs::write(temp.path().join("target"), b"old").unwrap(); - std::fs::set_permissions( - temp.path().join("target"), - std::fs::Permissions::from_mode(0o200), + let root = temp.path().canonicalize().unwrap(); + std::fs::create_dir_all(root.join("target/old")).unwrap(); + std::fs::write(root.join("target/old/kept"), b"reused bytes").unwrap(); + std::fs::write(root.join("target/unrelated"), vec![7u8; 100_000]).unwrap(); + let fs = filesystem(&root, true); + + let error = begin( + &fs, + "refused", + TransferSelection::Materialize { + destination: path(&root, "target"), + on_existing: TransferOnExisting::Error, + }, ) - .unwrap(); - fs.transfer(TransferRequest::Begin { - operation_id: "private".into(), - selection: TransferSelection::Materialize { - destination: path("target"), + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::AlreadyExists); + + begin( + &fs, + "tree", + TransferSelection::Materialize { + destination: path(&root, "target"), on_existing: TransferOnExisting::Replace, }, - limits: Default::default(), - }) + ) .await .unwrap(); - for entry in std::fs::read_dir(temp.path()).unwrap() { - let entry = entry.unwrap(); - if entry - .file_name() - .to_string_lossy() - .starts_with(".env-transfer-") - { - assert_eq!( - entry.metadata().unwrap().permissions().mode() & 0o777, - 0o700 - ); - } - } - assert_eq!( - fs.transfer(TransferRequest::Begin { - operation_id: "overlap".into(), - selection: TransferSelection::Materialize { - destination: path("target/nested"), - on_existing: TransferOnExisting::Replace + // A file entry without its digest is refused and changes nothing. + let error = fs + .transfer_append( + &context(), + AppendTransferInventoryParams { + operation_id: id("tree"), + index: 0, + entries: vec![ + directory_entry("."), + InventoryEntry { + path: TreePath::new("bad").unwrap(), + content: TreeContent::File { + size_bytes: 1, + executable: false, + digest: None, + }, + }, + ], + last: true, }, - limits: Default::default() - }) + ) .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Conflict - ); - fs.transfer(TransferRequest::Abort { - operation_id: "private".into(), - }) - .await - .unwrap(); - let (remote, _, _) = remote(runtime.clone()); - let store = harness::storage::InMemoryBlobStore::new(); - let blob = store.put_bytes(b"new".to_vec()).await.unwrap(); - let entry = vfs::VfsEntry::File(vfs::VfsFile { - blob_ref: blob, - size_bytes: 3, - executable: false, - media_type: None, - }); - transfer::materialize( - &remote, - &store, - "write-only", - &entry, - path("target"), - TransferOnExisting::Replace, + .unwrap_err(); + assert!(matches!( + &error.data, + ErrorData::InvalidParams { field: Some(field), .. } if field == "entries[1].content.digest" + )); + let uploaded = b"uploaded content".to_vec(); + append_all( + &fs, + "tree", + vec![ + directory_entry("."), + directory_entry("bin"), + file_entry("bin/run", &uploaded, true), + file_entry("copy", b"reused bytes", false), + file_entry("empty", b"", false), + ], ) - .await - .unwrap(); - assert_eq!(std::fs::read(temp.path().join("target")).unwrap(), b"new"); + .await; + advance_until(&fs, "tree", TransferPhase::Content) + .await + .unwrap(); + let missing = fs + .transfer_list_missing( + &context(), + ListTransferMissingParams { + operation_id: id("tree"), + limit: 10, + }, + ) + .await + .unwrap(); + assert_eq!(missing.digests, [digest(&uploaded)]); + assert_eq!(missing.remaining_count, 0); + let write = |offset: u64, data: &[u8]| WriteTransferContentParams { + operation_id: id("tree"), + digest: digest(&uploaded), + offset, + data: data.into(), + }; + let next = fs + .transfer_write(&context(), write(0, &uploaded[..8])) + .await + .unwrap() + .next_offset; + assert_eq!(next, 8); + // An identical repeat is accepted; a gap is refused without failing. assert_eq!( - std::fs::read(outside.path().join("secret")).unwrap(), - b"untouched" + fs.transfer_write(&context(), write(0, &uploaded[..8])) + .await + .unwrap() + .next_offset, + 8 ); - transfer::materialize( - &remote, - &store, - "to-directory", - &vfs::VfsEntry::Directory(vfs::VfsDirectory::default()), - path("target"), - TransferOnExisting::Replace, - ) - .await - .unwrap(); - assert!(temp.path().join("target").is_dir()); - transfer::materialize( - &remote, - &store, - "back-to-file", - &entry, - path("target"), - TransferOnExisting::Replace, + let error = fs + .transfer_write(&context(), write(10, &uploaded[10..])) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::TransferOffset)); + fs.transfer_write(&context(), write(8, &uploaded[8..])) + .await + .unwrap(); + let ready = advance_until(&fs, "tree", TransferPhase::Ready) + .await + .unwrap(); + assert_eq!(ready.reused_bytes, b"reused bytes".len() as u64); + fs.transfer_commit(&context(), operation("tree")) + .await + .unwrap(); + assert_eq!( + std::fs::read(root.join("target/bin/run")).unwrap(), + uploaded + ); + assert_eq!( + std::fs::read(root.join("target/copy")).unwrap(), + b"reused bytes" + ); + assert!(root.join("target/empty").exists()); + assert!(!root.join("target/unrelated").exists()); + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(root.join("target/bin/run")) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o111, 0o111); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_digest_mismatch_fails_the_operation() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + begin( + &fs, + "mismatch", + TransferSelection::Materialize { + destination: path(&root, "out"), + on_existing: TransferOnExisting::Replace, + }, ) .await .unwrap(); - - let capture = fs - .transfer(TransferRequest::Begin { - operation_id: "observe".into(), - selection: TransferSelection::Capture { - source: path("target"), - }, - limits: Default::default(), - }) + append_all(&fs, "mismatch", vec![file_entry(".", b"expected", false)]).await; + advance_until(&fs, "mismatch", TransferPhase::Content) .await .unwrap(); - assert!(matches!(capture, TransferResponse::Status(_))); - loop { - if let TransferResponse::Status(status) = fs - .transfer(TransferRequest::Advance { - operation_id: "observe".into(), - }) - .await - .unwrap() - && status.phase == TransferPhase::Ready - { - break; - } - } - std::fs::write(temp.path().join("target"), b"changed").unwrap(); - assert_eq!( - fs.transfer(TransferRequest::Commit { - operation_id: "observe".into() - }) + let error = fs + .transfer_write( + &context(), + WriteTransferContentParams { + operation_id: id("mismatch"), + digest: digest(b"expected"), + offset: 0, + data: b"tampered".as_slice().into(), + }, + ) .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Conflict - ); + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::DigestMismatch)); + let status = fs + .transfer_get_status(&context(), operation("mismatch")) + .await + .unwrap(); + assert_eq!(status.phase, TransferPhase::Failed); + assert!(!root.join("out").exists()); } #[tokio::test(flavor = "current_thread")] -async fn persisted_receipts_survive_restart_and_interrupted_operations_fail_closed() { +async fn transfer_write_file_follows_fs_write_file_semantics() { let temp = tempfile::tempdir().unwrap(); - let cas = tempfile::tempdir().unwrap(); - let store = store_fs::FsBlobStore::open(cas.path()).await.unwrap(); - let blob = store.put_bytes(b"content".to_vec()).await.unwrap(); - let entry = vfs::VfsEntry::File(vfs::VfsFile { - blob_ref: blob, - size_bytes: 7, + let root = temp.path().canonicalize().unwrap(); + std::fs::write(root.join("target"), b"old").unwrap(); + symlink("target", root.join("link")).unwrap(); + let fs = filesystem(&root, true); + let selection = |create: WriteCondition| TransferSelection::WriteFile { + destination: path(&root, "link"), + create, + if_match: None, executable: false, - media_type: None, - }); - { - let (remote, _, _) = remote(runtime(temp.path(), false)); - transfer::materialize( - &remote, - &store, - "completed", - &entry, - path("target"), - TransferOnExisting::Replace, - ) + }; + let error = begin(&fs, "absent", selection(WriteCondition::IfAbsent)) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::CreateCondition)); + begin(&fs, "write", selection(WriteCondition::IfPresent)) .await .unwrap(); - transfer::capture(&remote, &store, None, "captured", path("target")) - .await - .unwrap(); - use tools::transfer::EnvironmentTransfer; - remote - .request(TransferRequest::Begin { - operation_id: "interrupted".into(), - selection: TransferSelection::Materialize { - destination: path("target"), - on_existing: TransferOnExisting::Replace, - }, - limits: Default::default(), - }) - .await - .unwrap(); - } - std::fs::write(temp.path().join("target"), b"later edit").unwrap(); - let (remote, _, _) = remote(runtime(temp.path(), false)); - let receipt = transfer::materialize( - &remote, - &store, - "completed", - &entry, - path("target"), - TransferOnExisting::Replace, + append_all(&fs, "write", vec![file_entry(".", b"new content", false)]).await; + advance_until(&fs, "write", TransferPhase::Content) + .await + .unwrap(); + fs.transfer_write( + &context(), + WriteTransferContentParams { + operation_id: id("write"), + digest: digest(b"new content"), + offset: 0, + data: b"new content".as_slice().into(), + }, ) .await .unwrap(); - assert_eq!(receipt.phase, TransferPhase::Complete); - assert_eq!( - std::fs::read(temp.path().join("target")).unwrap(), - b"later edit" - ); - let capture = transfer::capture(&remote, &store, None, "captured", path("target")) + advance_until(&fs, "write", TransferPhase::Ready) .await .unwrap(); - assert_eq!(capture.entry, entry); - assert!( - transfer::materialize( - &remote, - &store, - "interrupted", - &entry, - path("target"), - TransferOnExisting::Replace - ) + fs.transfer_commit(&context(), operation("write")) .await - .is_err() - ); - assert_eq!( - std::fs::read(temp.path().join("target")).unwrap(), - b"later edit" + .unwrap(); + assert_eq!(std::fs::read(root.join("target")).unwrap(), b"new content"); + assert!( + std::fs::symlink_metadata(root.join("link")) + .unwrap() + .file_type() + .is_symlink() ); } + #[tokio::test(flavor = "current_thread")] -async fn bad_chunks_invalid_manifests_and_read_only_requests_never_publish() { +async fn read_only_daemons_refuse_publishes() { let temp = tempfile::tempdir().unwrap(); - std::fs::write(temp.path().join("old"), b"keep").unwrap(); - let runtime = runtime(temp.path(), false); - let fs = runtime.filesystem(); - fs.transfer(TransferRequest::Begin { - operation_id: "invalid".into(), - selection: TransferSelection::Materialize { - destination: path("old"), + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, false); + let error = begin( + &fs, + "denied", + TransferSelection::Materialize { + destination: path(&root, "out"), on_existing: TransferOnExisting::Replace, }, - limits: Default::default(), - }) + ) .await - .unwrap(); - loop { - if let TransferResponse::Status(status) = fs - .transfer(TransferRequest::Advance { - operation_id: "invalid".into(), - }) - .await - .unwrap() - && status.phase == TransferPhase::Inventory - { - break; - } - } - let digest = BlobRef::from_bytes(b"new").to_string(); - let file = |path: &str| InventoryEntry { - path: path.into(), - content: InventoryContent::File { - size_bytes: 3, - executable: false, - digest: digest.clone(), - }, + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::CapabilityUnavailable); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_restart_fails_active_operations_and_discards_their_staging() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let selection = TransferSelection::Materialize { + destination: path(&root, "out"), + on_existing: TransferOnExisting::Replace, }; - for bad in ["../escape", "/absolute", "a//b", "a/../b", "a\\b"] { - assert_eq!( - fs.transfer(TransferRequest::Append { - operation_id: "invalid".into(), - offset: 0, - entries: vec![file(bad)], - last: true - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::InvalidRequest - ); + { + let fs = filesystem(&root, true); + begin(&fs, "interrupted", selection.clone()).await.unwrap(); + let staged = std::fs::read_dir(&root).unwrap().flatten().any(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".lightspeed-transfer-") + }); + assert!(staged); } - let page = TransferRequest::Append { - operation_id: "invalid".into(), - offset: 0, - entries: vec![file("")], - last: true, - }; - fs.transfer(page.clone()).await.unwrap(); - fs.transfer(page).await.unwrap(); - assert_eq!( - fs.transfer(TransferRequest::Write { - operation_id: "invalid".into(), - digest: digest.clone(), - offset: 0, - data: vec![0; MAX_CONTENT_CHUNK + 1].into() - }) + let fs = filesystem(&root, true); + let status = fs + .transfer_get_status(&context(), operation("interrupted")) .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::InvalidRequest - ); + .unwrap(); + assert_eq!(status.phase, TransferPhase::Failed); assert_eq!( - fs.transfer(TransferRequest::Write { - operation_id: "invalid".into(), - digest: digest.clone(), - offset: 0, - data: b"bad".as_slice().into() - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Conflict - ); - assert_eq!(std::fs::read(temp.path().join("old")).unwrap(), b"keep"); - let chunk = TransferRequest::Write { - operation_id: "invalid".into(), - digest, - offset: 0, - data: b"new".as_slice().into(), - }; - fs.transfer(chunk.clone()).await.unwrap(); - fs.transfer(chunk).await.unwrap(); - fs.transfer(TransferRequest::Abort { - operation_id: "invalid".into(), - }) + status.error.as_ref().map(ProtocolError::kind), + Some(ProtocolErrorKind::Aborted) + ); + let error = begin(&fs, "interrupted", selection).await.unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::Aborted); + // Staging cleanup of the dropped instance and recovery both remove it. + for _ in 0..100 { + let staged = std::fs::read_dir(&root).unwrap().flatten().any(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".lightspeed-transfer-") + }); + if !staged { + return; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + panic!("staging survived the restart"); +} + +#[tokio::test(flavor = "current_thread")] +async fn operations_fail_when_their_duration_runs_out() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + std::fs::write(root.join("file"), b"x").unwrap(); + let fs = filesystem(&root, false); + fs.transfer_begin( + &context(), + BeginTransferParams { + operation_id: id("short"), + selection: capture(path(&root, "file"), TreeFilter::default()), + limits: TransferLimits { + max_duration_ms: 1, + ..limits() + }, + }, + ) .await .unwrap(); - assert_eq!( - fs.transfer(TransferRequest::Commit { - operation_id: "invalid".into() - }) + std::thread::sleep(std::time::Duration::from_millis(5)); + let error = fs + .transfer_advance(&context(), operation("short")) .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::Conflict - ); - let read_only = crate::server::streaming_transfer_tests::runtime(temp.path(), true); - assert_eq!( - read_only - .filesystem() - .transfer(TransferRequest::Begin { - operation_id: "denied".into(), - selection: TransferSelection::Materialize { - destination: path("old"), - on_existing: TransferOnExisting::Replace - }, - limits: Default::default() - }) - .await - .unwrap_err() - .code, - EnvironmentProtocolErrorCode::CapabilityUnavailable - ); - assert_eq!(std::fs::read(temp.path().join("old")).unwrap(), b"keep"); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::DeadlineExceeded); + let status = fs + .transfer_get_status(&context(), operation("short")) + .await + .unwrap(); + assert_eq!(status.phase, TransferPhase::Failed); } #[tokio::test(flavor = "current_thread")] -async fn configured_cwd_aliases_work_under_the_host_root() { +async fn a_retried_sealing_append_answers_the_status() { let temp = tempfile::tempdir().unwrap(); - let alias = temp.path().join("cwd-alias"); - let actual = temp.path().join("actual"); - std::fs::create_dir(&actual).unwrap(); - symlink(&actual, &alias).unwrap(); - std::fs::write(actual.join("file"), b"bytes").unwrap(); - let fs = crate::filesystem::LocalFileSystem::new(Path::new("/").into(), alias, true); - let result = fs - .scan(ScanParams { - follow_symlinks: false, - roots: vec![path("file")], - include_patterns: vec![], - read_content: true, - digest_algorithm: None, - limits: Default::default(), - if_none_match: None, - }) + let root = temp.path().canonicalize().unwrap(); + let fs = filesystem(&root, true); + begin( + &fs, + "sealed", + TransferSelection::Materialize { + destination: path(&root, "out"), + on_existing: TransferOnExisting::Replace, + }, + ) + .await + .unwrap(); + let entries = vec![directory_entry("."), file_entry("file", b"bytes", false)]; + append_all(&fs, "sealed", entries.clone()).await; + // The response to the sealing page was lost; the client sends it again. + let retried = fs + .transfer_append( + &context(), + AppendTransferInventoryParams { + operation_id: id("sealed"), + index: 0, + entries, + last: true, + }, + ) .await .unwrap(); - assert!(result.complete, "{:?}", result.diagnostics); - assert_eq!( - result.entries[0].data.as_ref().unwrap().as_slice(), - b"bytes" - ); - symlink(&actual, actual.join("user-link")).unwrap(); - let result = fs - .scan(ScanParams { - follow_symlinks: false, - roots: vec![path("user-link/file")], - include_patterns: vec![], - read_content: true, - digest_algorithm: None, - limits: Default::default(), - if_none_match: None, - }) + assert_ne!(retried.phase, TransferPhase::Inventory); + assert_eq!(retried.entry_count, 2); + let error = fs + .transfer_append( + &context(), + AppendTransferInventoryParams { + operation_id: id("sealed"), + index: 0, + entries: vec![directory_entry(".")], + last: true, + }, + ) .await - .unwrap(); - assert!(!result.complete, "user-selected symlinks remain rejected"); + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::TransferPhase)); } #[tokio::test(flavor = "current_thread")] -async fn scan_installer_layouts_aliases_collisions_and_semantic_publication() { - use tools::skills::environment::*; +async fn a_refused_begin_and_an_abort_leave_no_created_directories() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().canonicalize().unwrap(); - let home = root.join("home"); - let cwd = root.join("project/subdir"); - let canonical = root.join("installed/review"); - for directory in [ - &home, - &cwd, - &canonical, - &root.join("project/.agents/skills"), - &home.join(".codex/skills/copy"), - ] { - std::fs::create_dir_all(directory).unwrap(); - } - let doc = "---\nname: review\ndescription: >-\n Review code\n carefully.\nmetadata:\n hooks: [ignored]\n---\nbody one"; - std::fs::write(canonical.join("SKILL.md"), doc).unwrap(); - std::fs::write(home.join(".codex/skills/copy/SKILL.md"), doc).unwrap(); - symlink(&canonical, root.join("project/.agents/skills/review")).unwrap(); - symlink(&canonical, root.join("project/.agents/skills/alias")).unwrap(); - let config = harness::EnvironmentSkillsConfig { - roots: Some(vec![ - root.join("project/.agents/skills") - .to_string_lossy() - .into_owned(), - home.join(".codex/skills").to_string_lossy().into_owned(), - ]), + let fs = filesystem(&root, true); + let materialize = |destination: &str| TransferSelection::Materialize { + destination: path(&root, destination), + on_existing: TransferOnExisting::Replace, }; - let runtime = runtime(&root, true); - let mut query = environment_skill_scan_query(&config, cwd.to_str(), home.to_str()).unwrap(); - let first = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(first.complete, "{:?}", first.diagnostics); - let catalog = environment_skill_catalog("machine-a", &first).unwrap(); - assert_eq!( - catalog.skills.len(), - 2, - "aliases collapse but independent copies remain" - ); - assert!( - catalog - .skills - .iter() - .all(|skill| skill.description == "Review code carefully.") - ); - assert_ne!(catalog.skills[0].skill_id, catalog.skills[1].skill_id); - let other = environment_skill_catalog("machine-b", &first).unwrap(); - assert_ne!(catalog.skills[0].skill_id, other.skills[0].skill_id); - query.if_none_match = first.fingerprint; - assert!( - runtime - .filesystem() - .scan(query.clone()) - .await - .unwrap() - .unchanged - ); - std::fs::write(canonical.join("SKILL.md"), doc.replace("one", "two")).unwrap(); - let body_edit = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(!body_edit.unchanged); - assert_eq!( - catalog, - environment_skill_catalog("machine-a", &body_edit).unwrap() - ); - assert!( - String::from_utf8(std::fs::read(canonical.join("SKILL.md")).unwrap()) - .unwrap() - .ends_with("body two") - ); - let blobs = harness::storage::InMemoryBlobStore::new(); - let Some(harness::CoreAgentCommand::UpsertContext { entry, .. }) = - publish_environment_skill_catalog(&blobs, None, &catalog) - .await - .unwrap() - else { - panic!("publication"); - }; - assert!( - publish_environment_skill_catalog( - &blobs, - Some(&entry), - &environment_skill_catalog("machine-a", &body_edit).unwrap() - ) + begin(&fs, "outer", materialize("a/b")).await.unwrap(); + assert!(root.join("a").is_dir(), "begin creates missing parents"); + let error = begin(&fs, "inner", materialize("a/b/c/d")) .await - .unwrap() - .is_none() - ); - let prefix = blobs.read_bytes(&entry.content.content_ref).await.unwrap(); - std::fs::write( - canonical.join("SKILL.md"), - doc.replace("carefully", "thoroughly"), - ) - .unwrap(); - let edited = runtime.filesystem().scan(query.clone()).await.unwrap(); + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::Busy { + reason: Some(BusyReason::TransferCapacity), + .. + } + )); assert!( - publish_environment_skill_catalog( - &blobs, - Some(&entry), - &environment_skill_catalog("machine-a", &edited).unwrap() - ) - .await - .unwrap() - .is_some() - ); - assert_eq!( - blobs.read_bytes(&entry.content.content_ref).await.unwrap(), - prefix - ); - std::fs::remove_file(canonical.join("SKILL.md")).unwrap(); - let removed = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert_eq!( - environment_skill_catalog("machine-a", &removed) - .unwrap() - .skills - .len(), - 1 + !root.join("a/b").exists(), + "a refused begin changes nothing" ); - std::fs::write( - canonical.join("SKILL.md"), - "---\nname: broken\ndescription: [invalid]\n---", - ) - .unwrap(); - let invalid = environment_skill_catalog( - "machine-a", - &runtime.filesystem().scan(query).await.unwrap(), - ) - .unwrap(); - assert_eq!(invalid.skills.len(), 1); - assert_eq!(invalid.warnings.len(), 1); + + fs.transfer_abort(&context(), operation("outer")) + .await + .unwrap(); + for _ in 0..100 { + if !root.join("a").exists() { + return; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + panic!("the aborted begin's parent directory survived"); } #[tokio::test(flavor = "current_thread")] -async fn scan_retarget_scope_loops_and_aggregate_limits_are_explicit() { +async fn a_receipt_names_its_staging_until_removal_is_confirmed() { + use super::journal::{self, Record, StageRecord}; + let temp = tempfile::tempdir().unwrap(); - let outside = tempfile::tempdir().unwrap(); - for name in ["one", "two"] { - std::fs::create_dir(temp.path().join(name)).unwrap(); - std::fs::write(temp.path().join(name).join("SKILL.md"), "same").unwrap(); - } - symlink(temp.path().join("one"), temp.path().join("alias")).unwrap(); - let runtime = runtime(temp.path(), true); - let mut query = ScanParams { - roots: vec![path("alias")], - include_patterns: vec!["**/SKILL.md".into(), "SKILL.md".into()], - read_content: true, - follow_symlinks: true, - digest_algorithm: Some(ScanDigestAlgorithm::Sha256), - limits: InventoryLimits::default(), - if_none_match: None, + let root = temp.path().canonicalize().unwrap(); + let selection = TransferSelection::Materialize { + destination: path(&root, "out"), + on_existing: TransferOnExisting::Replace, }; - let first = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(first.complete); - query.if_none_match = first.fingerprint; - let parent_scope = environment_daemon_scope_for_test(temp.path()); - assert!( - !parent_scope.scan(query.clone()).await.unwrap().unchanged, - "changed access scope cannot authorize reuse" - ); - let mut shallow = query.clone(); - shallow.limits.max_depth = 0; - let truncated = runtime.filesystem().scan(shallow).await.unwrap(); - assert!(!truncated.complete && !truncated.unchanged); - let mut directories = query.clone(); - directories.include_patterns.clear(); - directories.read_content = false; - directories.digest_algorithm = None; - assert!( - runtime - .filesystem() - .scan(directories) - .await - .unwrap() - .entries - .iter() - .any(|entry| matches!(entry.content, ScanContent::Directory)) - ); - std::fs::remove_file(temp.path().join("alias")).unwrap(); - symlink(temp.path().join("two"), temp.path().join("alias")).unwrap(); - let retargeted = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(retargeted.complete && !retargeted.unchanged); - assert_ne!( - first.entries[0].canonical_path, - retargeted.entries[0].canonical_path - ); - assert!( - matches!(&retargeted.entries[0].content, ScanContent::File { digest: Some(digest), .. } if digest == BlobRef::from_bytes(b"same").as_str()) - ); - query.if_none_match = retargeted.fingerprint; - query.read_content = false; - assert!( - !runtime - .filesystem() - .scan(query.clone()) - .await - .unwrap() - .unchanged - ); - query.roots = vec![path("one"), path("two")]; - query.limits.max_total_bytes = 7; - let limited = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(!limited.complete && !limited.unchanged && limited.fingerprint.is_none()); - assert!(!limited.diagnostics.is_empty()); - query.limits = InventoryLimits::default(); - query.roots = vec![path("alias")]; - symlink(temp.path().join("two"), temp.path().join("two/loop")).unwrap(); - assert!( - !runtime - .filesystem() - .scan(query.clone()) + let status = { + let fs = filesystem(&root, true); + begin(&fs, "done", selection.clone()).await.unwrap(); + fs.transfer_abort(&context(), operation("done")) .await .unwrap() - .complete - ); - std::fs::remove_file(temp.path().join("two/loop")).unwrap(); - std::fs::remove_file(temp.path().join("alias")).unwrap(); - symlink(outside.path(), temp.path().join("alias")).unwrap(); - let escaped = runtime.filesystem().scan(query.clone()).await.unwrap(); - assert!(!escaped.complete); - assert_eq!( - escaped.diagnostics[0].error.code, - EnvironmentProtocolErrorCode::Forbidden - ); - std::fs::remove_file(temp.path().join("alias")).unwrap(); - symlink(temp.path().join("missing"), temp.path().join("alias")).unwrap(); - let dangling = runtime.filesystem().scan(query).await.unwrap(); - assert!(!dangling.complete && !dangling.unchanged); -} - -fn environment_daemon_scope_for_test(cwd: &std::path::Path) -> crate::filesystem::LocalFileSystem { - crate::filesystem::LocalFileSystem::new( - cwd.parent().unwrap().to_path_buf(), - cwd.to_path_buf(), - false, + }; + assert_eq!(status.phase, TransferPhase::Aborted); + // An exit before the background removal finished: the stage is still + // there and the receipt names it. + let stage = ".lightspeed-transfer-0123456789abcdef0123456789abcdef"; + std::fs::create_dir_all(root.join(stage).join("tree/replaced")).unwrap(); + let journal = root.join(".state/transfers"); + journal::write( + &journal, + &id("done"), + &Record { + params: BeginTransferParams { + operation_id: id("done"), + selection, + limits: limits(), + }, + status: status.clone(), + stage: Some(StageRecord::new(&[], stage)), + spool: None, + committing: false, + }, ) + .unwrap(); + + let fs = filesystem(&root, true); + assert!(!root.join(stage).exists(), "recovery removes the stage"); + let record = journal::read(&journal, &id("done")).unwrap().unwrap(); + assert!(record.stage.is_none()); + let answered = fs + .transfer_get_status(&context(), operation("done")) + .await + .unwrap(); + assert_eq!(answered.phase, TransferPhase::Aborted); } diff --git a/crates/environment-daemon/src/filesystem/walk.rs b/crates/environment-daemon/src/filesystem/walk.rs new file mode 100644 index 000000000..a56bf5871 --- /dev/null +++ b/crates/environment-daemon/src/filesystem/walk.rs @@ -0,0 +1,754 @@ +//! One traversal for `fs/searchText`, `fs/globFiles`, `fs/scan` and transfer +//! captures: the [`TreeFilter`] pattern grammar and a resumable walk beneath +//! the root descriptor. +//! +//! Patterns match paths relative to the walk's root, gitignore-style: `*` +//! and `?` within one segment, `**` across segments, `[...]` and `{a,b}`. A +//! pattern without `/` matches the entry's name at any depth, a `/` anywhere +//! but at the end anchors it to the root, and a trailing `/` matches +//! directories only. The backslash is an ordinary character, as in names. +//! +//! The walk visits children in byte order of their names, depth first, so +//! results are sorted by path. It always prunes VCS metadata directories and +//! the daemon state directory; by default it respects `.gitignore`, +//! `.ignore` and `.git/info/exclude` (in that ascending precedence, each +//! level overriding its ancestors) and skips names starting with `.`. + +use std::{ + collections::HashSet, + ffi::{OsStr, OsString}, + os::unix::ffi::OsStrExt, + path::PathBuf, + sync::Arc, +}; + +use environment_protocol::{ + data::fs::{FileKind, TreeFilter}, + error::{PermissionReason, PreconditionReason, ProtocolError}, + paths::TreePath, +}; +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use ignore::gitignore::{Gitignore, GitignoreBuilder}; + +use super::{ + confinement::{Confinement, Resolved}, + io_error, + os::{self, Directory, EntryStat}, +}; + +/// Directories every walk prunes, whatever the filter says. +const VCS_DIRECTORIES: [&str; 4] = [".git", ".hg", ".svn", ".jj"]; +/// Steps (pruned entries, listings) a walk takes without an event before it +/// yields, so its caller can check its deadline and cancellation. +const STEPS_PER_YIELD: u32 = 1024; +/// Largest ignore file read; longer files are cut. +const MAX_IGNORE_FILE_BYTES: u64 = 1024 * 1024; + +/// A compiled pattern list. +#[derive(Clone, Debug, Default)] +pub(crate) struct PatternSet { + any: Option, + directories: Option, + count: usize, +} + +impl PatternSet { + /// Compile `patterns`; `field` names the list in errors + /// (`filter.include`). + pub fn compile( + patterns: &[String], + ignore_case: bool, + field: &str, + ) -> Result { + let mut any = GlobSetBuilder::new(); + let mut directories = GlobSetBuilder::new(); + let (mut any_count, mut directory_count) = (0, 0); + for (index, pattern) in patterns.iter().enumerate() { + let field = format!("{field}[{index}]"); + let (glob, directory_only) = translate(pattern) + .ok_or_else(|| ProtocolError::invalid_field(&field, "empty pattern"))?; + let glob = GlobBuilder::new(&glob) + .literal_separator(true) + .backslash_escape(false) + .case_insensitive(ignore_case) + .build() + .map_err(|error| { + ProtocolError::invalid_field(&field, format!("invalid pattern: {error}")) + })?; + if directory_only { + directories.add(glob); + directory_count += 1; + } else { + any.add(glob); + any_count += 1; + } + } + let build = |builder: GlobSetBuilder, count: usize| -> Result<_, ProtocolError> { + if count == 0 { + return Ok(None); + } + builder + .build() + .map(Some) + .map_err(|error| ProtocolError::invalid_field(field, error.to_string())) + }; + Ok(Self { + any: build(any, any_count)?, + directories: build(directories, directory_count)?, + count: patterns.len(), + }) + } + + pub fn is_empty(&self) -> bool { + self.count == 0 + } + + pub fn matches(&self, path: &TreePath, is_dir: bool) -> bool { + let path = path.as_str(); + self.any.as_ref().is_some_and(|set| set.is_match(path)) + || (is_dir + && self + .directories + .as_ref() + .is_some_and(|set| set.is_match(path))) + } +} + +/// Gitignore-style anchoring into a globset pattern: `(glob, directory +/// only)`, or `None` for an empty pattern. +fn translate(pattern: &str) -> Option<(String, bool)> { + let (body, directory_only) = match pattern.strip_suffix('/') { + Some(body) if !body.is_empty() => (body, true), + _ => (pattern, false), + }; + if body.is_empty() || body == "/" { + return None; + } + let glob = if let Some(anchored) = body.strip_prefix('/') { + anchored.to_owned() + } else if body.contains('/') { + body.to_owned() + } else { + format!("**/{body}") + }; + Some((glob, directory_only)) +} + +/// A [`TreeFilter`] ready to walk. +#[derive(Clone, Debug)] +pub(crate) struct CompiledFilter { + pub include: PatternSet, + pub exclude: PatternSet, + pub include_hidden: bool, + pub include_ignored: bool, + pub max_depth: Option, +} + +impl CompiledFilter { + /// `prefix` is the wire field holding the filter (`filter`). + pub fn compile(filter: &TreeFilter, prefix: &str) -> Result { + Ok(Self { + include: PatternSet::compile( + &filter.include, + filter.ignore_case, + &format!("{prefix}.include"), + )?, + exclude: PatternSet::compile( + &filter.exclude, + filter.ignore_case, + &format!("{prefix}.exclude"), + )?, + include_hidden: filter.include_hidden, + include_ignored: filter.include_ignored, + max_depth: filter.max_depth, + }) + } + + /// Visit everything: no patterns, hidden and ignored entries included. + pub fn everything() -> Self { + Self { + include: PatternSet::default(), + exclude: PatternSet::default(), + include_hidden: true, + include_ignored: true, + max_depth: None, + } + } + + pub fn selects(&self, path: &TreePath, is_dir: bool) -> bool { + self.include.is_empty() || self.include.matches(path, is_dir) + } +} + +/// What a walk does with a symlink it meets below its root. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LinkMode { + /// Report the link itself as an entry of kind symlink. + Report, + /// Follow it when its target stays inside the filesystem root; report + /// it as skipped otherwise. + Follow, +} + +/// One step of a walk. +#[derive(Debug)] +pub(crate) enum Event { + Entry(Entry), + /// Nothing to report yet: the walk pruned or listed for a while. The + /// caller checks its bounds and steps again. + Yield, + Skip { + /// Absent when the name is not UTF-8. + path: Option, + reason: Skip, + /// The skipped entry matters to the filter: its path, taken as a + /// non-directory, matches `include` (always without includes), or + /// it is a directory the walk had to descend. Always true without a + /// path. + selected: bool, + }, +} + +/// Why a walk left an entry out. +#[derive(Debug)] +pub(crate) enum Skip { + NonUtf8Name, + /// The entry vanished, could not be read, or (a directory) could not be + /// listed. + Unreadable(ProtocolError), + /// A followed link left the root, addressed the state directory, looped + /// or closed a directory cycle. + LinkOutsideRoot(ProtocolError), + /// A followed link whose target does not exist. + DanglingLink, +} + +/// An entry the walk visited. +#[derive(Debug)] +pub(crate) struct Entry { + /// Relative to the walk's root, as walked. + pub path: TreePath, + pub depth: u32, + /// The directory holding the entry: for a followed link, its target. + pub parent: Arc, + pub name: OsString, + /// Canonical segments below the filesystem root. + pub canonical: Vec, + /// `lstat` of the entry, or of the link's target when it was followed. + pub stat: EntryStat, + /// Reached through a followed symlink. + pub via_link: bool, + /// Matches the filter's `include` (always true without includes). + pub selected: bool, +} + +struct Frame { + directory: Arc, + canonical: Vec, + path: TreePath, + depth: u32, + identity: (u64, u64), + /// Children left to visit, in reverse order; `None` until listed. + names: Option>, + /// Ignore levels pushed when this frame was listed. + ignore_levels: usize, +} + +struct IgnoreLevel { + matcher: Gitignore, +} + +/// A depth-first walk beneath a resolved directory. Resumable: callers step +/// it with [`Walker::next_event`] and may stop at any time. +pub(crate) struct Walker { + confinement: Arc, + filter: CompiledFilter, + links: LinkMode, + max_depth: Option, + /// Coordinates of ignore matching: the walk root's canonical segments. + base: PathBuf, + stack: Vec, + ignores: Vec, + identities: HashSet<(u64, u64)>, +} + +impl Walker { + /// Walk below `root`, which must be a resolved directory. `max_depth` + /// overrides the filter's. + pub fn new( + confinement: Arc, + root: &Resolved, + filter: CompiledFilter, + links: LinkMode, + max_depth: Option, + subject: &dyn std::fmt::Display, + ) -> Result { + let directory = root + .open_directory() + .map_err(|error| io_error(error, subject))?; + let stat = EntryStat::of_directory(&directory).map_err(|error| io_error(error, subject))?; + let base = segments_path(&root.segments); + let mut walker = Self { + confinement, + max_depth: max_depth.or(filter.max_depth), + filter, + links, + base, + stack: Vec::new(), + ignores: Vec::new(), + identities: HashSet::new(), + }; + if !walker.filter.include_ignored { + walker.load_ancestor_ignores(&root.segments); + } + walker.identities.insert((stat.dev, stat.ino)); + walker.stack.push(Frame { + directory: Arc::new(directory), + canonical: root.segments.clone(), + path: TreePath::root(), + depth: 0, + identity: (stat.dev, stat.ino), + names: None, + ignore_levels: 0, + }); + Ok(walker) + } + + /// The next entry or skip, [`Event::Yield`] after a stretch without + /// one, or `None` when the walk is done. + pub fn next_event(&mut self) -> Option { + for _ in 0..STEPS_PER_YIELD { + let frame = self.stack.last_mut()?; + if frame.names.is_none() { + if let Some(event) = self.list_top() { + return Some(event); + } + continue; + } + let frame = self.stack.last_mut().expect("frame listed above"); + let Some(name) = frame.names.as_mut().and_then(Vec::pop) else { + self.pop_frame(); + continue; + }; + if let Some(event) = self.visit(name) { + return Some(event); + } + } + Some(Event::Yield) + } + + /// List the top frame; an unlistable directory is skipped and popped. + fn list_top(&mut self) -> Option { + let frame = self.stack.last_mut().expect("caller checked"); + if self.max_depth.is_some_and(|max| frame.depth >= max) { + frame.names = Some(Vec::new()); + return None; + } + match frame.directory.names() { + Ok(mut names) => { + names.sort_by(|left, right| right.as_bytes().cmp(left.as_bytes())); + frame.names = Some(names); + if !self.filter.include_ignored { + let directory = frame.directory.clone(); + let at = self.base.join(segments_path_tree(&frame.path)); + let pushed = self.load_ignores(&directory, at); + self.stack.last_mut().expect("frame").ignore_levels = pushed; + } + None + } + Err(error) => { + let path = frame.path.clone(); + let error = io_error(error, &path); + self.pop_frame(); + Some(Event::Skip { + path: (!path.is_root()).then_some(path), + reason: Skip::Unreadable(error), + selected: true, + }) + } + } + } + + fn pop_frame(&mut self) { + if let Some(frame) = self.stack.pop() { + self.ignores + .truncate(self.ignores.len().saturating_sub(frame.ignore_levels)); + self.identities.remove(&frame.identity); + } + } + + fn visit(&mut self, name: OsString) -> Option { + let frame = self.stack.last().expect("visiting a listed frame"); + let depth = frame.depth + 1; + let Some(utf8) = os::utf8_name(&name) else { + return Some(Event::Skip { + path: None, + reason: Skip::NonUtf8Name, + selected: true, + }); + }; + if VCS_DIRECTORIES.contains(&utf8) { + return None; + } + let mut canonical = frame.canonical.clone(); + canonical.push(name.clone()); + if !self.filter.include_hidden && utf8.starts_with('.') { + return None; + } + let path = frame + .path + .join(utf8) + .expect("a UTF-8 name without '/' joins"); + let parent = frame.directory.clone(); + let parent_canonical = frame.canonical.clone(); + let stat = match parent.stat(&name) { + Ok(stat) => stat, + Err(error) => { + let error = io_error(error, &path); + return self.skip_entry(path, Skip::Unreadable(error)); + } + }; + if self.confinement.is_state(&stat) { + return None; + } + let mut entry = Entry { + path, + depth, + parent, + name, + canonical, + stat, + via_link: false, + selected: false, + }; + if stat.kind == FileKind::Symlink + && self.links == LinkMode::Follow + && let Err(skip) = self.follow(&parent_canonical, &mut entry) + { + return self.skip_entry(entry.path, skip); + } + let is_dir = entry.stat.is_dir(); + if self.ignored(&entry.path, is_dir) || self.filter.exclude.matches(&entry.path, is_dir) { + return None; + } + entry.selected = self.filter.selects(&entry.path, is_dir); + if is_dir { + if entry.via_link && self.identities.contains(&(entry.stat.dev, entry.stat.ino)) { + return Some(Event::Skip { + reason: Skip::LinkOutsideRoot(ProtocolError::failed_precondition( + PreconditionReason::SymlinkLoop, + format!("{}: the link closes a directory cycle", entry.path), + )), + path: Some(entry.path), + selected: true, + }); + } + match entry.parent.child(&entry.name) { + Ok(directory) => { + let identity = (entry.stat.dev, entry.stat.ino); + self.identities.insert(identity); + self.stack.push(Frame { + directory: Arc::new(directory), + canonical: entry.canonical.clone(), + path: entry.path.clone(), + depth, + identity, + names: None, + ignore_levels: 0, + }); + } + Err(error) => { + return Some(Event::Skip { + reason: Skip::Unreadable(io_error(error, &entry.path)), + path: Some(entry.path), + selected: true, + }); + } + } + } + Some(Event::Entry(entry)) + } + + /// A skip of an entry whose kind is unknown (it vanished, or a link + /// that could not be followed), taken as a non-directory: pruned like + /// one when ignored or excluded, selected like one otherwise. + fn skip_entry(&self, path: TreePath, reason: Skip) -> Option { + if self.ignored(&path, false) || self.filter.exclude.matches(&path, false) { + return None; + } + Some(Event::Skip { + selected: self.filter.selects(&path, false), + path: Some(path), + reason, + }) + } + + /// Replace a link entry by its in-root target. + fn follow(&self, parent: &[OsString], entry: &mut Entry) -> Result<(), Skip> { + let resolved = self + .confinement + .lookup_link(parent, &entry.name, &entry.path) + .map_err(|error| match &error.data { + environment_protocol::error::ErrorData::PermissionDenied { + reason: Some(PermissionReason::OutsideRoot | PermissionReason::DaemonState), + } + | environment_protocol::error::ErrorData::FailedPrecondition { + reason: PreconditionReason::SymlinkLoop, + } => Skip::LinkOutsideRoot(error), + environment_protocol::error::ErrorData::NotFound { .. } => Skip::DanglingLink, + _ => Skip::Unreadable(error), + })?; + let stat = resolved.stat.expect("a followed lookup reports its target"); + entry.parent = Arc::new(resolved.parent); + entry.name = resolved.name; + entry.canonical = resolved.segments; + entry.stat = stat; + entry.via_link = true; + Ok(()) + } + + fn ignored(&self, path: &TreePath, is_dir: bool) -> bool { + if self.filter.include_ignored { + return false; + } + let candidate = self.base.join(segments_path_tree(path)); + for level in self.ignores.iter().rev() { + let matched = level.matcher.matched(&candidate, is_dir); + if matched.is_ignore() { + return true; + } + if matched.is_whitelist() { + return false; + } + } + false + } + + fn load_ancestor_ignores(&mut self, root: &[OsString]) { + for depth in 0..root.len() { + let segments = &root[..depth]; + let Ok(resolved) = self.confinement.reopen(segments, &"ignore file ancestor") else { + continue; + }; + let Ok(directory) = resolved.open_directory() else { + continue; + }; + self.load_ignores(&directory, segments_path(segments)); + } + } + + /// Push the ignore level of one directory; returns how many levels were + /// pushed (0 or 1). + fn load_ignores(&mut self, directory: &Directory, at: PathBuf) -> usize { + let mut builder = GitignoreBuilder::new(&at); + let mut lines = 0usize; + let mut add = |content: Vec| { + for line in String::from_utf8_lossy(&content).lines() { + if builder.add_line(None, line).is_ok() { + lines += 1; + } + } + }; + if let Ok(git) = directory.child(OsStr::new(".git")) + && let Ok(info) = git.child(OsStr::new("info")) + && let Some(content) = read_small(&info, "exclude") + { + add(content); + } + for name in [".gitignore", ".ignore"] { + if let Some(content) = read_small(directory, name) { + add(content); + } + } + if lines == 0 { + return 0; + } + match builder.build() { + Ok(matcher) => { + self.ignores.push(IgnoreLevel { matcher }); + 1 + } + Err(_) => 0, + } + } +} + +fn read_small(directory: &Directory, name: &str) -> Option> { + let mut file = directory.open_read(OsStr::new(name)).ok()?; + if !EntryStat::of_file(&file).ok()?.is_file() { + return None; + } + os::read_up_to(&mut file, MAX_IGNORE_FILE_BYTES).ok() +} + +/// `/` joined with segments: the coordinate space of ignore matching. +fn segments_path(segments: &[OsString]) -> PathBuf { + let mut path = PathBuf::from("/"); + path.extend(segments); + path +} + +fn segments_path_tree(path: &TreePath) -> PathBuf { + path.segments().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::filesystem::confinement::Lookup; + use environment_protocol::paths::EnvironmentPath; + + fn set(patterns: &[&str]) -> PatternSet { + PatternSet::compile( + &patterns.iter().map(|p| (*p).to_owned()).collect::>(), + false, + "filter.include", + ) + .unwrap() + } + + fn tree(path: &str) -> TreePath { + TreePath::new(path).unwrap() + } + + #[test] + fn patterns_follow_gitignore_anchoring() { + let star = set(&["src/*.rs"]); + assert!(star.matches(&tree("src/lib.rs"), false)); + assert!(!star.matches(&tree("src/deep/nested.rs"), false)); + + let name = set(&["*.md"]); + assert!(name.matches(&tree("readme.md"), false)); + assert!(name.matches(&tree("docs/deep/guide.md"), false)); + + let anchored = set(&["/build"]); + assert!(anchored.matches(&tree("build"), true)); + assert!(!anchored.matches(&tree("sub/build"), true)); + + let spanning = set(&["**/SKILL.md"]); + assert!(spanning.matches(&tree("SKILL.md"), false)); + assert!(spanning.matches(&tree("a/b/SKILL.md"), false)); + + let directories = set(&["target/"]); + assert!(directories.matches(&tree("crates/target"), true)); + assert!(!directories.matches(&tree("crates/target"), false)); + + let alternation = set(&["*.{rs,toml}"]); + assert!(alternation.matches(&tree("Cargo.toml"), false)); + + assert!(PatternSet::compile(&["".into()], false, "filter.include").is_err()); + assert!(PatternSet::compile(&["[".into()], false, "filter.include").is_err()); + } + + fn walk(root: &std::path::Path, filter: TreeFilter, links: LinkMode) -> Vec { + let confinement = Arc::new(Confinement::new(root.into(), root.join(".state")).unwrap()); + let resolved = confinement + .lookup( + &EnvironmentPath::new(root.to_str().unwrap()).unwrap(), + Lookup::FOLLOW, + ) + .unwrap(); + let mut walker = Walker::new( + confinement, + &resolved, + CompiledFilter::compile(&filter, "filter").unwrap(), + links, + None, + &"root", + ) + .unwrap(); + let mut seen = Vec::new(); + while let Some(event) = walker.next_event() { + match event { + Event::Entry(entry) if entry.selected => seen.push(entry.path.to_string()), + Event::Entry(_) | Event::Yield => {} + Event::Skip { path, .. } => seen.push(format!( + "skip:{}", + path.map(|path| path.to_string()).unwrap_or_default() + )), + } + } + seen + } + + #[test] + fn walks_prune_vcs_state_hidden_and_ignored_entries_by_default() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + for directory in [ + ".git/objects", + "src", + "node_modules/pkg", + ".hidden", + ".state", + ] { + std::fs::create_dir_all(root.join(directory)).unwrap(); + } + std::fs::write(root.join(".git/objects/pack"), "pack").unwrap(); + std::fs::write(root.join(".gitignore"), "node_modules/\n*.log\n!keep.log\n").unwrap(); + std::fs::write(root.join("src/lib.rs"), "lib").unwrap(); + std::fs::write(root.join("src/debug.log"), "log").unwrap(); + std::fs::write(root.join("src/keep.log"), "log").unwrap(); + std::fs::write(root.join("node_modules/pkg/index.js"), "js").unwrap(); + std::fs::write(root.join(".hidden/file"), "hidden").unwrap(); + std::fs::write(root.join(".state/daemon-key"), "key").unwrap(); + + assert_eq!( + walk(&root, TreeFilter::default(), LinkMode::Report), + ["src", "src/keep.log", "src/lib.rs"] + ); + let everything = TreeFilter { + include_hidden: true, + include_ignored: true, + ..TreeFilter::default() + }; + // VCS metadata and the state directory stay pruned. + assert_eq!( + walk(&root, everything, LinkMode::Report), + [ + ".gitignore", + ".hidden", + ".hidden/file", + "node_modules", + "node_modules/pkg", + "node_modules/pkg/index.js", + "src", + "src/debug.log", + "src/keep.log", + "src/lib.rs", + ] + ); + } + + #[test] + fn followed_links_stay_inside_the_root_and_cycles_are_skipped() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let base = temp.path().canonicalize().unwrap(); + let root = base.join("root"); + std::fs::create_dir_all(root.join("real")).unwrap(); + std::fs::create_dir_all(base.join("outside")).unwrap(); + std::fs::write(root.join("real/file"), "x").unwrap(); + symlink("real", root.join("alias")).unwrap(); + symlink("..", root.join("real/up")).unwrap(); + symlink(base.join("outside"), root.join("escape")).unwrap(); + + let filter = TreeFilter { + include_hidden: true, + ..TreeFilter::default() + }; + assert_eq!( + walk(&root, filter, LinkMode::Follow), + [ + "alias", + "alias/file", + "skip:alias/up", + "skip:escape", + "real", + "real/file", + "skip:real/up", + ] + ); + } +} diff --git a/crates/environment-daemon/src/identity.rs b/crates/environment-daemon/src/identity.rs index bfe1ed4e5..8fa9d2d1c 100644 --- a/crates/environment-daemon/src/identity.rs +++ b/crates/environment-daemon/src/identity.rs @@ -10,7 +10,13 @@ use std::path::{Path, PathBuf}; use anyhow::{Context as _, Result, bail}; use ed25519_dalek::{Signer as _, SigningKey, VerifyingKey}; -use environment_protocol::registration::{decode_hex, encode_hex, signed_registration_message}; +use environment_protocol::{ + bytes::{decode_hex_array, encode_hex}, + registration::{ + ChallengeNonce, DaemonPublicKey, Ed25519Signature, signed_registration_message, + }, + routes::CanonicalOrigin, +}; /// Name of the key file inside the daemon state directory. pub const DAEMON_KEY_FILE: &str = "daemon-key"; @@ -24,7 +30,7 @@ pub struct DaemonIdentity { impl std::fmt::Debug for DaemonIdentity { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DaemonIdentity") - .field("public_key", &self.public_key_hex()) + .field("public_key", &self.public_key()) .field("path", &self.path) .finish() } @@ -32,60 +38,73 @@ impl std::fmt::Debug for DaemonIdentity { impl DaemonIdentity { /// Load the key from `/daemon-key`, or generate one there. - /// The directory is created `0700` and the file `0600`; generation is - /// atomic so a crash cannot leave a half-written key behind. + /// The file is created `0600` and installed without ever replacing an + /// existing key: when two first starts race, the key installed first + /// wins and both processes load it. pub fn load_or_create(state_dir: &Path) -> Result { let path = state_dir.join(DAEMON_KEY_FILE); - if path.exists() { - let contents = std::fs::read_to_string(&path) - .with_context(|| format!("read daemon key {}", path.display()))?; - let seed = decode_hex(contents.trim()) - .filter(|seed| seed.len() == 32) - .with_context(|| { - format!("daemon key {} is not a 32-byte hex seed", path.display()) - })?; - let seed: [u8; 32] = seed.try_into().expect("length checked"); - return Ok(Self { - signing_key: SigningKey::from_bytes(&seed), - path, - }); + if let Some(identity) = Self::load(&path)? { + return Ok(identity); } - std::fs::create_dir_all(state_dir) - .with_context(|| format!("create state dir {}", state_dir.display()))?; - restrict_dir(state_dir)?; let mut seed = [0u8; 32]; rand::TryRng::try_fill_bytes(&mut rand::rngs::SysRng, &mut seed) .context("generate daemon identity seed")?; let signing_key = SigningKey::from_bytes(&seed); - let temp = state_dir.join(format!(".{DAEMON_KEY_FILE}.{}", std::process::id())); + let mut suffix = [0u8; 8]; + rand::TryRng::try_fill_bytes(&mut rand::rngs::SysRng, &mut suffix) + .context("generate temporary key name")?; + let temp = state_dir.join(format!(".{DAEMON_KEY_FILE}.{}", encode_hex(&suffix))); write_private(&temp, &encode_hex(&signing_key.to_bytes()))?; - if let Err(error) = std::fs::rename(&temp, &path) { - let _ = std::fs::remove_file(&temp); - if path.exists() { - // Lost a race with a concurrent start of the same daemon: - // the other key wins so both processes share one identity. - return Self::load_or_create(state_dir); + // A hard link fails when the destination exists, unlike rename, + // which would silently replace a key a concurrent start installed. + let installed = std::fs::hard_link(&temp, &path); + let _ = std::fs::remove_file(&temp); + match installed { + Ok(()) => { + sync_dir(state_dir); + Ok(Self { signing_key, path }) } - bail!("install daemon key {}: {error}", path.display()); + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Self::load(&path)? + .with_context(|| format!("daemon key {} vanished during install", path.display())), + Err(error) => bail!("install daemon key {}: {error}", path.display()), } - Ok(Self { signing_key, path }) } - /// Lowercase hex of the raw 32-byte public key. - pub fn public_key_hex(&self) -> String { - encode_hex(self.signing_key.verifying_key().as_bytes()) + fn load(path: &Path) -> Result> { + let contents = match std::fs::read_to_string(path) { + Ok(contents) => contents, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| format!("read daemon key {}", path.display())); + } + }; + let seed = decode_hex_array::<32>(contents.trim()) + .with_context(|| format!("daemon key {} is not a 32-byte hex seed", path.display()))?; + Ok(Some(Self { + signing_key: SigningKey::from_bytes(&seed), + path: path.to_path_buf(), + })) + } + + /// `daemonPublicKey`. + pub fn public_key(&self) -> DaemonPublicKey { + DaemonPublicKey::from_bytes(self.signing_key.verifying_key().to_bytes()) } pub fn verifying_key(&self) -> VerifyingKey { self.signing_key.verifying_key() } - /// Hex signature over the domain-separated challenge. - pub fn sign_challenge(&self, nonce: &[u8]) -> String { - encode_hex( - &self - .signing_key - .sign(&signed_registration_message(nonce)) + /// `signature` over the challenge nonce bound to the canonical origin of + /// the gateway URL this daemon dialed. + pub fn sign_challenge( + &self, + nonce: &ChallengeNonce, + origin: &CanonicalOrigin, + ) -> Ed25519Signature { + Ed25519Signature::from_bytes( + self.signing_key + .sign(&signed_registration_message(nonce, origin)) .to_bytes(), ) } @@ -113,30 +132,27 @@ fn write_private(path: &Path, contents: &str) -> Result<()> { Ok(()) } -fn restrict_dir(path: &Path) -> Result<()> { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) - .with_context(|| format!("restrict state dir {}", path.display()))?; +/// Make the new directory entry durable; best effort. +fn sync_dir(path: &Path) { + if let Ok(directory) = std::fs::File::open(path) { + let _ = directory.sync_all(); } - Ok(()) } #[cfg(test)] mod tests { use ed25519_dalek::Verifier as _; + use environment_protocol::routes::canonical_origin; use super::*; #[test] fn identity_is_generated_once_and_reloaded_from_the_state_dir() { let temp = tempfile::tempdir().expect("tempdir"); - let state = temp.path().join("state"); - let first = DaemonIdentity::load_or_create(&state).expect("create"); - let second = DaemonIdentity::load_or_create(&state).expect("reload"); - assert_eq!(first.public_key_hex(), second.public_key_hex()); - assert_eq!(first.public_key_hex().len(), 64); + let state = temp.path(); + let first = DaemonIdentity::load_or_create(state).expect("create"); + let second = DaemonIdentity::load_or_create(state).expect("reload"); + assert_eq!(first.public_key(), second.public_key()); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt as _; @@ -146,37 +162,66 @@ mod tests { .mode() & 0o777; assert_eq!(mode, 0o600); - let dir_mode = std::fs::metadata(&state) - .expect("meta") - .permissions() - .mode() - & 0o777; - assert_eq!(dir_mode, 0o700); } + assert_eq!( + std::fs::read_dir(state).unwrap().count(), + 1, + "no temporary file is left behind" + ); std::fs::remove_file(first.path()).expect("reset identity"); - let reset = DaemonIdentity::load_or_create(&state).expect("regenerate"); - assert_ne!(reset.public_key_hex(), first.public_key_hex()); + let reset = DaemonIdentity::load_or_create(state).expect("regenerate"); + assert_ne!(reset.public_key(), first.public_key()); + } + + #[test] + fn concurrent_first_starts_share_the_key_installed_first() { + let temp = tempfile::tempdir().expect("tempdir"); + let state = temp.path().to_path_buf(); + let handles: Vec<_> = (0..8) + .map(|_| { + let state = state.clone(); + std::thread::spawn(move || DaemonIdentity::load_or_create(&state).unwrap()) + }) + .collect(); + let keys: std::collections::BTreeSet = handles + .into_iter() + .map(|handle| handle.join().unwrap().public_key().to_hex()) + .collect(); + assert_eq!(keys.len(), 1, "every racing start holds the installed key"); + let on_disk = DaemonIdentity::load_or_create(&state).unwrap(); + assert_eq!( + keys.into_iter().next().unwrap(), + on_disk.public_key().to_hex() + ); } #[test] - fn challenge_signatures_verify_against_the_public_key_and_domain() { + fn challenge_signatures_bind_the_nonce_and_the_dialed_origin() { let temp = tempfile::tempdir().expect("tempdir"); let identity = DaemonIdentity::load_or_create(temp.path()).expect("create"); - let nonce = [7u8; 32]; - let signature = decode_hex(&identity.sign_challenge(&nonce)).expect("hex"); - let signature = ed25519_dalek::Signature::from_slice(&signature).expect("signature"); - identity - .verifying_key() - .verify(&signed_registration_message(&nonce), &signature) + let nonce = ChallengeNonce::from_bytes([7u8; 32]); + let origin = canonical_origin("wss://gateway.example/environment-gateway/connect").unwrap(); + let signature = identity.sign_challenge(&nonce, &origin); + let signature = ed25519_dalek::Signature::from_bytes(signature.as_bytes()); + let verifying = identity.verifying_key(); + verifying + .verify_strict(&signed_registration_message(&nonce, &origin), &signature) .expect("verifies"); + assert!(verifying.verify(nonce.as_bytes(), &signature).is_err()); + let elsewhere = canonical_origin("wss://other.example").unwrap(); assert!( - identity.verifying_key().verify(&nonce, &signature).is_err(), - "signature must not verify without the domain separator" + verifying + .verify(&signed_registration_message(&nonce, &elsewhere), &signature) + .is_err(), + "a signature for one gateway does not verify at another" ); + let other_nonce = ChallengeNonce::from_bytes([8u8; 32]); assert!( - identity - .verifying_key() - .verify(&signed_registration_message(&[8u8; 32]), &signature) + verifying + .verify( + &signed_registration_message(&other_nonce, &origin), + &signature + ) .is_err() ); } @@ -184,7 +229,9 @@ mod tests { #[test] fn corrupt_key_files_are_rejected_rather_than_replaced() { let temp = tempfile::tempdir().expect("tempdir"); - std::fs::write(temp.path().join(DAEMON_KEY_FILE), "not-hex").expect("write"); + let path = temp.path().join(DAEMON_KEY_FILE); + std::fs::write(&path, "not-hex").expect("write"); assert!(DaemonIdentity::load_or_create(temp.path()).is_err()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "not-hex"); } } diff --git a/crates/environment-daemon/src/jobs.rs b/crates/environment-daemon/src/jobs.rs index 19fdd3553..b34b1d28e 100644 --- a/crates/environment-daemon/src/jobs.rs +++ b/crates/environment-daemon/src/jobs.rs @@ -1,2051 +1,2082 @@ +//! `job/*`: durable batch commands with dependencies and queues, on the +//! shared execution primitive. +//! +//! A job is identified by its `jobId` together with the `requestId` of the +//! `job/start` that created it and a hash of its spec: an identical retry +//! replays the job, any other reuse of the id is `jobIdInUse`. A job runs +//! once its dependencies satisfy its policy and its queue key is free. When +//! its root exits the daemon sweeps its process group (`SIGTERM`, the +//! cancel grace, `SIGKILL`) and reports what it swept. +//! +//! Records and output persist under `/jobs/` through the job store, +//! off the runtime thread; output keeps a head and a tail with the middle +//! as an overflow gap. Startup loads summaries only, quarantines unreadable +//! records instead of failing, marks unfinished jobs `interrupted` and kills +//! the groups an earlier instance left running in this boot. + use std::{ - collections::{BTreeMap, BTreeSet}, - path::{Component, PathBuf}, - process::Stdio, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::{Duration, SystemTime, UNIX_EPOCH}, + collections::{BTreeMap, BTreeSet, VecDeque}, + path::{Path, PathBuf}, + sync::{Arc, Mutex, MutexGuard}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use environment_protocol::{ - data::jobs::{ - CancelJobsParams, CancelJobsResponse, JobCancelScope, JobDependencyPolicy, JobOutputChunk, - JobOutputStream, JobReadResult, JobStartSpec, JobStatus, JobSummary, ListJobsParams, - ListJobsResponse, ReadJobsParams, ReadJobsResponse, StartJobsParams, StartJobsResponse, + bytes::encode_hex, + data::{ + exec::{ + CommandSpec, ExecInput, ExitStatus, LeftoverDisposition, LeftoverReport, OutputPage, + OutputStream, ReturnOn, StoppedBy, + }, + handshake::JobLimits, + job::{ + CancelJobsParams, CancelJobsResponse, JobCancelScope, JobDependency, + JobDependencyPolicy, JobReadEntry, JobReadRequest, JobStatus, JobSummary, + JobSummaryEntry, ReadJobsParams, ReadJobsResponse, StartJobsParams, StartJobsResponse, + }, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::{ByteChunk, EnvironmentPath, JobId, SecretString}, + error::{PreconditionReason, ProtocolError}, + ids::{DaemonInstanceId, JobId, JobName, JobRequestId, QueueKey}, + limits::MAX_PAGE_CHUNKS, + paths::EnvironmentPath, }; use serde::{Deserialize, Serialize}; -use tokio::{ - io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, - process::Command, - sync::{Mutex, Notify}, - task::JoinHandle, -}; +use sha2::{Digest as _, Sha256}; +use tokio::sync::watch; use crate::{ + dispatch::RequestContext, + exec::{ + AfterExit, ExecOptions, Execution, OutputSink, PreparedCommand, + job_store::{JobStore, load_output}, + log::OutputLog, + outcome::describe_exit, + spawn_record::{SpawnRecord, write_json_atomic}, + validate_command, + }, + filesystem::Confinement, process_group, - redaction::{redact_bytes, redactions_for_secret_env}, }; +/// Version of the persisted job record; a record of another version is +/// quarantined. +const JOB_RECORD_FORMAT: u32 = 1; + +/// Terminal jobs whose output stays in memory after completion or a load. +const MAX_CACHED_OUTPUTS: usize = 16; + +/// Everything a [`JobManager`] needs from the daemon. +pub struct JobConfig { + /// Canonical default `cwd`. + pub cwd: PathBuf, + /// Resolves every `cwd` when the job starts running. + pub confinement: Arc, + /// Records and output live in `/jobs`. + pub state_dir: PathBuf, + /// Daemon variables no child inherits. + pub scrubbed_env: Vec, + pub instance_id: DaemonInstanceId, + pub limits: JobLimits, + /// `limits.maxWaitMs`, the `waitMs` ceiling. + pub max_wait_ms: u64, +} + +/// The daemon's jobs. Cheap to clone. #[derive(Clone)] pub struct JobManager { - cwd: PathBuf, - fs_root: PathBuf, - jobs_root: PathBuf, - state: Arc>, - notify: Arc, - /// Daemon configuration variables no job may inherit. - scrubbed_env: Arc>, + inner: Arc, +} + +struct JobsInner { + config: JobConfig, + store: JobStore, + state: Mutex, + /// Bumped on every status change and every output append. + changed: watch::Sender, } #[derive(Default)] -struct JobManagerState { - accepting: bool, - jobs: BTreeMap, - running: BTreeMap>, - secret_envs: BTreeMap>, - next_accept_seq: u64, +struct JobsState { + jobs: BTreeMap, + next_seq: u64, + /// Terminal jobs holding their output in memory, oldest first. + cached: VecDeque, } -type JobKey = (String, JobId); +struct JobEntry { + record: StoredJob, + /// The command of a queued job, secrets included. Memory only. + pending: Option, + run: Option, + /// The output of a terminal job, while cached. + output: Option>, +} -struct RunningJob { - cancel: Notify, - interrupted: AtomicBool, +struct Run { + execution: Execution, + started: Instant, } -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -struct JobRecord { - namespace: String, +/// A running job's output: retained head and tail in memory, every byte +/// forwarded to the store. +struct JobOutput { job_id: JobId, - name: Option, - argv: Vec, - cwd: Option, - env: BTreeMap, - #[serde(default)] - secret_env_names: Vec, - stdin: Option, - timeout_ms: Option, - dependency_policy: JobDependencyPolicy, - dependencies: Vec, - queue_key: Option, + log: OutputLog, + cap: u64, + store: JobStore, + changed: watch::Sender, +} + +impl OutputSink for JobOutput { + fn append(&mut self, stream: OutputStream, data: &[u8]) { + self.store + .append(&self.job_id, self.log.end(), stream, data); + self.log.append(stream, data); + self.log.enforce_cap(self.cap, 0); + self.changed + .send_modify(|version| *version = version.wrapping_add(1)); + } +} + +/// The persisted record of one job. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct StoredJob { + format: u32, + job_id: JobId, + request_id: JobRequestId, + #[serde(default, skip_serializing_if = "Option::is_none")] + name: Option, spec_hash: String, - #[serde(default)] - accept_seq: u64, + seq: u64, status: JobStatus, - created_at_ms: u64, - queued_at_ms: Option, - started_at_ms: Option, - finished_at_ms: Option, - exit_code: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + depends_on: Vec, + dependency_policy: JobDependencyPolicy, + #[serde(default, skip_serializing_if = "Option::is_none")] + queue_key: Option, + created_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + queued_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + started_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + finished_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cancel_requested_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + exit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + stopped_by: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + leftovers: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + error: Option, + /// The running job's process group, for restart reconciliation. + #[serde(default, skip_serializing_if = "Option::is_none")] + spawn: Option, + /// Output bytes produced, final once the job is terminal. #[serde(default)] - orphaned_descendants: bool, - failure: Option, - output_chunks: Vec, - output_next_seq: u64, + output_end_bytes: u64, } -#[derive(Clone)] -struct ResolvedJob { - spec: JobStartSpec, - dependencies: Vec, - secret_env: BTreeMap, - spec_hash: String, -} +impl StoredJob { + fn summary(&self) -> JobSummary { + JobSummary { + job_id: self.job_id.clone(), + request_id: self.request_id.clone(), + name: self.name.clone(), + status: self.status, + depends_on: self.depends_on.clone(), + queue_key: self.queue_key.clone(), + created_at_ms: self.created_at_ms, + queued_at_ms: self.queued_at_ms, + started_at_ms: self.started_at_ms, + finished_at_ms: self.finished_at_ms, + cancel_requested_at_ms: self.cancel_requested_at_ms, + duration_ms: self.duration_ms, + exit: self.exit.clone(), + stopped_by: self.stopped_by, + leftovers: self.leftovers.clone(), + error: self.error.clone(), + } + } -#[derive(Clone, Copy)] -enum FinishKind { - Exit, - Cancelled, - Interrupted, - TimedOut, + fn encode(&self) -> Vec { + serde_json::to_vec(self).expect("job records encode") + } } -struct SpawnOutcome { - finish: FinishKind, - exit_code: Option, - orphaned_descendants: bool, +/// A validated `job/start` entry with its dependencies resolved. +struct ResolvedJob { + job_id: JobId, + name: Option, + command: CommandSpec, + depends_on: Vec, + dependency_policy: JobDependencyPolicy, + queue_key: Option, + spec_hash: String, } impl JobManager { - pub fn new(cwd: PathBuf, fs_root: PathBuf, state_dir: PathBuf) -> anyhow::Result { - let cwd = normalize_path(cwd); - let fs_root = normalize_path(fs_root); - let jobs_root = normalize_path(state_dir).join("jobs"); - std::fs::create_dir_all(&jobs_root)?; - let mut state = JobManagerState { - accepting: true, - ..JobManagerState::default() - }; + /// Loads job summaries and reconciles what an earlier instance left. + /// Never fails on a record: unreadable ones are quarantined. + pub fn new(config: JobConfig) -> anyhow::Result { + let jobs_dir = config.state_dir.join("jobs"); + std::fs::create_dir_all(&jobs_dir)?; + let quarantine_dir = config.state_dir.join("jobs-quarantine"); + let boot = process_group::boot_id(); let now = now_ms(); - - for entry in std::fs::read_dir(&jobs_root)? { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|extension| extension.to_str()) != Some("json") { - continue; - } - let bytes = std::fs::read(&path)?; - let mut record: JobRecord = serde_json::from_slice(&bytes)?; + let mut state = JobsState::default(); + for entry in std::fs::read_dir(&jobs_dir)? { + let path = entry?.path(); + let mut record = match load_record(&path) { + Ok(record) => record, + Err(reason) => { + quarantine(&path, &quarantine_dir, &reason); + continue; + } + }; if !record.status.is_terminal() { + if record.status == JobStatus::Running + && let Some(spawn) = &record.spawn + && spawn.same_boot(boot.as_deref()) + { + spawn.group().signal(process_group::GroupSignal::Kill); + } + record.error = Some(ProtocolError::aborted( + None, + format!( + "the daemon stopped while the job was {}", + status_name(record.status) + ), + )); record.status = JobStatus::Interrupted; record.finished_at_ms = Some(now); - record.failure = - Some("lightspeed-envd restarted before this job could be recovered".to_owned()); - persist_record_at(&jobs_root, &record)?; + // The last record predates the output the job produced. + let cap = config.limits.max_retained_output_bytes; + if let Ok(log) = load_output(&path, record.output_end_bytes, cap) { + record.output_end_bytes = log.end(); + } + if let Err(error) = write_json_atomic(&path.join("record.json"), &record) { + eprintln!( + "lightspeed-envd: could not persist interrupted job {}: {error}", + record.job_id + ); + } } - state.next_accept_seq = state - .next_accept_seq - .max(record.accept_seq.saturating_add(1)); - state - .jobs - .insert(job_key(&record.namespace, &record.job_id), record); + state.next_seq = state.next_seq.max(record.seq.saturating_add(1)); + state.jobs.insert( + record.job_id.clone(), + JobEntry { + record, + pending: None, + run: None, + output: None, + }, + ); } - + let cap = config.limits.max_retained_output_bytes; + let store = JobStore::start(jobs_dir, cap / 2, cap - cap / 2)?; Ok(Self { - cwd, - fs_root, - jobs_root, - state: Arc::new(Mutex::new(state)), - notify: Arc::new(Notify::new()), - scrubbed_env: Arc::new(Vec::new()), + inner: Arc::new(JobsInner { + config, + store, + state: Mutex::new(state), + changed: watch::Sender::new(0), + }), }) } - pub fn with_scrubbed_env(mut self, names: Vec) -> Self { - self.scrubbed_env = Arc::new(names); - self - } - - /// Jobs currently executing (running or cancelling). + /// Jobs that are `running`. pub async fn running_count(&self) -> u32 { - u32::try_from(self.state.lock().await.running.len()).unwrap_or(u32::MAX) + self.maintain(); + let running = self + .state() + .jobs + .values() + .filter(|entry| entry.record.status == JobStatus::Running) + .count(); + u32::try_from(running).unwrap_or(u32::MAX) } - pub async fn start_jobs( + pub async fn start( &self, + _ctx: &RequestContext, params: StartJobsParams, - ) -> Result { - let now = now_ms(); - let resolved = { - let state = self.state.lock().await; - require_accepting(&state)?; - validate_and_resolve_start(&state, ¶ms)? - }; - let mut accepted = Vec::new(); - { - let mut state = self.state.lock().await; - require_accepting(&state)?; - for job in &resolved { - let key = job_key(¶ms.namespace, &job.spec.job_id); - if state.jobs.contains_key(&key) { - state - .secret_envs - .entry(key.clone()) - .or_insert_with(|| job.secret_env.clone()); - accepted.push(key); - continue; + ) -> Result { + let resolved = self.resolve_start(¶ms)?; + self.maintain(); + let records = { + let mut state = self.state(); + if let Some(replay) = replay_or_conflict(&state, ¶ms.request_id, &resolved)? { + return Ok(StartJobsResponse { jobs: replay }); + } + for (index, job) in resolved.iter().enumerate() { + for dependency in &job.depends_on { + let known = state.jobs.contains_key(dependency) + || resolved.iter().any(|other| &other.job_id == dependency); + if !known { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].dependsOn"), + format!("unknown job id {dependency}"), + )); + } } - let accept_seq = state.next_accept_seq; - state.next_accept_seq = state.next_accept_seq.saturating_add(1); - - let record = JobRecord { - namespace: params.namespace.clone(), - job_id: job.spec.job_id.clone(), - name: job.spec.name.clone(), - argv: job.spec.argv.clone(), - cwd: job.spec.cwd.clone(), - env: job.spec.env.clone(), - secret_env_names: job.secret_env.keys().cloned().collect(), - stdin: job.spec.stdin.clone(), - timeout_ms: job.spec.timeout_ms, - dependency_policy: job.spec.dependency_policy, - dependencies: job.dependencies.clone(), - queue_key: job.spec.queue_key.clone(), - spec_hash: job.spec_hash.clone(), - accept_seq, + } + let now = now_ms(); + let mut started = Vec::new(); + let mut written = Vec::new(); + for job in resolved { + let seq = state.next_seq; + state.next_seq += 1; + let record = StoredJob { + format: JOB_RECORD_FORMAT, + job_id: job.job_id.clone(), + request_id: params.request_id.clone(), + name: job.name, + spec_hash: job.spec_hash, + seq, status: JobStatus::Queued, + depends_on: job.depends_on, + dependency_policy: job.dependency_policy, + queue_key: job.queue_key, created_at_ms: now, - queued_at_ms: Some(now), + queued_at_ms: None, started_at_ms: None, finished_at_ms: None, - exit_code: None, - orphaned_descendants: false, - failure: None, - output_chunks: Vec::new(), - output_next_seq: 0, + cancel_requested_at_ms: None, + duration_ms: None, + exit: None, + stopped_by: None, + leftovers: None, + error: None, + spawn: None, + output_end_bytes: 0, }; - self.persist_record(&record)?; - accepted.push(key.clone()); - state - .secret_envs - .insert(key.clone(), job.secret_env.clone()); - state.jobs.insert(key, record); + written.push(self.inner.store.write_record(&job.job_id, record.encode())); + started.push(job.job_id.clone()); + state.jobs.insert( + job.job_id, + JobEntry { + record, + pending: Some(job.command), + run: None, + output: None, + }, + ); } + (started, written) + }; + // The writes were queued under the lock, so the scheduler's later + // writes land behind them; the start is answered once they are on + // disk. + let (started, written) = records; + for written in written { + let _ = written.await; } - - self.schedule_ready_jobs().await; - - let state = self.state.lock().await; + self.schedule(); + let state = self.state(); Ok(StartJobsResponse { - jobs: accepted - .into_iter() - .map(|key| { - state - .jobs - .get(&key) - .map(JobRecord::summary) - .unwrap_or_else(|| lost_summary(&key.0, &key.1)) - }) + jobs: started + .iter() + .filter_map(|job_id| state.jobs.get(job_id)) + .map(|entry| entry.record.summary()) .collect(), }) } - pub async fn read_jobs( + pub async fn read( &self, + ctx: &RequestContext, params: ReadJobsParams, - ) -> Result { - validate_id_component("namespace", ¶ms.namespace)?; - if params.jobs.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "job/read requires at least one job id", + ) -> Result { + let limits = &self.inner.config.limits; + check_job_count("jobs", params.jobs.len(), limits.max_request_jobs)?; + if params.max_bytes > limits.max_read_bytes { + return Err(ProtocolError::above_ceiling( + "maxBytes", + params.max_bytes, + limits.max_read_bytes, )); } + if params.wait_ms > self.inner.config.max_wait_ms { + return Err(ProtocolError::above_ceiling( + "waitMs", + params.wait_ms, + self.inner.config.max_wait_ms, + )); + } + for (index, entry) in params.jobs.iter().enumerate() { + if entry.after.is_some() && entry.tail_bytes.is_some() { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].tailBytes"), + "after and tailBytes exclude each other", + )); + } + if let Some(max_bytes) = entry.max_bytes + && max_bytes > params.max_bytes + { + return Err(ProtocolError::above_ceiling( + format!("jobs[{index}].maxBytes"), + max_bytes, + params.max_bytes, + )); + } + } + self.maintain(); + // The read pages from its own handles on terminal jobs' output: the + // shared cache may evict a log between loading and paging. + let mut logs: BTreeMap> = BTreeMap::new(); + for entry in params + .jobs + .iter() + .filter(|entry| needs_output(¶ms, entry)) + { + if !logs.contains_key(&entry.job_id) + && let Some(log) = self.ensure_output(&entry.job_id).await + { + logs.insert(entry.job_id.clone(), log); + } + } - let deadline = params - .wait_ms - .map(|wait_ms| tokio::time::Instant::now() + Duration::from_millis(wait_ms)); + let (starts, initial) = { + let state = self.state(); + let mut starts = Vec::new(); + let mut initial = Vec::new(); + for (index, entry) in params.jobs.iter().enumerate() { + let job = state.jobs.get(&entry.job_id); + let end = job.map_or(0, output_end); + let start = match (entry.after, entry.tail_bytes) { + (Some(after), _) => after, + (None, Some(tail)) => end.saturating_sub(tail), + (None, None) => 0, + }; + if job.is_some() && start > end { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].after"), + format!("after {start} is past the end of the output ({end})"), + )); + } + starts.push(start); + initial.push(job.map(|job| job.record.status)); + } + (starts, initial) + }; + let deadline = tokio::time::Instant::now() + Duration::from_millis(params.wait_ms); + let mut changes = self.inner.changed.subscribe(); loop { - let response = { - let state = self.state.lock().await; - let response = build_read_response(&state, ¶ms); - let all_terminal = response - .jobs - .iter() - .all(|result| result.summary.status.is_terminal()); - let should_wait = params.wait_ms.is_some() - && !all_terminal - && deadline.is_some_and(|deadline| tokio::time::Instant::now() < deadline); - if should_wait { None } else { Some(response) } - }; - if let Some(response) = response { - return Ok(response); + changes.borrow_and_update(); + if self.read_ready(¶ms, &starts, &initial) { + break; } - - let Some(deadline) = deadline else { - continue; - }; tokio::select! { - _ = self.notify.notified() => {} - _ = tokio::time::sleep_until(deadline) => {} + () = ctx.cancel.cancelled() => return Err(ProtocolError::cancelled()), + () = tokio::time::sleep_until(deadline) => break, + changed = changes.changed() => if changed.is_err() { break }, } } - } - pub async fn list_jobs( - &self, - params: ListJobsParams, - ) -> Result { - validate_id_component("namespace", ¶ms.namespace)?; - if matches!(params.limit, Some(0)) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "job/list limit must be greater than zero", - )); - } - let state = self.state.lock().await; - let mut jobs = state - .jobs - .values() - .filter(|record| record.namespace == params.namespace) - .map(JobRecord::summary) - .collect::>(); - jobs.sort_by(|left, right| { - right - .created_at_ms - .cmp(&left.created_at_ms) - .then_with(|| left.job_id.cmp(&right.job_id)) - }); - if let Some(limit) = params.limit { - jobs.truncate(limit); + // Jobs that finished during the wait: take their logs under the + // lock that pages them, loading any the cache no longer holds. + let mut attempted = BTreeSet::new(); + let state = loop { + let missing = match self.take_logs(¶ms, &mut logs, &mut attempted) { + Ok(state) => break state, + Err(missing) => missing, + }; + for job_id in missing { + if let Some(log) = self.ensure_output(&job_id).await { + logs.insert(job_id, log); + } + } + }; + let mut budget = params.max_bytes; + let mut items = MAX_PAGE_CHUNKS; + let mut jobs = Vec::with_capacity(params.jobs.len()); + for (entry, start) in params.jobs.iter().zip(starts) { + let Some(job) = state.jobs.get(&entry.job_id) else { + jobs.push(JobReadEntry::Error { + job_id: entry.job_id.clone(), + error: unknown_job(&entry.job_id), + }); + continue; + }; + let entry_budget = entry.max_bytes.map_or(budget, |max| max.min(budget)); + let page = |log: &OutputLog| (log.page(start, entry_budget, items), log.end()); + let output = match &job.run { + Some(_) => with_log(job, page), + None => logs.get(&entry.job_id).map(|log| page(log)), + }; + let output = match output { + Some((page, end)) => { + budget -= page.bytes; + items = items.saturating_sub(page.items()); + OutputPage { + chunks: page.chunks, + next: page.next, + end, + gaps: page.gaps, + } + } + None => OutputPage { + next: start, + end: output_end(job), + ..OutputPage::default() + }, + }; + jobs.push(JobReadEntry::Job { + summary: Box::new(job.record.summary()), + output, + }); } - Ok(ListJobsResponse { jobs }) + Ok(ReadJobsResponse { jobs }) } - pub async fn cancel_jobs( + pub async fn cancel( &self, + _ctx: &RequestContext, params: CancelJobsParams, - ) -> Result { - validate_id_component("namespace", ¶ms.namespace)?; - if params.jobs.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "job/cancel requires at least one job id", - )); - } - - let now = now_ms(); - let mut cancel_handles = Vec::new(); - let target_ids = { - let state = self.state.lock().await; - resolve_cancel_scope(&state, ¶ms) - }; - - { - let mut state = self.state.lock().await; - for key in &target_ids { - let Some(status) = state.jobs.get(key).map(|record| record.status) else { - continue; - }; - if status.is_terminal() { - continue; + ) -> Result { + let limits = &self.inner.config.limits; + check_job_count("jobIds", params.job_ids.len(), limits.max_request_jobs)?; + self.maintain(); + let response = { + let mut state = self.state(); + let mut named = Vec::new(); + for job_id in ¶ms.job_ids { + if !named.contains(job_id) { + named.push(job_id.clone()); } - let running = state.running.get(key).cloned(); - let Some(record) = state.jobs.get_mut(key) else { - continue; - }; - match status { - JobStatus::Running | JobStatus::CancelRequested => { - record.status = JobStatus::CancelRequested; - record.failure = Some(if params.force { - "job force cancellation requested".to_owned() - } else { - "job cancellation requested".to_owned() - }); - if let Some(running) = running { - cancel_handles.push(running); - } + } + let mut dependents = Vec::new(); + if params.scope == JobCancelScope::Dependents { + let mut affected: BTreeSet = named.iter().cloned().collect(); + loop { + let next: Vec = state + .jobs + .values() + .filter(|entry| { + !entry.record.status.is_terminal() + && !affected.contains(&entry.record.job_id) + && entry + .record + .depends_on + .iter() + .any(|dependency| affected.contains(dependency)) + }) + .map(|entry| entry.record.job_id.clone()) + .collect(); + if next.is_empty() { + break; } - _ => { - record.status = JobStatus::Cancelled; - record.finished_at_ms = Some(now); - record.failure = Some("job cancelled before start".to_owned()); + for job_id in next { + affected.insert(job_id.clone()); + dependents.push(job_id); } } - self.persist_record(record)?; } - } - - for running in cancel_handles { - running.cancel.notify_one(); - } - self.notify.notify_waiters(); - self.schedule_ready_jobs().await; - - let state = self.state.lock().await; - Ok(CancelJobsResponse { - jobs: target_ids + let now = now_ms(); + for job_id in named.iter().chain(&dependents) { + if let Some(entry) = state.jobs.get_mut(job_id) { + self.cancel_entry(entry, params.force, now); + } + } + let max_entries = limits.max_request_jobs as usize; + let mut jobs: Vec = named .iter() - .map(|key| { - state - .jobs - .get(key) - .map(JobRecord::summary) - .unwrap_or_else(|| lost_summary(&key.0, &key.1)) + .map(|job_id| match state.jobs.get(job_id) { + Some(entry) => JobSummaryEntry::Job { + summary: Box::new(entry.record.summary()), + }, + None => JobSummaryEntry::Error { + job_id: job_id.clone(), + error: unknown_job(job_id), + }, }) - .collect(), - }) + .collect(); + let room = max_entries.saturating_sub(jobs.len()); + let unlisted = dependents.len().saturating_sub(room); + jobs.extend(dependents.iter().take(room).filter_map(|job_id| { + state.jobs.get(job_id).map(|entry| JobSummaryEntry::Job { + summary: Box::new(entry.record.summary()), + }) + })); + CancelJobsResponse { + jobs, + unlisted_job_count: u32::try_from(unlisted).unwrap_or(u32::MAX), + } + }; + self.bump(); + self.schedule(); + Ok(response) + } + + /// Cancels one job: a queued job ends at once; a running one is + /// signalled and ends `cancelled` when its root exits. + fn cancel_entry(&self, entry: &mut JobEntry, force: bool, now: i64) { + match entry.record.status { + JobStatus::Queued => { + entry.pending = None; + entry.record.status = JobStatus::Cancelled; + entry.record.finished_at_ms = Some(now); + self.inner + .store + .queue_record(&entry.record.job_id, entry.record.encode()); + } + JobStatus::Running => { + if entry.record.cancel_requested_at_ms.is_none() { + entry.record.cancel_requested_at_ms = Some(now); + self.inner + .store + .queue_record(&entry.record.job_id, entry.record.encode()); + } + if let Some(run) = &entry.run { + run.execution.stop(if force { + StoppedBy::Kill + } else { + StoppedBy::Terminate + }); + } + } + _ => {} + } } - pub async fn interrupt_all(&self) -> Result<(), EnvironmentProtocolError> { - let now = now_ms(); - let mut running = Vec::new(); - { - let mut state = self.state.lock().await; - state.accepting = false; - let keys = state - .jobs - .iter() - .filter(|(_, record)| !record.status.is_terminal()) - .map(|(key, _)| key.clone()) - .collect::>(); - for key in keys { - let active = state.running.get(&key).cloned(); - let Some(record) = state.jobs.get_mut(&key) else { - continue; + /// Validates a start and resolves its dependencies within the request. + fn resolve_start(&self, params: &StartJobsParams) -> Result, ProtocolError> { + let limits = &self.inner.config.limits; + check_job_count("jobs", params.jobs.len(), limits.max_request_jobs)?; + let mut ids = BTreeSet::new(); + let mut names = BTreeMap::new(); + for (index, job) in params.jobs.iter().enumerate() { + let field = format!("jobs[{index}].command"); + validate_command(&job.command, &field)?; + if matches!(job.command.input, ExecInput::Pipe {}) { + return Err(ProtocolError::invalid_field( + format!("{field}.input"), + "jobs take no pipe input; use bytes", + )); + } + if !ids.insert(job.job_id.clone()) { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].jobId"), + format!("job id {} appears twice", job.job_id), + )); + } + if let Some(name) = &job.name + && names.insert(name.clone(), job.job_id.clone()).is_some() + { + return Err(ProtocolError::invalid_field( + format!("jobs[{index}].name"), + format!("job name {name} appears twice"), + )); + } + } + let mut resolved = Vec::new(); + for (index, job) in params.jobs.iter().enumerate() { + let mut depends_on = Vec::new(); + for dependency in &job.depends_on { + let job_id = match dependency { + JobDependency::JobId { job_id } => job_id.clone(), + JobDependency::Name { name } => names.get(name).cloned().ok_or_else(|| { + ProtocolError::invalid_field( + format!("jobs[{index}].dependsOn"), + format!("no job of this request is named {name}"), + ) + })?, }; - if let Some(active) = active { - active.interrupted.store(true, Ordering::Release); - record.status = JobStatus::CancelRequested; - record.failure = - Some("environment close requested job interruption".to_owned()); - running.push(active); - } else { - record.status = JobStatus::Interrupted; - record.finished_at_ms = Some(now); - record.failure = Some("environment closed before job started".to_owned()); + if !depends_on.contains(&job_id) { + depends_on.push(job_id); } - self.persist_record(record)?; } + let spec_hash = spec_hash( + job.name.as_ref(), + &job.command, + &depends_on, + job.dependency_policy, + job.queue_key.as_ref(), + ); + resolved.push(ResolvedJob { + job_id: job.job_id.clone(), + name: job.name.clone(), + command: job.command.clone(), + depends_on, + dependency_policy: job.dependency_policy, + queue_key: job.queue_key.clone(), + spec_hash, + }); } - for active in running { - active.cancel.notify_one(); - } - self.notify.notify_waiters(); - Ok(()) + reject_cycles(&resolved)?; + Ok(resolved) } - async fn schedule_ready_jobs(&self) { + /// Fails jobs whose dependencies can no longer succeed and starts every + /// runnable job whose queue is free, in start order. + fn schedule(&self) { + let mut state = self.state(); + let mut changed = false; loop { - let ready = { - let mut state = self.state.lock().await; - if let Err(error) = self.mark_dependency_failures(&mut state) { - eprintln!("lightspeed-envd job dependency update failed: {error:?}"); + let now = now_ms(); + let mut progressed = false; + let doomed: Vec = state + .jobs + .values() + .filter(|entry| { + entry.record.status == JobStatus::Queued + && entry.record.dependency_policy == JobDependencyPolicy::AllSucceeded + && entry.record.depends_on.iter().any(|dependency| { + state.jobs.get(dependency).is_none_or(|dependency| { + dependency.record.status.is_terminal() + && dependency.record.status != JobStatus::Succeeded + }) + }) + }) + .map(|entry| entry.record.job_id.clone()) + .collect(); + for job_id in doomed { + if let Some(entry) = state.jobs.get_mut(&job_id) { + entry.pending = None; + entry.record.status = JobStatus::DependencyFailed; + entry.record.finished_at_ms = Some(now); + self.inner + .store + .queue_record(&job_id, entry.record.encode()); + progressed = true; } + } - let mut busy_queues = state - .running - .keys() - .filter_map(|key| state.jobs.get(key)) - .filter_map(|record| { - record - .queue_key - .as_ref() - .map(|queue_key| (record.namespace.clone(), queue_key.clone())) - }) - .collect::>(); - let mut candidates = state - .jobs - .iter() - .filter_map(|(key, record)| { - if !matches!(record.status, JobStatus::Accepted | JobStatus::Queued) - || state.running.contains_key(key) - || !dependencies_satisfied(&state, record) - { - return None; - } - Some(( - key.clone(), - record.accept_seq, - record.queued_at_ms.unwrap_or(record.created_at_ms), - record.created_at_ms, - record.job_id.clone(), - )) - }) - .collect::>(); - candidates.sort_by(|left, right| { - left.1 - .cmp(&right.1) - .then_with(|| left.2.cmp(&right.2)) - .then_with(|| left.3.cmp(&right.3)) - .then_with(|| left.4.cmp(&right.4)) - }); - - let mut ready_ids = Vec::new(); - for (key, _, _, _, _) in candidates { - let Some(record) = state.jobs.get(&key) else { - continue; - }; - if let Some(queue_key) = record.queue_key.as_ref() - && !busy_queues.insert((record.namespace.clone(), queue_key.clone())) - { - continue; - } - ready_ids.push(key.clone()); + let mut busy: BTreeSet = state + .jobs + .values() + .filter(|entry| entry.record.status == JobStatus::Running) + .filter_map(|entry| entry.record.queue_key.clone()) + .collect(); + let mut runnable: Vec<(u64, JobId)> = state + .jobs + .values() + .filter(|entry| { + entry.record.status == JobStatus::Queued + && entry.record.depends_on.iter().all(|dependency| { + state.jobs.get(dependency).is_none_or(|dependency| { + match entry.record.dependency_policy { + JobDependencyPolicy::AllSucceeded => { + dependency.record.status == JobStatus::Succeeded + } + JobDependencyPolicy::AllTerminal => { + dependency.record.status.is_terminal() + } + } + }) + }) + }) + .map(|entry| (entry.record.seq, entry.record.job_id.clone())) + .collect(); + runnable.sort(); + for (_, job_id) in runnable { + let entry = state.jobs.get_mut(&job_id).expect("runnable job exists"); + let newly_runnable = entry.record.queued_at_ms.is_none(); + if newly_runnable { + entry.record.queued_at_ms = Some(now); } - let now = now_ms(); - let mut ready = Vec::new(); - for key in ready_ids { - let mut ready_record = None; - if let Some(record) = state.jobs.get_mut(&key) { - record.status = JobStatus::Running; - record.started_at_ms = Some(now); - record.failure = None; - if let Err(error) = self.persist_record(record) { - eprintln!( - "lightspeed-envd failed to persist running job {}: {error:?}", - job_key_id(&key) - ); - } - ready_record = Some(record.clone()); - } - if let Some(record) = ready_record { - let secret_env = state.secret_envs.get(&key).cloned().unwrap_or_default(); - let running = Arc::new(RunningJob { - cancel: Notify::new(), - interrupted: AtomicBool::new(false), - }); - state.running.insert(key, running.clone()); - ready.push((record.clone(), running, secret_env)); + if let Some(key) = &entry.record.queue_key + && !busy.insert(key.clone()) + { + if newly_runnable { + self.inner + .store + .queue_record(&job_id, entry.record.encode()); } + continue; } - ready - }; - - if ready.is_empty() { - break; + self.start_entry(entry, now); + progressed = true; } - for (record, running, secret_env) in ready { - let manager = self.clone(); - tokio::spawn(async move { - manager.run_job(record, running, secret_env).await; - }); + changed |= progressed; + if !progressed { + break; } } + drop(state); + if changed { + self.bump(); + } } - fn mark_dependency_failures( - &self, - state: &mut JobManagerState, - ) -> Result<(), EnvironmentProtocolError> { - loop { - let mut changed = false; - let failed = state - .jobs - .iter() - .filter(|(_, record)| { - matches!(record.status, JobStatus::Accepted | JobStatus::Queued) - && record.dependency_policy == JobDependencyPolicy::AllSucceeded - && record.dependencies.iter().any(|dependency| { - state - .jobs - .get(&job_key(&record.namespace, dependency)) - .map(|dependency| { - dependency.status.is_terminal() - && dependency.status != JobStatus::Succeeded - }) - .unwrap_or(true) - }) - }) - .map(|(job_id, _)| job_id.clone()) - .collect::>(); - if failed.is_empty() { - return Ok(()); - } - let now = now_ms(); - for job_id in failed { - let Some(record) = state.jobs.get_mut(&job_id) else { - continue; - }; - if record.status.is_terminal() { - continue; - } - record.status = JobStatus::DependencyFailed; - record.finished_at_ms = Some(now); - record.failure = Some("one or more dependencies did not succeed".to_owned()); - self.persist_record(record)?; - changed = true; + /// Starts one runnable job, or fails it when its `cwd` or spawn fails. + fn start_entry(&self, entry: &mut JobEntry, now: i64) { + let config = &self.inner.config; + let job_id = entry.record.job_id.clone(); + let Some(command) = entry.pending.take() else { + entry.record.status = JobStatus::Failed; + entry.record.finished_at_ms = Some(now); + entry.record.error = Some(ProtocolError::internal("the job's command was lost")); + self.inner + .store + .queue_record(&job_id, entry.record.encode()); + return; + }; + let options = ExecOptions { + timeout: command.timeout_ms.map(Duration::from_millis), + terminate_grace: Duration::from_millis(config.limits.cancel_grace_ms), + after_exit: AfterExit::Manual, + }; + let spawned = resolve_cwd(config, command.cwd.as_ref()).and_then(|cwd| { + let sink = JobOutput { + job_id: job_id.clone(), + log: OutputLog::default(), + cap: config.limits.max_retained_output_bytes, + store: self.inner.store.clone(), + changed: self.inner.changed.clone(), + }; + PreparedCommand::new(command, cwd, &config.scrubbed_env, None).spawn(sink, options, 0) + }); + match spawned { + Ok(execution) => { + entry.record.status = JobStatus::Running; + entry.record.started_at_ms = Some(now); + entry.record.spawn = Some(SpawnRecord::new(execution.group(), &config.instance_id)); + self.inner + .store + .queue_record(&job_id, entry.record.encode()); + execution.start_input(); + entry.run = Some(Run { + execution: execution.clone(), + started: Instant::now(), + }); + let manager = self.clone(); + tokio::spawn(async move { manager.supervise(job_id, execution).await }); } - if !changed { - return Ok(()); + Err(error) => { + entry.record.status = JobStatus::Failed; + entry.record.finished_at_ms = Some(now); + entry.record.error = Some(error); + self.inner + .store + .queue_record(&job_id, entry.record.encode()); } - self.notify.notify_waiters(); } } - async fn run_job( - &self, - record: JobRecord, - running: Arc, - secret_env: BTreeMap, - ) { - let result = self - .spawn_and_wait(record.clone(), running.clone(), secret_env) - .await; - let (status, exit_code, orphaned_descendants, failure) = match result { - Ok(outcome) => { - let (status, failure) = match outcome.finish { - FinishKind::Exit => { - if outcome.exit_code == Some(0) { - (JobStatus::Succeeded, None) - } else { - ( - JobStatus::Failed, - Some(match outcome.exit_code { - Some(code) => format!("job exited with status {code}"), - None => "job exited without a status code".to_owned(), - }), - ) - } - } - FinishKind::Cancelled => { - (JobStatus::Cancelled, Some("job cancelled".to_owned())) - } - FinishKind::Interrupted => ( - JobStatus::Interrupted, - Some("job interrupted because the environment closed".to_owned()), - ), - FinishKind::TimedOut => (JobStatus::TimedOut, Some("job timed out".to_owned())), - }; - ( - status, - outcome.exit_code, - outcome.orphaned_descendants, - failure, - ) - } - Err(error) => (JobStatus::Failed, None, false, Some(error.message)), + /// Follows a running job to its end: root exit, sweep, settled output, + /// final record, then the scheduler. + async fn supervise(self, job_id: JobId, execution: Execution) { + execution.wait_until(|state| state.exited()).await; + let leftovers = execution.sample_leftovers(LeftoverDisposition::Swept); + if leftovers.is_some() || execution.group_alive() { + execution + .group() + .terminate(Duration::from_millis( + self.inner.config.limits.cancel_grace_ms, + )) + .await; + } + execution.settle(true).await; + let (exit, stopped_by, error, log) = execution.with_state(|state| { + ( + state.exit().cloned(), + state.stopped_by(), + state.error().cloned(), + Arc::new(state.sink.log.clone()), + ) + }); + let status = match (&exit, stopped_by) { + (_, Some(StoppedBy::Timeout)) => JobStatus::TimedOut, + (_, Some(StoppedBy::Terminate | StoppedBy::Kill)) => JobStatus::Cancelled, + (Some(ExitStatus::Code { code: 0 }), None) if error.is_none() => JobStatus::Succeeded, + _ => JobStatus::Failed, }; + let error = error.or_else(|| match (&exit, status) { + (Some(exit), JobStatus::Failed) if !matches!(exit, ExitStatus::Code { .. }) => { + Some(ProtocolError::process_failed(describe_exit(exit))) + } + _ => None, + }); + let final_record = { + let state = self.state(); + let Some(entry) = state.jobs.get(&job_id) else { + return; + }; + let mut record = entry.record.clone(); + record.status = status; + record.exit = exit; + record.stopped_by = stopped_by; + record.leftovers = leftovers; + record.error = error; + record.finished_at_ms = Some(now_ms()); + record.duration_ms = entry + .run + .as_ref() + .map(|run| u64::try_from(run.started.elapsed().as_millis()).unwrap_or(u64::MAX)); + record.output_end_bytes = log.end(); + record + }; + // The output and the record are on disk before anyone sees the job + // terminal, so a later load from disk finds everything. + self.inner + .store + .finish(&job_id, final_record.encode()) + .await; { - let mut state = self.state.lock().await; - let key = job_key(&record.namespace, &record.job_id); - state.running.remove(&key); - state.secret_envs.remove(&key); - if let Some(record) = state.jobs.get_mut(&key) { - record.status = status; - record.exit_code = exit_code; - record.orphaned_descendants = orphaned_descendants; - record.failure = failure; - record.finished_at_ms = Some(now_ms()); - if let Err(error) = self.persist_record(record) { - eprintln!( - "lightspeed-envd failed to persist finished job {}: {error:?}", - record.job_id - ); + let mut state = self.state(); + let JobsState { jobs, cached, .. } = &mut *state; + if let Some(entry) = jobs.get_mut(&job_id) { + let cancel_requested_at_ms = entry.record.cancel_requested_at_ms; + entry.record = final_record; + if entry.record.cancel_requested_at_ms != cancel_requested_at_ms { + entry.record.cancel_requested_at_ms = cancel_requested_at_ms; + self.inner + .store + .queue_record(&job_id, entry.record.encode()); } + entry.run = None; + entry.output = Some(log); + cache_output(jobs, cached, &job_id); } } - self.notify.notify_waiters(); - self.kick_scheduler(); - } - - fn kick_scheduler(&self) { - let manager = self.clone(); - tokio::spawn(async move { - manager.schedule_ready_jobs().await; - }); - } - - async fn spawn_and_wait( - &self, - record: JobRecord, - running: Arc, - secret_env: BTreeMap, - ) -> Result { - if record.argv.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "job argv must not be empty", - )); - } - let cwd = record - .cwd - .as_ref() - .map(|path| self.resolve_cwd(path)) - .transpose()? - .unwrap_or_else(|| self.cwd.clone()); - - let mut command = Command::new(&record.argv[0]); - for name in secret_env.keys() { - if record.env.contains_key(name) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("job env collides with secret env: {name}"), - )); - } - } - for name in self.scrubbed_env.iter() { - command.env_remove(name); - } - command - .args(&record.argv[1..]) - .current_dir(cwd) - .envs(record.env.iter()) - .stdin(if record.stdin.is_some() { - Stdio::piped() - } else { - Stdio::null() - }) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - for (name, value) in &secret_env { - command.env(name, value.expose()); - } - process_group::spawn_in_own_group(&mut command); - - let mut child = command.spawn().map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - format!("spawn job {:?}: {error}", record.argv), - ) - })?; - let pgid = child.id(); - let stdout = child.stdout.take(); - let stderr = child.stderr.take(); - if let Some(input) = record.stdin.as_ref() { - let Some(stdin) = child.stdin.as_mut() else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - "job stdin was not available", - )); - }; - stdin.write_all(input.as_slice()).await.map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - })?; - child.stdin.take(); - } - - let stdout_task = stdout.map(|stdout| { - tokio::spawn(read_job_stream( - self.clone(), - record.namespace.clone(), - record.job_id.clone(), - stdout, - JobOutputStream::Stdout, - redactions_for_secret_env(&secret_env), - )) - }); - let stderr_task = stderr.map(|stderr| { - tokio::spawn(read_job_stream( - self.clone(), - record.namespace.clone(), - record.job_id.clone(), - stderr, - JobOutputStream::Stderr, - redactions_for_secret_env(&secret_env), - )) - }); - - let kill_job_group = || { - if let Some(pgid) = pgid { - process_group::kill_group(pgid); - } - }; - let (finish, status) = if let Some(timeout_ms) = record.timeout_ms { - tokio::select! { - status = child.wait() => { - (FinishKind::Exit, status) - } - _ = running.cancel.notified() => { - kill_job_group(); - let _ = child.start_kill(); - let finish = if running.interrupted.load(Ordering::Acquire) { - FinishKind::Interrupted - } else { - FinishKind::Cancelled - }; - (finish, child.wait().await) - } - _ = tokio::time::sleep(Duration::from_millis(timeout_ms)) => { - kill_job_group(); - let _ = child.start_kill(); - (FinishKind::TimedOut, child.wait().await) - } - } - } else { - tokio::select! { - status = child.wait() => { - (FinishKind::Exit, status) - } - _ = running.cancel.notified() => { - kill_job_group(); - let _ = child.start_kill(); - let finish = if running.interrupted.load(Ordering::Acquire) { - FinishKind::Interrupted - } else { - FinishKind::Cancelled - }; - (finish, child.wait().await) - } + self.bump(); + self.schedule(); + } + + /// A terminal job's output: the cached log, or the log loaded from disk + /// (and cached). `None` for an unknown or running job. + async fn ensure_output(&self, job_id: &JobId) -> Option> { + let (dir, end) = { + let mut state = self.state(); + let JobsState { jobs, cached, .. } = &mut *state; + let entry = jobs.get(job_id)?; + if entry.run.is_some() || !entry.record.status.is_terminal() { + return None; } - }; - - // The root process's exit is ground truth for the job outcome. - // Descendants may still hold the output pipes open, so drain them for - // a bounded grace period, sweep whatever the script left running, and - // abandon the pipes rather than letting completion depend on EOF. - let mut stream_tasks = [stdout_task, stderr_task]; - // Probe for survivors only on a natural exit; on cancel/timeout the - // group was killed deliberately and not-yet-reaped children would - // read as false positives. - let mut orphaned_descendants = - matches!(finish, FinishKind::Exit) && pgid.is_some_and(process_group::group_alive); - let drained = - drain_stream_tasks(&mut stream_tasks, process_group::OUTPUT_DRAIN_GRACE).await; - orphaned_descendants |= !drained; - if orphaned_descendants && let Some(pgid) = pgid { - process_group::sweep_group(pgid).await; - } - if !drained - && !drain_stream_tasks(&mut stream_tasks, process_group::OUTPUT_DRAIN_GRACE).await - { - for task in stream_tasks.iter().flatten() { - task.abort(); + if let Some(log) = entry.output.clone() { + cache_output(jobs, cached, job_id); + return Some(log); } - } - - let status = status.map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - })?; - Ok(SpawnOutcome { - finish, - exit_code: status.code(), - orphaned_descendants, - }) - } - - fn resolve_cwd(&self, path: &EnvironmentPath) -> Result { - let candidate = if path.is_absolute() { - PathBuf::from(path.as_str()) - } else if path.as_str() == "." { - self.cwd.clone() - } else { - self.cwd.join(path.as_str()) - }; - let normalized = normalize_path(candidate); - if !normalized.starts_with(&self.fs_root) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Forbidden, - format!( - "job cwd is outside bridge fs root: {} (root {})", - normalized.display(), - self.fs_root.display() - ), - )); - } - Ok(normalized) - } - - fn persist_record(&self, record: &JobRecord) -> Result<(), EnvironmentProtocolError> { - persist_record_at(&self.jobs_root, record).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - format!("persist job {}: {error}", record.job_id), + ( + self.inner.store.root().join(job_id.as_str()), + entry.record.output_end_bytes, ) - }) - } -} - -/// Joins the output-reader tasks against one shared deadline. Returns true -/// when every reader finished; unfinished readers stay in their slot. -async fn drain_stream_tasks(tasks: &mut [Option>; 2], grace: Duration) -> bool { - let deadline = tokio::time::Instant::now() + grace; - let mut drained = true; - for slot in tasks.iter_mut() { - let Some(task) = slot.as_mut() else { - continue; }; - if tokio::time::timeout_at(deadline, task).await.is_ok() { - *slot = None; - } else { - drained = false; - } - } - drained -} - -fn require_accepting(state: &JobManagerState) -> Result<(), EnvironmentProtocolError> { - if state.accepting { - Ok(()) - } else { - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Conflict, - "environment target is closed and no longer accepts jobs", - )) - } -} - -async fn read_job_stream( - manager: JobManager, - namespace: String, - job_id: JobId, - mut reader: R, - stream: JobOutputStream, - redactions: Vec>, -) where - R: AsyncRead + Unpin, -{ - let mut buffer = vec![0; 8192]; - loop { - let read = match reader.read(&mut buffer).await { - Ok(0) => { - manager.notify.notify_waiters(); - return; + let cap = self.inner.config.limits.max_retained_output_bytes; + let loaded = tokio::task::spawn_blocking(move || load_output(&dir, end, cap)).await; + let log = match loaded { + Ok(Ok(log)) => log, + Ok(Err(error)) => { + eprintln!("lightspeed-envd: could not load output of job {job_id}: {error}"); + OutputLog::from_records([], end) } - Ok(read) => read, Err(error) => { - let mut state = manager.state.lock().await; - if let Some(record) = state.jobs.get_mut(&job_key(&namespace, &job_id)) - && !record.status.is_terminal() - { - record.failure = Some(error.to_string()); - if let Err(error) = manager.persist_record(record) { - eprintln!( - "lightspeed-envd failed to persist stream failure for {job_id}: {error:?}" - ); - } - } - manager.notify.notify_waiters(); - return; + eprintln!("lightspeed-envd: could not load output of job {job_id}: {error}"); + OutputLog::from_records([], end) } }; - let mut state = manager.state.lock().await; - if let Some(record) = state.jobs.get_mut(&job_key(&namespace, &job_id)) { - if record.status.is_terminal() { - // The job completed without waiting for pipe EOF; late output - // from orphaned descendants must not mutate the terminal - // record. - return; - } - let seq = record.output_next_seq; - record.output_next_seq += 1; - record.output_chunks.push(JobOutputChunk { - seq, - stream, - chunk: ByteChunk::from(redact_bytes(&buffer[..read], &redactions)), - }); - if let Err(error) = manager.persist_record(record) { - eprintln!("lightspeed-envd failed to persist output for {job_id}: {error:?}"); + let mut state = self.state(); + let JobsState { jobs, cached, .. } = &mut *state; + let entry = jobs.get_mut(job_id)?; + let log = match &entry.output { + Some(log) => log.clone(), + None => { + entry.record.output_end_bytes = entry.record.output_end_bytes.max(log.end()); + let log = Arc::new(log); + entry.output = Some(log.clone()); + log } - } - manager.notify.notify_waiters(); + }; + cache_output(jobs, cached, job_id); + Some(log) } -} -fn validate_and_resolve_start( - state: &JobManagerState, - params: &StartJobsParams, -) -> Result, EnvironmentProtocolError> { - if params.jobs.is_empty() { - return Err(invalid_request("job/start requires at least one job")); - } - validate_id_component("namespace", ¶ms.namespace)?; - validate_id_component("request_id", ¶ms.request_id)?; - - let mut seen_job_ids = BTreeSet::new(); - let mut seen_names = BTreeMap::new(); - for spec in ¶ms.jobs { - validate_id_component("job id", spec.job_id.as_str())?; - if spec.argv.is_empty() { - return Err(invalid_request(format!( - "job {} argv must not be empty", - spec.job_id - ))); - } - for name in spec.secret_env.keys() { - validate_nonempty("secret env name", name)?; - if spec.env.contains_key(name) { - return Err(invalid_request(format!( - "job {} env collides with secret env: {name}", - spec.job_id - ))); + /// Adds to `logs` the cached output of every terminal job the read + /// pages, and answers the locked state to page from. When a terminal + /// job's output is in neither the cache nor `logs` (and was not + /// `attempted`), answers those jobs instead: load them and call again. + fn take_logs( + &self, + params: &ReadJobsParams, + logs: &mut BTreeMap>, + attempted: &mut BTreeSet, + ) -> Result, Vec> { + let state = self.state(); + let mut missing = Vec::new(); + for entry in params + .jobs + .iter() + .filter(|entry| needs_output(params, entry)) + { + let Some(job) = state.jobs.get(&entry.job_id) else { + continue; + }; + if job.run.is_some() || logs.contains_key(&entry.job_id) { + continue; } - } - if !seen_job_ids.insert(spec.job_id.as_str().to_owned()) { - return Err(invalid_request(format!( - "duplicate job id in start request: {}", - spec.job_id - ))); - } - if let Some(name) = spec.name.as_deref() { - validate_nonempty("job name", name)?; - if seen_names - .insert(name.to_owned(), spec.job_id.clone()) - .is_some() - { - return Err(invalid_request(format!("duplicate job name: {name}"))); + if let Some(log) = &job.output { + logs.insert(entry.job_id.clone(), log.clone()); + } else if job.record.status.is_terminal() && attempted.insert(entry.job_id.clone()) { + missing.push(entry.job_id.clone()); } } - validate_optional_id_component("queue_key", spec.queue_key.as_deref())?; + if missing.is_empty() { + Ok(state) + } else { + Err(missing) + } } - let request_job_ids = params - .jobs - .iter() - .map(|spec| spec.job_id.as_str().to_owned()) - .collect::>(); - - let mut resolved = Vec::new(); - for spec in ¶ms.jobs { - let dependencies = resolve_explicit_dependencies(&seen_names, spec)?; - for dependency in &dependencies { - if !request_job_ids.contains(dependency.as_str()) - && !state - .jobs - .contains_key(&job_key(¶ms.namespace, dependency)) - { - return Err(invalid_request(format!( - "job {} depends on unknown job id {}", - spec.job_id, dependency - ))); - } + fn read_ready( + &self, + params: &ReadJobsParams, + starts: &[u64], + initial: &[Option], + ) -> bool { + let state = self.state(); + let entries = params.jobs.iter().zip(starts).zip(initial); + let all_terminal = params.jobs.iter().all(|entry| { + state + .jobs + .get(&entry.job_id) + .is_none_or(|job| job.record.status.is_terminal()) + }); + if all_terminal { + return true; } - - let spec_hash = job_spec_hash(params, spec, &dependencies)?; - let key = job_key(¶ms.namespace, &spec.job_id); - if let Some(existing) = state.jobs.get(&key) - && existing.spec_hash != spec_hash - { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Conflict, - format!( - "job id already exists with different input in namespace {}: {}", - params.namespace, spec.job_id - ), - )); + match params.return_on { + ReturnOn::Exit => false, + ReturnOn::AnyOutput => entries.into_iter().any(|((entry, start), initial)| { + let job = state.jobs.get(&entry.job_id); + job.map(|job| job.record.status) != *initial + || job.is_some_and(|job| output_end(job) > *start) + }), + ReturnOn::Window => entries.into_iter().any(|((entry, start), _)| { + let budget = entry.max_bytes.unwrap_or(params.max_bytes); + state + .jobs + .get(&entry.job_id) + .is_some_and(|job| output_end(job).saturating_sub(*start) >= budget) + }), } - - resolved.push(ResolvedJob { - spec: spec.clone(), - dependencies, - secret_env: spec.secret_env.clone(), - spec_hash, - }); } - reject_cycles(&resolved)?; - Ok(resolved) -} - -fn resolve_explicit_dependencies( - names: &BTreeMap, - spec: &JobStartSpec, -) -> Result, EnvironmentProtocolError> { - let mut dependencies = Vec::new(); - for dependency in &spec.depends_on { - match (&dependency.job_id, &dependency.name) { - (Some(job_id), None) => push_unique_job_id(&mut dependencies, job_id.clone()), - (None, Some(name)) => { - validate_nonempty("dependency name", name)?; - let Some(job_id) = names.get(name) else { - return Err(invalid_request(format!( - "job {} depends on unknown local name {name}", - spec.job_id - ))); - }; - push_unique_job_id(&mut dependencies, job_id.clone()); - } - (Some(_), Some(_)) => { - return Err(invalid_request(format!( - "job {} dependency must use either jobId or name, not both", - spec.job_id - ))); - } - (None, None) => { - return Err(invalid_request(format!( - "job {} dependency must include jobId or name", - spec.job_id - ))); - } - } - } - for dependency in &dependencies { - if dependency == &spec.job_id { - return Err(invalid_request(format!( - "job {} cannot depend on itself", - spec.job_id - ))); + /// Garbage-collects terminal jobs past the record retention. + fn maintain(&self) { + let retention = + i64::try_from(self.inner.config.limits.record_retention_ms).unwrap_or(i64::MAX); + let now = now_ms(); + let mut state = self.state(); + let expired: Vec = state + .jobs + .values() + .filter(|entry| { + entry.record.status.is_terminal() + && entry + .record + .finished_at_ms + .is_some_and(|finished| now.saturating_sub(finished) >= retention) + }) + .map(|entry| entry.record.job_id.clone()) + .collect(); + for job_id in expired { + state.jobs.remove(&job_id); + state.cached.retain(|cached| cached != &job_id); + self.inner.store.remove(&job_id); } } - Ok(dependencies) -} -fn reject_cycles(resolved: &[ResolvedJob]) -> Result<(), EnvironmentProtocolError> { - let graph = resolved - .iter() - .map(|job| { - ( - job.spec.job_id.as_str().to_owned(), - job.dependencies - .iter() - .filter(|dependency| { - resolved - .iter() - .any(|job| job.spec.job_id.as_str() == dependency.as_str()) - }) - .map(|dependency| dependency.as_str().to_owned()) - .collect::>(), - ) - }) - .collect::>(); - let mut visiting = BTreeSet::new(); - let mut visited = BTreeSet::new(); - for job_id in graph.keys() { - visit_job(job_id, &graph, &mut visiting, &mut visited)?; + fn state(&self) -> MutexGuard<'_, JobsState> { + self.inner + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) } - Ok(()) -} -fn visit_job( - job_id: &str, - graph: &BTreeMap>, - visiting: &mut BTreeSet, - visited: &mut BTreeSet, -) -> Result<(), EnvironmentProtocolError> { - if visited.contains(job_id) { - return Ok(()); - } - if !visiting.insert(job_id.to_owned()) { - return Err(invalid_request(format!( - "job dependency cycle includes {job_id}" - ))); - } - for dependency in graph.get(job_id).into_iter().flatten() { - visit_job(dependency, graph, visiting, visited)?; + fn bump(&self) { + self.inner + .changed + .send_modify(|version| *version = version.wrapping_add(1)); } - visiting.remove(job_id); - visited.insert(job_id.to_owned()); - Ok(()) } -fn push_unique_job_id(dependencies: &mut Vec, job_id: JobId) { - if !dependencies +/// An identical retry replays every job; any other overlap with existing +/// jobs is `jobIdInUse`. `None` when every job is new. +fn replay_or_conflict( + state: &JobsState, + request_id: &JobRequestId, + resolved: &[ResolvedJob], +) -> Result>, ProtocolError> { + let existing = resolved .iter() - .any(|existing| existing.as_str() == job_id.as_str()) - { - dependencies.push(job_id); - } -} - -fn dependencies_satisfied(state: &JobManagerState, record: &JobRecord) -> bool { - record.dependencies.iter().all(|dependency| { - let Some(dependency) = state.jobs.get(&job_key(&record.namespace, dependency)) else { - return false; + .filter(|job| state.jobs.contains_key(&job.job_id)) + .count(); + if existing == 0 { + return Ok(None); + } + let mut summaries = Vec::new(); + for job in resolved { + let in_use = |detail: &str| { + ProtocolError::failed_precondition( + PreconditionReason::JobIdInUse, + format!( + "job id {} {detail}; choose a new job id for a new run", + job.job_id + ), + ) }; - match record.dependency_policy { - JobDependencyPolicy::AllSucceeded => dependency.status == JobStatus::Succeeded, - JobDependencyPolicy::AllTerminal => dependency.status.is_terminal(), + let Some(entry) = state.jobs.get(&job.job_id) else { + return Err(in_use( + "is new while other jobs of this request already exist", + )); + }; + if &entry.record.request_id != request_id { + return Err(in_use("was started by another request")); } - }) -} - -fn build_read_response(state: &JobManagerState, params: &ReadJobsParams) -> ReadJobsResponse { - ReadJobsResponse { - jobs: params - .jobs - .iter() - .map( - |job_id| match state.jobs.get(&job_key(¶ms.namespace, job_id)) { - Some(record) => { - let (output_chunks, output_next_seq) = - select_output_chunks(record, params.after_seq, params.max_bytes); - JobReadResult { - summary: record.summary(), - output_chunks, - output_next_seq, - artifacts: Vec::new(), - } - } - None => JobReadResult { - summary: lost_summary(¶ms.namespace, job_id), - output_chunks: Vec::new(), - output_next_seq: 0, - artifacts: Vec::new(), - }, - }, - ) - .collect(), + if entry.record.spec_hash != job.spec_hash { + return Err(in_use("was started with a different spec")); + } + summaries.push(entry.record.summary()); } + Ok(Some(summaries)) } -fn select_output_chunks( - record: &JobRecord, - after_seq: Option, - max_bytes: Option, -) -> (Vec, u64) { - let after_seq = after_seq.unwrap_or(0); - let mut chunks = Vec::new(); - let mut bytes = 0usize; - let mut next_seq = after_seq; - - for chunk in record - .output_chunks +fn reject_cycles(resolved: &[ResolvedJob]) -> Result<(), ProtocolError> { + let index: BTreeMap<&JobId, usize> = resolved .iter() - .filter(|chunk| chunk.seq >= after_seq) - { - let chunk_bytes = chunk.chunk.as_slice(); - if let Some(max_bytes) = max_bytes { - if bytes >= max_bytes { - break; - } - let remaining = max_bytes - bytes; - if chunk_bytes.len() > remaining { - chunks.push(JobOutputChunk { - seq: chunk.seq, - stream: chunk.stream, - chunk: ByteChunk::from(chunk_bytes[..remaining].to_vec()), - }); - next_seq = chunk.seq + 1; - break; + .enumerate() + .map(|(position, job)| (&job.job_id, position)) + .collect(); + // 0 unvisited, 1 on the stack, 2 done. + let mut marks = vec![0u8; resolved.len()]; + fn visit( + position: usize, + resolved: &[ResolvedJob], + index: &BTreeMap<&JobId, usize>, + marks: &mut [u8], + ) -> Result<(), usize> { + match marks[position] { + 2 => return Ok(()), + 1 => return Err(position), + _ => {} + } + marks[position] = 1; + for dependency in &resolved[position].depends_on { + if let Some(&next) = index.get(dependency) { + visit(next, resolved, index, marks)?; } } - bytes += chunk_bytes.len(); - next_seq = chunk.seq + 1; - chunks.push(chunk.clone()); - } - - if chunks.is_empty() && next_seq < record.output_next_seq { - next_seq = record.output_next_seq; + marks[position] = 2; + Ok(()) } - - (chunks, next_seq) -} - -fn resolve_cancel_scope(state: &JobManagerState, params: &CancelJobsParams) -> Vec { - let mut target_ids = params - .jobs - .iter() - .map(|job_id| job_key(¶ms.namespace, job_id)) - .collect::>(); - match params.scope { - JobCancelScope::Job => {} - JobCancelScope::Dependents => { - let mut changed = true; - while changed { - changed = false; - for (key, record) in &state.jobs { - if record.namespace != params.namespace - || record.status.is_terminal() - || target_ids.contains(key) - { - continue; - } - if record.dependencies.iter().any(|dependency| { - target_ids.contains(&job_key(¶ms.namespace, dependency)) - }) { - changed |= target_ids.insert(key.clone()); - } - } - } + for position in 0..resolved.len() { + if let Err(cycle) = visit(position, resolved, &index, &mut marks) { + return Err(ProtocolError::invalid_field( + format!("jobs[{cycle}].dependsOn"), + format!( + "job {} depends on itself through a cycle", + resolved[cycle].job_id + ), + )); } } - - target_ids.into_iter().collect() + Ok(()) } -fn job_spec_hash( - params: &StartJobsParams, - spec: &JobStartSpec, - dependencies: &[JobId], -) -> Result { - #[derive(Serialize)] - #[serde(rename_all = "camelCase")] - struct JobHashMaterial<'a> { - namespace: &'a str, - job: JobHashSpec<'a>, - } - +/// The identity hash of a job's spec. Secret values and input bytes enter +/// only as their own digests. +fn spec_hash( + name: Option<&JobName>, + command: &CommandSpec, + depends_on: &[JobId], + dependency_policy: JobDependencyPolicy, + queue_key: Option<&QueueKey>, +) -> String { #[derive(Serialize)] #[serde(rename_all = "camelCase")] - struct JobHashSpec<'a> { - job_id: &'a JobId, - name: &'a Option, + struct Material<'a> { + name: Option<&'a JobName>, argv: &'a [String], - cwd: &'a Option, + cwd: Option<&'a EnvironmentPath>, env: &'a BTreeMap, - secret_env_names: Vec<&'a String>, - stdin: &'a Option, + secret_env: BTreeMap<&'a str, String>, + input: String, timeout_ms: Option, - dependencies: &'a [JobId], + depends_on: &'a [JobId], dependency_policy: JobDependencyPolicy, - queue_key: &'a Option, - } - - let material = JobHashMaterial { - namespace: params.namespace.as_str(), - job: JobHashSpec { - job_id: &spec.job_id, - name: &spec.name, - argv: &spec.argv, - cwd: &spec.cwd, - env: &spec.env, - secret_env_names: spec.secret_env.keys().collect(), - stdin: &spec.stdin, - timeout_ms: spec.timeout_ms, - dependencies, - dependency_policy: spec.dependency_policy, - queue_key: &spec.queue_key, + queue_key: Option<&'a QueueKey>, + } + let digest = |bytes: &[u8]| encode_hex(&Sha256::digest(bytes)); + let material = Material { + name, + argv: &command.argv, + cwd: command.cwd.as_ref(), + env: &command.env, + secret_env: command + .secret_env + .iter() + .map(|(name, value)| (name.as_str(), digest(value.expose().as_bytes()))) + .collect(), + input: match &command.input { + ExecInput::None {} => "none".to_owned(), + ExecInput::Bytes { data } => format!("bytes:{}", digest(data.as_slice())), + ExecInput::Pipe {} => "pipe".to_owned(), }, + timeout_ms: command.timeout_ms, + depends_on, + dependency_policy, + queue_key, }; - let bytes = serde_json::to_vec(&material).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Internal, - format!("encode job idempotency hash: {error}"), - ) - })?; - Ok(format!("{:016x}", fnv1a64(&bytes))) + digest(&serde_json::to_vec(&material).expect("hash material encodes")) } -fn fnv1a64(bytes: &[u8]) -> u64 { - let mut hash = 0xcbf29ce484222325u64; - for byte in bytes { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x100000001b3); +fn resolve_cwd( + config: &JobConfig, + cwd: Option<&EnvironmentPath>, +) -> Result { + match cwd { + Some(path) => config.confinement.resolve_directory(path), + None => Ok(config.cwd.clone()), } - hash } -impl JobRecord { - fn summary(&self) -> JobSummary { - JobSummary { - namespace: self.namespace.clone(), - job_id: self.job_id.clone(), - name: self.name.clone(), - status: self.status, - dependencies: self.dependencies.clone(), - created_at_ms: self.created_at_ms, - queued_at_ms: self.queued_at_ms, - started_at_ms: self.started_at_ms, - finished_at_ms: self.finished_at_ms, - exit_code: self.exit_code, - orphaned_descendants: self.orphaned_descendants, - failure: self.failure.clone(), - queue_key: self.queue_key.clone(), - } - } +/// Whether a `job/read` entry pages output (rather than only a summary). +fn needs_output(params: &ReadJobsParams, entry: &JobReadRequest) -> bool { + params.max_bytes > 0 || entry.after.is_some() || entry.tail_bytes.is_some() } -fn lost_summary(namespace: &str, job_id: &JobId) -> JobSummary { - JobSummary { - namespace: namespace.to_owned(), - job_id: job_id.clone(), - name: None, - status: JobStatus::Lost, - dependencies: Vec::new(), - created_at_ms: 0, - queued_at_ms: None, - started_at_ms: None, - finished_at_ms: Some(now_ms()), - exit_code: None, - orphaned_descendants: false, - failure: Some("unknown job id".to_owned()), - queue_key: None, +/// Runs `f` on the job's retained output: the live log while it runs, the +/// cached log once terminal. `None` when neither is in memory. +fn with_log(entry: &JobEntry, f: impl FnOnce(&OutputLog) -> R) -> Option { + if let Some(run) = &entry.run { + return Some(run.execution.with_state(|state| f(&state.sink.log))); } + entry.output.as_deref().map(f) } -fn persist_record_at(jobs_root: &PathBuf, record: &JobRecord) -> anyhow::Result<()> { - std::fs::create_dir_all(jobs_root)?; - let stem = record_file_stem(&record.namespace, &record.job_id); - let path = jobs_root.join(format!("{stem}.json")); - let temp_path = jobs_root.join(format!("{stem}.json.tmp")); - let bytes = serde_json::to_vec_pretty(record)?; - std::fs::write(&temp_path, bytes)?; - std::fs::rename(temp_path, path)?; - Ok(()) -} - -fn job_key(namespace: &str, job_id: &JobId) -> JobKey { - (namespace.to_owned(), job_id.clone()) -} - -fn job_key_id(key: &JobKey) -> String { - format!("{}/{}", key.0, key.1) +fn output_end(entry: &JobEntry) -> u64 { + with_log(entry, OutputLog::end).unwrap_or(entry.record.output_end_bytes) } -fn record_file_stem(namespace: &str, job_id: &JobId) -> String { - format!("{namespace}--{}", job_id.as_str()) +fn cache_output( + jobs: &mut BTreeMap, + cached: &mut VecDeque, + job_id: &JobId, +) { + cached.retain(|cached| cached != job_id); + cached.push_back(job_id.clone()); + while cached.len() > MAX_CACHED_OUTPUTS { + if let Some(evicted) = cached.pop_front() + && let Some(entry) = jobs.get_mut(&evicted) + { + entry.output = None; + } + } } -fn normalize_path(path: PathBuf) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(std::path::MAIN_SEPARATOR.to_string()), - Component::CurDir => {} - Component::ParentDir => { - normalized.pop(); - } - Component::Normal(segment) => normalized.push(segment), - } +fn check_job_count(field: &str, count: usize, max: u32) -> Result<(), ProtocolError> { + if count == 0 { + return Err(ProtocolError::invalid_field(field, "name at least one job")); } - normalized + if count > max as usize { + return Err(ProtocolError::above_ceiling( + field, + count as u64, + u64::from(max), + )); + } + Ok(()) } -fn now_ms() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64) - .unwrap_or_default() +fn unknown_job(job_id: &JobId) -> ProtocolError { + ProtocolError::not_found(format!("unknown job id: {job_id}")) } -fn validate_optional_id_component( - label: &str, - value: Option<&str>, -) -> Result<(), EnvironmentProtocolError> { - if let Some(value) = value { - validate_id_component(label, value)?; +fn status_name(status: JobStatus) -> &'static str { + match status { + JobStatus::Queued => "queued", + JobStatus::Running => "running", + JobStatus::Succeeded => "succeeded", + JobStatus::Failed => "failed", + JobStatus::TimedOut => "timedOut", + JobStatus::Cancelled => "cancelled", + JobStatus::DependencyFailed => "dependencyFailed", + JobStatus::Interrupted => "interrupted", } - Ok(()) } -fn validate_id_component(label: &str, value: &str) -> Result<(), EnvironmentProtocolError> { - validate_nonempty(label, value)?; - if value.len() > 128 { - return Err(invalid_request(format!( - "{label} must be at most 128 bytes" - ))); +/// Reads one job directory's record. +fn load_record(dir: &Path) -> Result { + if !dir.is_dir() { + return Err("not a job directory".to_owned()); } - let mut chars = value.chars(); - let Some(first) = chars.next() else { - return Err(invalid_request(format!("{label} must not be empty"))); - }; - if !first.is_ascii_alphanumeric() { - return Err(invalid_request(format!( - "{label} must start with an ASCII alphanumeric character" - ))); + let bytes = std::fs::read(dir.join("record.json")).map_err(|error| error.to_string())?; + let record: StoredJob = serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + if record.format != JOB_RECORD_FORMAT { + return Err(format!( + "record format {} is not {JOB_RECORD_FORMAT}", + record.format + )); } - if !chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | ':' | '-')) { - return Err(invalid_request(format!( - "{label} may contain only ASCII alphanumeric characters, '_', '.', ':', or '-'" - ))); + if dir.file_name().and_then(|name| name.to_str()) != Some(record.job_id.as_str()) { + return Err("the directory does not match the job id".to_owned()); } - Ok(()) + Ok(record) } -fn validate_nonempty(label: &str, value: &str) -> Result<(), EnvironmentProtocolError> { - if value.trim().is_empty() { - return Err(invalid_request(format!("{label} must not be empty"))); +/// Moves an unreadable job aside so it never blocks startup again. +fn quarantine(path: &Path, quarantine_dir: &Path, reason: &str) { + let name = path + .file_name() + .map_or_else(|| "job".into(), |name| name.to_string_lossy().into_owned()); + let target = quarantine_dir.join(format!("{name}.{}", now_ms())); + let moved = + std::fs::create_dir_all(quarantine_dir).and_then(|()| std::fs::rename(path, &target)); + match moved { + Ok(()) => eprintln!( + "lightspeed-envd: quarantined unreadable job record {} ({reason}) at {}", + path.display(), + target.display() + ), + Err(error) => eprintln!( + "lightspeed-envd: could not quarantine unreadable job record {} ({reason}): {error}", + path.display() + ), } - Ok(()) } -fn invalid_request(message: impl Into) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::InvalidRequest, message) +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) + }) } #[cfg(test)] mod tests { - use std::{collections::BTreeMap, time::Duration}; - use environment_protocol::{ - data::jobs::{ - JobCancelScope, JobDependency, JobDependencyPolicy, JobStatus, ListJobsParams, - ReadJobsParams, StartJobsParams, - }, - shared::{JobId, SecretString}, + bytes::ByteChunk, + data::{exec::GapReason, job::JobSpec}, + error::{ErrorData, ProtocolErrorKind}, }; use super::*; - const TEST_NAMESPACE: &str = "session_1"; + fn limits() -> JobLimits { + JobLimits { + max_read_bytes: 4 * 1024 * 1024, + max_request_jobs: 64, + max_retained_output_bytes: 2 * 1024 * 1024, + record_retention_ms: 7 * 24 * 60 * 60 * 1000, + cancel_grace_ms: 200, + } + } - #[tokio::test(flavor = "current_thread")] - async fn job_manager_runs_parallel_jobs_and_retains_output() { + struct Fixture { + _temp: tempfile::TempDir, + root: PathBuf, + state_dir: PathBuf, + manager: JobManager, + } + + fn fixture_with(limits: JobLimits) -> Fixture { let temp = tempfile::tempdir().expect("tempdir"); let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); + let state_dir = root.join("state"); + let manager = manager_at(&root, &state_dir, limits); + Fixture { + _temp: temp, + root, + state_dir, + manager, + } + } - let response = manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "parallel".to_owned(), - jobs: vec![job("job-a", "printf a"), job("job-b", "printf b")], - }) - .await - .expect("start"); + fn fixture() -> Fixture { + fixture_with(limits()) + } - assert_eq!(response.jobs.len(), 2); - let read = wait_for_jobs(&manager, ["job-a", "job-b"]).await; - assert!( - read.jobs - .iter() - .all(|job| job.summary.status == JobStatus::Succeeded) - ); - assert!( - read.jobs - .iter() - .all(|job| !job.summary.orphaned_descendants) - ); - let output = read - .jobs - .iter() - .flat_map(|job| { - job.output_chunks - .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) - }) - .collect::>(); - assert!(output.contains(&b'a')); - assert!(output.contains(&b'b')); + fn manager_at(root: &Path, state_dir: &Path, limits: JobLimits) -> JobManager { + let confinement = + Confinement::new(root.to_owned(), state_dir.to_owned()).expect("confinement"); + JobManager::new(JobConfig { + cwd: root.to_owned(), + confinement: Arc::new(confinement), + state_dir: state_dir.to_owned(), + scrubbed_env: Vec::new(), + instance_id: DaemonInstanceId::new("envd_test").unwrap(), + limits, + max_wait_ms: 60_000, + }) + .expect("manager") } - #[tokio::test(flavor = "current_thread")] - async fn job_manager_injects_secret_env_without_persisting_values() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - let mut spec = job("secret-job", "printf \"$SECRET_TOKEN\""); - spec.secret_env.insert( - "SECRET_TOKEN".to_owned(), - SecretString::new("super-secret-token"), - ); + fn ctx() -> RequestContext { + RequestContext::detached() + } - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "secret".to_owned(), - jobs: vec![spec], - }) - .await - .expect("start"); + fn id(value: &str) -> JobId { + JobId::new(value).unwrap() + } - let read = wait_for_jobs(&manager, ["secret-job"]).await; - let result = result(&read, "secret-job"); - assert_eq!(result.summary.status, JobStatus::Succeeded); - let output = result - .output_chunks - .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) - .collect::>(); - assert_eq!(output, b""); - - let record_path = root - .join(".lightspeed") - .join("jobs") - .join("session_1--secret-job.json"); - let persisted = std::fs::read_to_string(record_path).expect("persisted job record"); - assert!(!persisted.contains("super-secret-token")); - assert!(persisted.contains("SECRET_TOKEN")); + fn request(value: &str) -> JobRequestId { + JobRequestId::new(value).unwrap() } - #[tokio::test(flavor = "current_thread")] - async fn job_manager_honors_queue_key_order() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let marker = root.join("order.txt"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut first = job("queued-z", "printf 1 >> order.txt"); - let mut second = job("queued-a", "printf 2 >> order.txt"); - let mut third = job("queued-m", "printf 3 >> order.txt"); - first.queue_key = Some("repo".to_owned()); - second.queue_key = Some("repo".to_owned()); - third.queue_key = Some("repo".to_owned()); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "queue".to_owned(), - jobs: vec![first, second, third], - }) + fn job(job_id: &str, script: &str) -> JobSpec { + JobSpec { + job_id: id(job_id), + name: None, + command: CommandSpec { + argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], + cwd: None, + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input: ExecInput::None {}, + timeout_ms: None, + }, + depends_on: Vec::new(), + dependency_policy: JobDependencyPolicy::AllSucceeded, + queue_key: None, + } + } + + fn start(request_id: &str, jobs: Vec) -> StartJobsParams { + StartJobsParams { + request_id: request(request_id), + jobs, + } + } + + fn entry(job_id: &str) -> JobReadRequest { + JobReadRequest { + job_id: id(job_id), + after: None, + tail_bytes: None, + max_bytes: None, + } + } + + /// Waits until every job is terminal and reads their output heads. + async fn finished(manager: &JobManager, job_ids: &[&str]) -> Vec<(JobSummary, OutputPage)> { + let response = manager + .read( + &ctx(), + ReadJobsParams { + jobs: job_ids.iter().map(|job_id| entry(job_id)).collect(), + max_bytes: 4 * 1024 * 1024, + wait_ms: 30_000, + return_on: ReturnOn::Exit, + }, + ) .await - .expect("start"); + .expect("read"); + response + .jobs + .into_iter() + .map(|entry| match entry { + JobReadEntry::Job { summary, output } => { + assert!(summary.status.is_terminal(), "{summary:?}"); + (*summary, output) + } + JobReadEntry::Error { job_id, error } => panic!("{job_id}: {error}"), + }) + .collect() + } - let read = wait_for_jobs(&manager, ["queued-z", "queued-a", "queued-m"]).await; - assert!( - read.jobs + fn text(page: &OutputPage) -> String { + String::from_utf8( + page.chunks .iter() - .all(|job| job.summary.status == JobStatus::Succeeded) - ); - assert_eq!(std::fs::read_to_string(marker).expect("marker"), "123"); + .flat_map(|chunk| chunk.data.as_slice().to_vec()) + .collect(), + ) + .unwrap() + } + + fn precondition(error: &ProtocolError) -> Option { + match &error.data { + ErrorData::FailedPrecondition { reason } => Some(*reason), + _ => None, + } } #[tokio::test(flavor = "current_thread")] - async fn job_manager_honors_explicit_dependencies_and_dependency_failure() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut dependent = job("dependent", "printf should-not-run"); - dependent.depends_on = vec![JobDependency::name("setup")]; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "deps".to_owned(), - jobs: vec![named_job("setup", "setup", "exit 7"), dependent], - }) + async fn jobs_run_and_report_exits() { + let fixture = fixture(); + let response = fixture + .manager + .start( + &ctx(), + start("r1", vec![job("ok", "printf done"), job("bad", "exit 4")]), + ) .await .expect("start"); - - let read = wait_for_jobs(&manager, ["setup", "dependent"]).await; - let setup = result(&read, "setup"); - let dependent = result(&read, "dependent"); - assert_eq!(setup.summary.status, JobStatus::Failed); - assert_eq!(dependent.summary.status, JobStatus::DependencyFailed); + assert_eq!(response.jobs.len(), 2); + let jobs = finished(&fixture.manager, &["ok", "bad"]).await; + assert_eq!(jobs[0].0.status, JobStatus::Succeeded); + assert_eq!(text(&jobs[0].1), "done"); + assert_eq!(jobs[0].1.end, 4); + assert!(jobs[0].0.duration_ms.is_some()); + assert_eq!(jobs[1].0.status, JobStatus::Failed); + assert_eq!(jobs[1].0.exit, Some(ExitStatus::Code { code: 4 })); + assert_eq!(fixture.manager.running_count().await, 0); } #[tokio::test(flavor = "current_thread")] - async fn job_manager_all_terminal_dependencies_run_after_failed_dependency() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut cleanup = job("cleanup", "printf cleanup"); - cleanup.depends_on = vec![JobDependency::job_id("fails")]; - cleanup.dependency_policy = JobDependencyPolicy::AllTerminal; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "cleanup".to_owned(), - jobs: vec![job("fails", "exit 2"), cleanup], - }) + async fn a_resubmitted_job_id_is_refused_and_an_identical_retry_replays() { + let fixture = fixture(); + let script = "echo run >> runs.txt"; + fixture + .manager + .start(&ctx(), start("first", vec![job("build", script)])) .await .expect("start"); + finished(&fixture.manager, &["build"]).await; - let read = wait_for_jobs(&manager, ["fails", "cleanup"]).await; - assert_eq!(result(&read, "fails").summary.status, JobStatus::Failed); - assert_eq!( - result(&read, "cleanup").summary.status, - JobStatus::Succeeded - ); + let replay = fixture + .manager + .start(&ctx(), start("first", vec![job("build", script)])) + .await + .expect("replay"); + assert_eq!(replay.jobs[0].status, JobStatus::Succeeded); + + let error = fixture + .manager + .start(&ctx(), start("second", vec![job("build", script)])) + .await + .unwrap_err(); + assert_eq!(precondition(&error), Some(PreconditionReason::JobIdInUse)); + let error = fixture + .manager + .start(&ctx(), start("first", vec![job("build", "echo changed")])) + .await + .unwrap_err(); + assert_eq!(precondition(&error), Some(PreconditionReason::JobIdInUse)); + let error = fixture + .manager + .start( + &ctx(), + start("first", vec![job("build", script), job("new", "true")]), + ) + .await + .unwrap_err(); + assert_eq!(precondition(&error), Some(PreconditionReason::JobIdInUse)); + + let runs = std::fs::read_to_string(fixture.root.join("runs.txt")).unwrap(); + assert_eq!(runs, "run\n", "nothing ran twice"); } #[tokio::test(flavor = "current_thread")] - async fn job_manager_cancels_running_jobs_and_queued_dependents() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut dependent = job("after", "printf after"); - dependent.depends_on = vec![JobDependency::job_id("sleep")]; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "cancel".to_owned(), - jobs: vec![job("sleep", "sleep 5"), dependent], - }) + async fn large_input_never_keeps_a_timeout_from_firing() { + // The child never reads its input, so the timeout must be armed + // before the input is written or the job runs forever. + let fixture = fixture(); + let mut stuck = job("stuck", "sleep 30"); + stuck.command.input = ExecInput::Bytes { + data: ByteChunk::from(vec![b'x'; 1024 * 1024]), + }; + stuck.command.timeout_ms = Some(300); + let mut echo = job("echo", "cat"); + let input: Vec = (0..512 * 1024u32).map(|i| b'a' + (i % 26) as u8).collect(); + echo.command.input = ExecInput::Bytes { + data: ByteChunk::from(input.clone()), + }; + fixture + .manager + .start(&ctx(), start("r", vec![stuck, echo])) .await .expect("start"); - - tokio::time::sleep(Duration::from_millis(50)).await; - manager - .cancel_jobs(CancelJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - jobs: vec![JobId::new("sleep")], - scope: JobCancelScope::Dependents, - force: false, - }) - .await - .expect("cancel"); - let read = wait_for_jobs(&manager, ["sleep", "after"]).await; - assert_eq!(result(&read, "sleep").summary.status, JobStatus::Cancelled); - assert_eq!(result(&read, "after").summary.status, JobStatus::Cancelled); + let jobs = finished(&fixture.manager, &["stuck", "echo"]).await; + assert_eq!(jobs[0].0.status, JobStatus::TimedOut); + assert_eq!(jobs[0].0.stopped_by, Some(StoppedBy::Timeout)); + assert_eq!(jobs[1].0.status, JobStatus::Succeeded); + assert_eq!(text(&jobs[1].1).as_bytes(), input.as_slice()); } #[tokio::test(flavor = "current_thread")] - async fn job_manager_interrupts_jobs_and_rejects_starts_after_close() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - let mut dependent = job("after-close", "printf should-not-run"); - dependent.depends_on = vec![JobDependency::job_id("active-at-close")]; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "close-active".to_owned(), - jobs: vec![job("active-at-close", "sleep 5"), dependent], - }) + async fn verbose_output_keeps_head_and_tail_and_reads_back_after_a_restart() { + let cap = 64 * 1024; + let fixture = fixture_with(JobLimits { + max_retained_output_bytes: cap, + ..limits() + }); + fixture + .manager + .start( + &ctx(), + start( + "r", + vec![job( + "verbose", + "i=0; while [ $i -lt 30000 ]; do echo line-$i; i=$((i+1)); done; echo 'FAILED: 3 tests'", + )], + ), + ) .await .expect("start"); - - tokio::time::sleep(Duration::from_millis(50)).await; - manager.interrupt_all().await.expect("interrupt all"); - let read = wait_for_jobs(&manager, ["active-at-close", "after-close"]).await; - assert_eq!( - result(&read, "active-at-close").summary.status, - JobStatus::Interrupted - ); - assert_eq!( - result(&read, "after-close").summary.status, - JobStatus::Interrupted - ); - - let error = manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "after-close".to_owned(), - jobs: vec![job("rejected", "printf should-not-run")], - }) + let jobs = finished(&fixture.manager, &["verbose"]).await; + let (summary, page) = &jobs[0]; + assert_eq!(summary.status, JobStatus::Succeeded); + assert_eq!(page.gaps.len(), 1); + assert_eq!(page.gaps[0].reason, GapReason::Overflow); + assert_eq!(page.gaps[0].offset, cap / 2); + let shown = text(page); + assert!(shown.starts_with("line-0\n")); + assert!(shown.ends_with("FAILED: 3 tests\n")); + assert_eq!(shown.len() as u64 + page.gaps[0].omitted_bytes, page.end); + + // Head and tail in one read, as the job service asks for them. + let response = fixture + .manager + .read( + &ctx(), + ReadJobsParams { + jobs: vec![ + JobReadRequest { + max_bytes: Some(10), + ..entry("verbose") + }, + JobReadRequest { + tail_bytes: Some(16), + ..entry("verbose") + }, + ], + max_bytes: 26, + wait_ms: 0, + return_on: ReturnOn::Exit, + }, + ) .await - .expect_err("closed manager must reject starts"); - assert_eq!(error.code, EnvironmentProtocolErrorCode::Conflict); + .expect("head and tail"); + let pages: Vec = response + .jobs + .into_iter() + .map(|entry| match entry { + JobReadEntry::Job { output, .. } => output, + JobReadEntry::Error { error, .. } => panic!("{error}"), + }) + .collect(); + assert_eq!(text(&pages[0]), "line-0\nlin"); + assert_eq!(text(&pages[1]), "FAILED: 3 tests\n"); + + let restarted = manager_at( + &fixture.root, + &fixture.state_dir, + JobLimits { + max_retained_output_bytes: cap, + ..limits() + }, + ); + let reloaded = finished(&restarted, &["verbose"]).await; + assert_eq!(reloaded[0].0.status, JobStatus::Succeeded); + let reread = &reloaded[0].1; + assert_eq!(text(reread), shown, "the output reads back from disk"); + assert_eq!(reread.gaps, page.gaps); + assert_eq!(reread.end, page.end); } - #[cfg(unix)] - async fn wait_for_process_gone(pid: i32) { - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - if rustix::process::test_kill_process( - rustix::process::Pid::from_raw(pid).expect("valid child pid"), + #[tokio::test(flavor = "current_thread")] + async fn pages_continue_exactly_where_they_stopped() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + start( + "r", + vec![job( + "page", + "i=0; while [ $i -lt 2000 ]; do printf '%09d\\n' $i; i=$((i+1)); done", + )], + ), ) - .is_err() - { - return; + .await + .expect("start"); + finished(&fixture.manager, &["page"]).await; + let mut after = 0; + let mut collected = String::new(); + loop { + let response = fixture + .manager + .read( + &ctx(), + ReadJobsParams { + jobs: vec![JobReadRequest { + after: Some(after), + ..entry("page") + }], + max_bytes: 100, + wait_ms: 0, + return_on: ReturnOn::Exit, + }, + ) + .await + .expect("page"); + let JobReadEntry::Job { output, .. } = &response.jobs[0] else { + panic!("job read failed"); + }; + collected.push_str(&text(output)); + if output.next == output.end { + break; } - assert!( - std::time::Instant::now() < deadline, - "orphaned descendant {pid} was not terminated" - ); - tokio::time::sleep(Duration::from_millis(20)).await; + assert!(output.next > after); + after = output.next; } + assert_eq!(collected.len(), 20_000, "no byte of 20 000 was skipped"); + assert!(collected.ends_with("000001999\n")); } - #[cfg(unix)] - fn read_pid_file(path: &std::path::Path) -> i32 { - std::fs::read_to_string(path) - .expect("orphan pid file") - .trim() - .parse() - .expect("orphan pid") - } - - #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn job_manager_completes_when_orphaned_descendants_hold_pipes() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "orphan".to_owned(), - jobs: vec![job( - "orphan", - "sleep 30 & echo $! > orphan.pid; printf started; exit 3", - )], - }) + async fn a_read_of_more_jobs_than_the_cache_holds_returns_every_output() { + let fixture = fixture(); + let count = MAX_CACHED_OUTPUTS + 4; + let names: Vec = (0..count).map(|index| format!("job-{index}")).collect(); + fixture + .manager + .start( + &ctx(), + start( + "r", + names + .iter() + .map(|name| job(name, &format!("printf {name}"))) + .collect(), + ), + ) .await .expect("start"); - - let read = wait_for_jobs(&manager, ["orphan"]).await; - let result = result(&read, "orphan"); - assert_eq!(result.summary.status, JobStatus::Failed); - assert_eq!(result.summary.exit_code, Some(3)); - assert!(result.summary.orphaned_descendants); - let output = result - .output_chunks - .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) - .collect::>(); - assert_eq!(output, b"started"); - - wait_for_process_gone(read_pid_file(&root.join("orphan.pid"))).await; + let ids: Vec<&str> = names.iter().map(String::as_str).collect(); + finished(&fixture.manager, &ids).await; + + // A restarted daemon holds no output in memory, so every log is + // loaded from disk during this one read. + let restarted = manager_at(&fixture.root, &fixture.state_dir, limits()); + for manager in [&fixture.manager, &restarted] { + let jobs = finished(manager, &ids).await; + for (name, (_, page)) in names.iter().zip(&jobs) { + assert_eq!(&text(page), name); + assert_eq!(page.next, page.end); + } + } } - #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn job_manager_completes_when_escaped_descendants_hold_pipes() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - // `set -m` puts the background child in its own process group, so the - // group sweep cannot reach it and completion must abandon the pipes. - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "escapee".to_owned(), - jobs: vec![job( - "escapee", - "set -m; sleep 30 & echo $! > escapee.pid; printf started; exit 7", - )], - }) + async fn unknown_jobs_are_per_entry_not_found() { + let fixture = fixture(); + let response = fixture + .manager + .read( + &ctx(), + ReadJobsParams { + jobs: vec![entry("missing")], + max_bytes: 0, + wait_ms: 1_000, + return_on: ReturnOn::Exit, + }, + ) .await - .expect("start"); - - let read = wait_for_jobs(&manager, ["escapee"]).await; - let result = result(&read, "escapee"); - assert_eq!(result.summary.status, JobStatus::Failed); - assert_eq!(result.summary.exit_code, Some(7)); - assert!(result.summary.orphaned_descendants); - - // The escapee is outside the job's process group; clean it up so it - // does not outlive the test. - let pid = read_pid_file(&root.join("escapee.pid")); - let _ = rustix::process::kill_process( - rustix::process::Pid::from_raw(pid).expect("valid child pid"), - rustix::process::Signal::KILL, - ); + .expect("read"); + match &response.jobs[0] { + JobReadEntry::Error { job_id, error } => { + assert_eq!(job_id, &id("missing")); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + } + JobReadEntry::Job { .. } => panic!("no summary is invented"), + } } + #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn job_manager_cancel_terminates_process_group() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "cancel-group".to_owned(), - jobs: vec![job("group", "sleep 30 & sleep 30")], - }) + async fn cancel_is_graceful_unless_forced() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + start( + "r", + vec![ + job( + "graceful", + "trap 'echo trapped; exit 0' TERM; sleep 30 & wait", + ), + job( + "forced", + "trap 'echo trapped; exit 0' TERM; sleep 30 & wait", + ), + job("queued", "true"), + ] + .into_iter() + .enumerate() + .map(|(index, mut spec)| { + if index == 2 { + spec.depends_on = vec![JobDependency::JobId { + job_id: id("forced"), + }]; + } + spec + }) + .collect(), + ), + ) .await .expect("start"); - - tokio::time::sleep(Duration::from_millis(50)).await; - manager - .cancel_jobs(CancelJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - jobs: vec![JobId::new("group")], - scope: JobCancelScope::Job, - force: false, - }) + tokio::time::sleep(Duration::from_millis(200)).await; + fixture + .manager + .cancel( + &ctx(), + CancelJobsParams { + job_ids: vec![id("graceful")], + scope: JobCancelScope::Job, + force: false, + }, + ) .await .expect("cancel"); - - // The backgrounded child holds the job's pipes; without group - // termination the drain would wedge until the read wait expires. - let read = wait_for_jobs(&manager, ["group"]).await; - let result = result(&read, "group"); - assert_eq!(result.summary.status, JobStatus::Cancelled); - assert!(!result.summary.orphaned_descendants); - } - - #[tokio::test(flavor = "current_thread")] - async fn job_manager_timeout_terminates_process_group() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - let mut spec = job("timeout-group", "sleep 30 & sleep 30"); - spec.timeout_ms = Some(50); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "timeout-group".to_owned(), - jobs: vec![spec], - }) + let cancelled = fixture + .manager + .cancel( + &ctx(), + CancelJobsParams { + job_ids: vec![id("forced"), id("missing")], + scope: JobCancelScope::Dependents, + force: true, + }, + ) .await - .expect("start"); - - let read = wait_for_jobs(&manager, ["timeout-group"]).await; + .expect("force cancel"); + assert_eq!(cancelled.jobs.len(), 3, "named jobs, then the dependent"); + assert!( + matches!(&cancelled.jobs[1], JobSummaryEntry::Error { error, .. } if error.kind() == ProtocolErrorKind::NotFound) + ); + let jobs = finished(&fixture.manager, &["graceful", "forced", "queued"]).await; + assert_eq!(jobs[0].0.status, JobStatus::Cancelled); + assert_eq!(jobs[0].0.stopped_by, Some(StoppedBy::Terminate)); + assert!(jobs[0].0.cancel_requested_at_ms.is_some()); assert_eq!( - result(&read, "timeout-group").summary.status, - JobStatus::TimedOut + text(&jobs[0].1), + "trapped\n", + "SIGTERM let the job clean up" ); + assert_eq!(jobs[1].0.status, JobStatus::Cancelled); + assert_eq!(jobs[1].0.stopped_by, Some(StoppedBy::Kill)); + assert_eq!(text(&jobs[1].1), "", "a forced cancel kills at once"); + assert_eq!(jobs[2].0.status, JobStatus::Cancelled); + assert_eq!(jobs[2].0.started_at_ms, None); } #[tokio::test(flavor = "current_thread")] - async fn job_manager_times_out_jobs() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - let mut spec = job("timeout", "sleep 5"); - spec.timeout_ms = Some(50); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "timeout".to_owned(), - jobs: vec![spec], - }) + async fn dependencies_and_queues_order_jobs() { + let fixture = fixture(); + let mut first = job("first", "sleep 0.2; echo first >> order.txt"); + first.name = Some(JobName::new("first job").unwrap()); + first.queue_key = Some(QueueKey::new("q").unwrap()); + let mut second = job("second", "echo second >> order.txt"); + second.queue_key = Some(QueueKey::new("q").unwrap()); + let mut after = job("after", "echo after >> order.txt; exit 1"); + after.depends_on = vec![JobDependency::Name { + name: JobName::new("first job").unwrap(), + }]; + let mut doomed = job("doomed", "echo doomed >> order.txt"); + doomed.depends_on = vec![JobDependency::JobId { + job_id: id("after"), + }]; + let mut tolerant = job("tolerant", "echo tolerant >> order.txt"); + tolerant.depends_on = vec![JobDependency::JobId { + job_id: id("after"), + }]; + tolerant.dependency_policy = JobDependencyPolicy::AllTerminal; + fixture + .manager + .start( + &ctx(), + start("r", vec![first, second, after, doomed, tolerant]), + ) .await .expect("start"); - - let read = wait_for_jobs(&manager, ["timeout"]).await; - assert_eq!(result(&read, "timeout").summary.status, JobStatus::TimedOut); - } - - #[tokio::test(flavor = "current_thread")] - async fn job_manager_retries_same_start_idempotently_and_rejects_conflict() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - let params = StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "retry".to_owned(), - jobs: vec![job("retry", "printf once")], - }; - - manager.start_jobs(params.clone()).await.expect("first"); - manager.start_jobs(params).await.expect("retry"); - let conflict = manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "retry-conflict".to_owned(), - jobs: vec![job("retry", "printf different")], - }) + let jobs = finished( + &fixture.manager, + &["first", "second", "after", "doomed", "tolerant"], + ) + .await; + assert_eq!(jobs[3].0.status, JobStatus::DependencyFailed); + assert_eq!(jobs[3].0.started_at_ms, None); + assert_eq!(jobs[4].0.status, JobStatus::Succeeded); + assert_eq!(jobs[2].0.depends_on, vec![id("first")]); + let order = std::fs::read_to_string(fixture.root.join("order.txt")).unwrap(); + let lines: Vec<&str> = order.lines().collect(); + assert_eq!(lines[0], "first", "one job per queue key, in start order"); + assert!(lines.iter().position(|line| *line == "second") > Some(0)); + assert!(!lines.contains(&"doomed")); + + let mut cyclic = job("a", "true"); + cyclic.depends_on = vec![JobDependency::JobId { job_id: id("b") }]; + let mut back = job("b", "true"); + back.depends_on = vec![JobDependency::JobId { job_id: id("a") }]; + let error = fixture + .manager + .start(&ctx(), start("cycle", vec![cyclic, back])) .await - .expect_err("conflict"); - assert_eq!(conflict.code, EnvironmentProtocolErrorCode::Conflict); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + let mut piped = job("piped", "cat"); + piped.command.input = ExecInput::Pipe {}; + let error = fixture + .manager + .start(&ctx(), start("pipe", vec![piped])) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); } + #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn job_manager_lists_latest_jobs_with_limit() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = - JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")).expect("manager"); - - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "list-1".to_owned(), - jobs: vec![job("job-old", "printf old")], - }) - .await - .expect("start old"); - tokio::time::sleep(Duration::from_millis(2)).await; - manager - .start_jobs(StartJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - request_id: "list-2".to_owned(), - jobs: vec![job("job-new", "printf new")], - }) - .await - .expect("start new"); - - let listed = manager - .list_jobs(ListJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - limit: Some(1), - }) + async fn leftovers_are_swept_and_reported() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + start( + "r", + vec![job("leaky", "sleep 30 & echo $! > orphan.pid; exit 0")], + ), + ) .await - .expect("list jobs"); - - assert_eq!(listed.jobs.len(), 1); - assert_eq!(listed.jobs[0].job_id.as_str(), "job-new"); + .expect("start"); + let jobs = finished(&fixture.manager, &["leaky"]).await; + assert_eq!(jobs[0].0.status, JobStatus::Succeeded); + let leftovers = jobs[0] + .0 + .leftovers + .as_ref() + .expect("swept leftovers reported"); + assert_eq!(leftovers.disposition, LeftoverDisposition::Swept); + let orphan: i32 = std::fs::read_to_string(fixture.root.join("orphan.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let alive = rustix::process::Pid::from_raw(orphan) + .is_some_and(|pid| rustix::process::test_kill_process(pid).is_ok()); + assert!(!alive, "the group was swept"); } + #[cfg(any(target_os = "linux", target_os = "macos"))] #[tokio::test(flavor = "current_thread")] - async fn job_manager_marks_unfinished_persisted_jobs_interrupted_on_startup() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let jobs_root = root.join(".lightspeed").join("jobs"); - std::fs::create_dir_all(&jobs_root).expect("jobs root"); - persist_record_at( - &jobs_root, - &JobRecord { - namespace: TEST_NAMESPACE.to_owned(), - job_id: JobId::new("recover"), - name: None, - argv: vec!["sleep".to_owned(), "5".to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env_names: Vec::new(), - stdin: None, - timeout_ms: None, - dependency_policy: JobDependencyPolicy::AllSucceeded, - dependencies: Vec::new(), - queue_key: None, - spec_hash: "seed".to_owned(), - accept_seq: 0, - status: JobStatus::Running, - created_at_ms: now_ms(), - queued_at_ms: Some(now_ms()), - started_at_ms: Some(now_ms()), - finished_at_ms: None, - exit_code: None, - orphaned_descendants: false, - failure: None, - output_chunks: Vec::new(), - output_next_seq: 0, - }, - ) - .expect("persist seed"); - - let recovered = JobManager::new(root.clone(), root.clone(), root.join(".lightspeed")) - .expect("recovered manager"); - let read = recovered - .read_jobs(ReadJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - jobs: vec![JobId::new("recover")], - after_seq: None, - max_bytes: None, - include_artifacts: false, - wait_ms: None, - }) + async fn a_restart_interrupts_unfinished_jobs_and_kills_their_groups() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + start("r", vec![job("long", "echo $$ > root.pid; sleep 30")]), + ) .await - .expect("read"); + .expect("start"); + let pid_file = fixture.root.join("root.pid"); + for _ in 0..250 { + if std::fs::read_to_string(&pid_file).is_ok_and(|pid| !pid.trim().is_empty()) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let root: i32 = std::fs::read_to_string(&pid_file) + .unwrap() + .trim() + .parse() + .unwrap(); + // The running record reaches the disk through the store thread. + tokio::time::sleep(Duration::from_millis(100)).await; + let restarted = manager_at(&fixture.root, &fixture.state_dir, limits()); + let jobs = finished(&restarted, &["long"]).await; + assert_eq!(jobs[0].0.status, JobStatus::Interrupted); assert_eq!( - result(&read, "recover").summary.status, - JobStatus::Interrupted + jobs[0].0.error.as_ref().map(ProtocolError::kind), + Some(ProtocolErrorKind::Aborted) ); - } - - fn job(job_id: &str, shell: &str) -> JobStartSpec { - named_job(job_id, job_id, shell) - } - - fn named_job(job_id: &str, name: &str, shell: &str) -> JobStartSpec { - JobStartSpec { - job_id: JobId::new(job_id), - name: Some(name.to_owned()), - argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), shell.to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(5_000), - depends_on: Vec::new(), - dependency_policy: JobDependencyPolicy::AllSucceeded, - queue_key: None, + for _ in 0..100 { + let alive = rustix::process::Pid::from_raw(root) + .is_some_and(|pid| rustix::process::test_kill_process(pid).is_ok()); + if !alive { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; } + panic!("the earlier instance's job group survived the restart"); } - async fn wait_for_jobs( - manager: &JobManager, - job_ids: [&str; N], - ) -> ReadJobsResponse { - manager - .read_jobs(ReadJobsParams { - namespace: TEST_NAMESPACE.to_owned(), - jobs: job_ids.iter().map(|job_id| JobId::new(*job_id)).collect(), - after_seq: None, - max_bytes: None, - include_artifacts: false, - wait_ms: Some(5_000), - }) - .await - .expect("read jobs") + #[tokio::test(flavor = "current_thread")] + async fn unreadable_records_are_quarantined_instead_of_failing_startup() { + let fixture = fixture(); + let bad = fixture.state_dir.join("jobs").join("broken"); + std::fs::create_dir_all(&bad).unwrap(); + std::fs::write(bad.join("record.json"), b"{not json").unwrap(); + std::fs::write(fixture.state_dir.join("jobs").join("stray.json"), b"{}").unwrap(); + let restarted = manager_at(&fixture.root, &fixture.state_dir, limits()); + assert_eq!(restarted.running_count().await, 0); + assert!(!bad.exists()); + let quarantined = std::fs::read_dir(fixture.state_dir.join("jobs-quarantine")) + .unwrap() + .count(); + assert_eq!(quarantined, 2); } - fn result<'a>(response: &'a ReadJobsResponse, job_id: &str) -> &'a JobReadResult { - response - .jobs - .iter() - .find(|result| result.summary.job_id.as_str() == job_id) - .expect("job result") + #[tokio::test(flavor = "current_thread")] + async fn request_bounds_are_ceilings_not_clamps() { + let fixture = fixture_with(JobLimits { + max_request_jobs: 2, + ..limits() + }); + let error = fixture + .manager + .start( + &ctx(), + start( + "r", + vec![job("a", "true"), job("b", "true"), job("c", "true")], + ), + ) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::InvalidParams { + ceiling: Some(2), + .. + } + )); + let error = fixture + .manager + .read( + &ctx(), + ReadJobsParams { + jobs: vec![JobReadRequest { + max_bytes: Some(20), + ..entry("a") + }], + max_bytes: 10, + wait_ms: 0, + return_on: ReturnOn::Exit, + }, + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + let error = fixture + .manager + .cancel( + &ctx(), + CancelJobsParams { + job_ids: Vec::new(), + scope: JobCancelScope::Job, + force: false, + }, + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); } } diff --git a/crates/environment-daemon/src/lib.rs b/crates/environment-daemon/src/lib.rs index 717fc7c14..53ff43904 100644 --- a/crates/environment-daemon/src/lib.rs +++ b/crates/environment-daemon/src/lib.rs @@ -1,4 +1,6 @@ pub mod config; +pub mod dispatch; +mod exec; pub mod filesystem; pub mod identity; pub mod jobs; @@ -6,7 +8,6 @@ pub mod process; mod process_group; mod redaction; pub mod registration; -pub mod rpc; pub mod server; pub mod upgrade; @@ -18,27 +19,59 @@ use std::{ time::Instant, }; +use anyhow::Context as _; use environment_protocol::{ - data::idle::IdleResponse, - shared::{ - CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentConnectionId, - ImplementationInfo, + ENVD_PROTOCOL_VERSION, + bootstrap::BuildFacts, + bytes::encode_hex, + data::{ + handshake::{ + DataCapabilities, DataLimits, FilesystemCapabilities, FilesystemLimits, GlobCeilings, + InitializeResponse, JobCapabilities, JobLimits, ProcessCapabilities, ProcessLimits, + ScanCeilings, SearchCeilings, TransferCeilings, + }, + idle::GetIdleResponse, }, + error::{Capability, ProtocolError}, + handshake::{ImplementationInfo, Plane}, + ids::DaemonInstanceId, + paths::EnvironmentPath, }; +use tokio_util::sync::CancellationToken; use crate::{ - config::DaemonConfig, filesystem::LocalFileSystem, jobs::JobManager, process::ProcessManager, + config::DaemonConfig, + filesystem::{Confinement, FilesystemConfig, LocalFileSystem}, + jobs::{JobConfig, JobManager}, + process::{ProcessConfig, ProcessManager}, }; +/// Product name in `serverInfo`, the registration `implementation` and +/// `--print-build`. +pub const DAEMON_NAME: &str = "lightspeed-envd"; + +/// Everything one daemon serves. Cheap to clone; every clone shares the +/// same managers, activity clock and shutdown token. #[derive(Clone)] pub struct DaemonRuntime { - config: Arc, - capabilities: EnvironmentCapabilities, + inner: Arc, +} + +struct RuntimeInner { + config: DaemonConfig, + instance_id: DaemonInstanceId, + server_info: ImplementationInfo, + capabilities: DataCapabilities, + limits: DataLimits, + default_cwd: EnvironmentPath, + home_directory: Option, + confinement: Arc, filesystem: LocalFileSystem, processes: ProcessManager, jobs: JobManager, - next_connection_id: Arc, - activity: Arc, + next_connection_id: AtomicU64, + activity: ActivityClock, + shutdown: CancellationToken, } /// Monotonic record of the last data-plane activity. Deliberately not a @@ -56,11 +89,17 @@ impl ActivityClock { } pub fn touch(&self) { - *self.last.lock().expect("activity clock poisoned") = Instant::now(); + *self + .last + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Instant::now(); } pub fn idle_for(&self) -> std::time::Duration { - self.last.lock().expect("activity clock poisoned").elapsed() + self.last + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .elapsed() } } @@ -80,304 +119,564 @@ pub fn install_crypto_provider() { } impl DaemonRuntime { - pub fn new(config: DaemonConfig) -> anyhow::Result { + /// Build the managers over one shared [`Confinement`]. A missing state + /// directory is created `0700`. + pub fn new(mut config: DaemonConfig) -> anyhow::Result { install_crypto_provider(); - let capabilities = environment_capabilities(&config); - let filesystem = LocalFileSystem::new( - config.fs_root.clone(), - config.cwd.clone(), - !config.read_only_fs, - ) - .with_transfer_state_dir(config.state_dir.join("transfers")) - .map_err(|e| anyhow::anyhow!("transfer state: {e:?}"))?; - let processes = ProcessManager::new(config.cwd.clone(), config.fs_root.clone()) - .with_scrubbed_env(config.scrubbed_env.clone()); - let jobs = JobManager::new( - config.cwd.clone(), - config.fs_root.clone(), - config.state_dir.clone(), - )? - .with_scrubbed_env(config.scrubbed_env.clone()); + config.state_dir = config::prepare_state_dir(&config.state_dir)?; + let default_cwd = environment_path(&config.cwd).context("cwd")?; + let home_directory = home_directory(&config); + let instance_id = mint_instance_id()?; + let capabilities = data_capabilities(&config); + let limits = data_limits(); + let confinement = Arc::new( + Confinement::new(config.fs_root.clone(), config.state_dir.clone()) + .map_err(|error| anyhow::anyhow!("filesystem confinement: {error}"))?, + ); + let filesystem = LocalFileSystem::new(FilesystemConfig { + confinement: confinement.clone(), + cwd: config.cwd.clone(), + writable: !config.read_only_fs, + limits: limits.filesystem.clone(), + }) + .map_err(|error| anyhow::anyhow!("filesystem: {error}"))?; + let processes = ProcessManager::new(ProcessConfig { + cwd: config.cwd.clone(), + confinement: confinement.clone(), + state_dir: config.state_dir.clone(), + scrubbed_env: config.scrubbed_env.clone(), + instance_id: instance_id.clone(), + limits: limits.process.clone(), + max_wait_ms: limits.max_wait_ms, + pty_supported: PTY_SUPPORTED, + }) + .context("process manager")?; + let jobs = JobManager::new(JobConfig { + cwd: config.cwd.clone(), + confinement: confinement.clone(), + state_dir: config.state_dir.clone(), + scrubbed_env: config.scrubbed_env.clone(), + instance_id: instance_id.clone(), + limits: limits.job.clone(), + max_wait_ms: limits.max_wait_ms, + }) + .context("job manager")?; Ok(Self { - config: Arc::new(config), - capabilities, - filesystem, - processes, - jobs, - next_connection_id: Arc::new(AtomicU64::new(1)), - activity: Arc::new(ActivityClock::new()), + inner: Arc::new(RuntimeInner { + config, + instance_id, + server_info: server_info(), + capabilities, + limits, + default_cwd, + home_directory, + confinement, + filesystem, + processes, + jobs, + next_connection_id: AtomicU64::new(1), + activity: ActivityClock::new(), + shutdown: CancellationToken::new(), + }), }) } /// Record one unit of data-plane activity. pub fn touch_activity(&self) { - self.activity.touch(); - } - - /// Idle report for the power reaper: zero idle time while any process or - /// job is still executing, otherwise the time since the last request. - /// Live work keeps the clock fresh so a long-running job that finishes - /// starts the idle countdown from its end, not its start. Leftover - /// process groups are counted but are not work: a server waiting for - /// requests must not keep the environment awake. - pub async fn idle_report(&self) -> IdleResponse { - let running_processes = self.processes.running_count().await; - let running_jobs = self.jobs.running_count().await; - let leftover_process_groups = self.processes.leftover_group_count(); - if running_processes > 0 || running_jobs > 0 { - self.activity.touch(); - return IdleResponse { - idle_for_ms: 0, - running_processes, - running_jobs, - leftover_process_groups, - }; + self.inner.activity.touch(); + } + + /// `env/getIdle`: zero idle time while any process or job is still + /// executing or any leftover process group has live members, otherwise + /// the time since the last `use` request. Live work keeps the clock + /// fresh so a long-running job that finishes starts the idle countdown + /// from its end, not its start. A leftover group is a service a command + /// started and left running, such as a dev server, so it holds the + /// environment awake like running work. + pub async fn idle_report(&self) -> GetIdleResponse { + let running_process_count = self.inner.processes.running_count().await; + let running_job_count = self.inner.jobs.running_count().await; + let leftover_process_group_count = self.inner.processes.leftover_group_count(); + let working = + running_process_count > 0 || running_job_count > 0 || leftover_process_group_count > 0; + if working { + self.inner.activity.touch(); + } + let idle_for_ms = if working { + 0 + } else { + u64::try_from(self.inner.activity.idle_for().as_millis()).unwrap_or(u64::MAX) + }; + GetIdleResponse { + idle_for_ms, + running_process_count, + running_job_count, + leftover_process_group_count, + } + } + + /// The data-plane `initialize` response; identical on every connection + /// of this daemon instance. + pub fn initialize_response(&self) -> InitializeResponse { + InitializeResponse { + protocol_version: ENVD_PROTOCOL_VERSION, + plane: Plane::Data, + daemon_instance_id: self.inner.instance_id.clone(), + server_info: self.inner.server_info.clone(), + capabilities: self.inner.capabilities.clone(), + limits: self.inner.limits.clone(), + default_cwd: self.inner.default_cwd.clone(), + home_directory: self.inner.home_directory.clone(), } - IdleResponse { - idle_for_ms: u64::try_from(self.activity.idle_for().as_millis()).unwrap_or(u64::MAX), - running_processes, - running_jobs, - leftover_process_groups, + } + + /// `capabilityUnavailable` unless this daemon serves `capability`. + pub fn require_capability(&self, capability: Capability) -> Result<(), ProtocolError> { + if dispatch::has_capability(&self.inner.capabilities, capability) { + return Ok(()); } + Err(ProtocolError::capability_unavailable( + capability, + format!("this daemon does not serve {}", capability_name(capability)), + )) } pub fn config(&self) -> &DaemonConfig { - &self.config + &self.inner.config } - pub fn implementation(&self) -> ImplementationInfo { - build_info().into() + pub fn instance_id(&self) -> &DaemonInstanceId { + &self.inner.instance_id } - pub fn capabilities(&self) -> EnvironmentCapabilities { - self.capabilities.clone() + pub fn server_info(&self) -> &ImplementationInfo { + &self.inner.server_info } - pub fn next_connection_id(&self) -> EnvironmentConnectionId { - let id = self.next_connection_id.fetch_add(1, Ordering::Relaxed); - EnvironmentConnectionId::new(format!("envd-{id}")) + pub fn capabilities(&self) -> &DataCapabilities { + &self.inner.capabilities + } + + pub fn limits(&self) -> &DataLimits { + &self.inner.limits + } + + pub fn confinement(&self) -> &Arc { + &self.inner.confinement + } + + /// Daemon-unique, increasing connection number. + pub fn next_connection_id(&self) -> u64 { + self.inner + .next_connection_id + .fetch_add(1, Ordering::Relaxed) + } + + /// Cancelled when the daemon begins a deliberate shutdown: listeners stop + /// accepting, data connections end and the registration channel says + /// goodbye. + pub fn shutdown(&self) -> &CancellationToken { + &self.inner.shutdown } pub fn filesystem(&self) -> &LocalFileSystem { - &self.filesystem + &self.inner.filesystem } pub fn processes(&self) -> &ProcessManager { - &self.processes + &self.inner.processes } pub fn jobs(&self) -> &JobManager { - &self.jobs + &self.inner.jobs + } +} + +/// PTYs are served wherever `portable-pty` has a Unix backend. +const PTY_SUPPORTED: bool = cfg!(unix); + +fn data_capabilities(config: &DaemonConfig) -> DataCapabilities { + DataCapabilities { + filesystem: Some(FilesystemCapabilities { + write: !config.read_only_fs, + }), + process: Some(ProcessCapabilities { pty: PTY_SUPPORTED }), + job: Some(JobCapabilities {}), } } -fn environment_capabilities(config: &DaemonConfig) -> EnvironmentCapabilities { - EnvironmentCapabilities { - filesystem_read: true, - filesystem_write: !config.read_only_fs, - filesystem_search: true, - filesystem_glob: true, - filesystem_ranged_read: true, - filesystem_transfer: cfg!(any(target_os = "linux", target_os = "macos")), - filesystem_scan: cfg!(any(target_os = "linux", target_os = "macos")), - filesystem_capture: cfg!(any(target_os = "linux", target_os = "macos")), - filesystem_materialize: cfg!(any(target_os = "linux", target_os = "macos")) - && !config.read_only_fs, - process_start: true, - process_stdin: true, - process_terminate: true, - process_output_polling: true, - process_output_notifications: false, - process_pty: true, - job_start: true, - job_list: true, - job_read: true, - job_cancel: true, - job_wait_hint: false, - job_dependencies: true, - job_queue_keys: true, - network: true, +const KIB: u64 = 1024; +const MIB: u64 = 1024 * KIB; +const GIB: u64 = 1024 * MIB; +const TIB: u64 = 1024 * GIB; + +/// The ceilings envd enforces and advertises. Every response they allow +/// fits below `MAX_MESSAGE_BYTES` after base64. +pub fn data_limits() -> DataLimits { + DataLimits { + max_concurrent_requests: 32, + max_wait_ms: 1_800_000, + process: ProcessLimits { + max_read_bytes: 4 * MIB, + max_retained_output_bytes: MIB, + retention_after_exit_ms: 60_000, + unread_retention_ms: 7_200_000, + max_exited_retained_bytes: 64 * MIB, + terminate_grace_ms: 5_000, + max_input_queue_bytes: MIB, + }, + job: JobLimits { + max_read_bytes: 4 * MIB, + max_request_jobs: 64, + max_retained_output_bytes: 2 * MIB, + record_retention_ms: 604_800_000, + cancel_grace_ms: 5_000, + }, + filesystem: FilesystemLimits { + max_read_file_bytes: 8 * MIB, + max_write_file_bytes: 8 * MIB, + max_directory_entries: 1_000, + search: SearchCeilings { + max_matches: 10_000, + max_files: 100_000, + max_bytes: GIB, + max_duration_ms: 60_000, + max_line_bytes: 4_096, + }, + glob: GlobCeilings { + max_matches: 100_000, + max_files: 1_000_000, + max_duration_ms: 60_000, + }, + scan: ScanCeilings { + max_roots: 32, + max_patterns: 32, + max_entries: 10_000, + max_file_bytes: 256 * KIB, + max_total_bytes: 4 * MIB, + max_duration_ms: 30_000, + }, + transfer: TransferCeilings { + max_active_operations: 8, + max_content_chunk_bytes: 4 * MIB, + max_inventory_page_entries: 1_000, + receipt_retention_ms: 86_400_000, + max_entries: 100_000, + max_depth: 64, + max_file_bytes: TIB, + max_total_bytes: TIB, + max_manifest_bytes: 32 * MIB, + max_duration_ms: 86_400_000, + }, + }, } } -pub fn protocol_version() -> u32 { - CURRENT_PROTOCOL_VERSION +fn capability_name(capability: Capability) -> &'static str { + match capability { + Capability::Filesystem => "the filesystem family", + Capability::FilesystemWrite => "filesystem writes", + Capability::Process => "the process family", + Capability::ProcessPty => "process PTYs", + Capability::Job => "the job family", + Capability::Power => "power control", + Capability::Ingress => "ingress", + } } -/// The facts this binary reports about itself: to the gateway in every -/// handshake, and to an operator through `--print-build`. The protocol -/// version is the one that decides whether a gateway admits the daemon; the -/// rest is provenance. -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] -#[serde(rename_all = "camelCase")] -pub struct BuildInfo { - pub name: &'static str, - pub version: &'static str, - pub git_sha: &'static str, - pub target: &'static str, - pub protocol_version: u32, +/// `daemonInstanceId`: `envd_` and 32 random hex digits, minted per start. +fn mint_instance_id() -> anyhow::Result { + let mut bytes = [0u8; 16]; + rand::TryRng::try_fill_bytes(&mut rand::rngs::SysRng, &mut bytes) + .context("generate daemon instance id")?; + Ok(DaemonInstanceId::new(format!( + "envd_{}", + encode_hex(&bytes) + ))?) } -pub fn build_info() -> BuildInfo { - BuildInfo { - name: "lightspeed-envd", - version: release_info::VERSION, - git_sha: release_info::GIT_SHA, - target: release_info::TARGET, - protocol_version: CURRENT_PROTOCOL_VERSION, - } +fn environment_path(path: &std::path::Path) -> anyhow::Result { + let text = path + .to_str() + .with_context(|| format!("{} is not UTF-8", path.display()))?; + Ok(EnvironmentPath::new(text)?) } -impl From for ImplementationInfo { - fn from(build: BuildInfo) -> Self { - ImplementationInfo { - name: build.name.to_owned(), - version: Some(build.version.to_owned()), - git_sha: Some(build.git_sha.to_owned()), - target: Some(build.target.to_owned()), - } +/// `$HOME` when it is a valid path inside the filesystem root. +fn home_directory(config: &DaemonConfig) -> Option { + let home = std::env::var_os("HOME")?; + let home = std::path::PathBuf::from(home); + if !home.starts_with(&config.fs_root) { + return None; } + environment_path(&home).ok() } -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use environment_protocol::{ - data::process::{ - ProcessSignal, ReadProcessParams, StartProcessParams, TerminateProcessParams, - }, - shared::ProcessId, +/// `serverInfo` and the registration `implementation`: this build's facts, +/// leaving out any the build left empty. +pub fn server_info() -> ImplementationInfo { + let info = ImplementationInfo::new(DAEMON_NAME).expect("the daemon name is a valid field"); + let info = match info.clone().with_version(release_info::VERSION) { + Ok(with) => with, + Err(_) => info, + }; + let info = match info.clone().with_git_sha(release_info::GIT_SHA) { + Ok(with) => with, + Err(_) => info, }; + match info.clone().with_target(release_info::TARGET) { + Ok(with) => with, + Err(_) => info, + } +} + +/// What `--print-build` prints: the facts an orchestrator checks a +/// downloaded daemon against the deployment's discovery document with. +pub fn build_facts() -> BuildFacts { + BuildFacts { + name: DAEMON_NAME.to_owned(), + version: release_info::VERSION.to_owned(), + git_sha: release_info::GIT_SHA.to_owned(), + target: release_info::TARGET.to_owned(), + protocol_version: ENVD_PROTOCOL_VERSION, + } +} +#[cfg(test)] +pub(crate) mod test_support { use super::*; - fn runtime() -> (tempfile::TempDir, DaemonRuntime) { + /// A runtime rooted in a fresh temporary directory, its state dir + /// beside the root rather than inside it. + pub(crate) fn runtime() -> (tempfile::TempDir, DaemonRuntime) { + runtime_with(|_| {}) + } + + pub(crate) fn runtime_with( + adjust: impl FnOnce(&mut DaemonConfig), + ) -> (tempfile::TempDir, DaemonRuntime) { let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let runtime = DaemonRuntime::new(DaemonConfig { - listen: Some("127.0.0.1:0".parse().expect("listen")), + let base = temp.path().canonicalize().expect("canonical temp"); + let root = base.join("root"); + let state = base.join("state"); + std::fs::create_dir_all(&root).expect("root"); + std::fs::create_dir_all(&state).expect("state"); + let mut config = DaemonConfig { + listen: None, cwd: root.clone(), - fs_root: root.clone(), - state_dir: root.join("state"), + fs_root: root, + state_dir: state, read_only_fs: false, registration: None, scrubbed_env: vec!["LIGHTSPEED_ENVD_REGISTRATION_KEY".to_owned()], - }) - .expect("runtime"); + }; + adjust(&mut config); + let runtime = DaemonRuntime::new(config).expect("runtime"); (temp, runtime) } +} + +#[cfg(test)] +mod tests { + use environment_protocol::{ + data::handshake::DataLimits, + error::ProtocolErrorKind, + limits::{MAX_MESSAGE_BYTES, MIN_CONCURRENT_REQUESTS}, + manifest::{EnvGetIdle, ProcessStart, ProcessTerminate}, + }; + use serde_json::json; + + use super::{test_support::runtime, *}; + use crate::dispatch::{RequestContext, handle_request}; + + async fn call( + runtime: &DaemonRuntime, + method: &str, + params: serde_json::Value, + ) -> serde_json::Value { + handle_request(runtime, &RequestContext::detached(), method, params) + .await + .unwrap_or_else(|error| panic!("{method}: {error}")) + .value + } + + #[test] + fn instance_ids_are_random_per_start_and_use_the_general_grammar() { + let first = mint_instance_id().unwrap(); + let second = mint_instance_id().unwrap(); + assert_ne!(first, second); + assert!(first.as_str().starts_with("envd_")); + assert_eq!(first.as_str().len(), "envd_".len() + 32); + } + + /// Every advertised ceiling keeps its worst-case response, base64 + /// included, below the message limit. + #[test] + fn advertised_payload_ceilings_fit_the_message_limit() { + let limits: DataLimits = data_limits(); + let base64 = |bytes: u64| bytes.div_ceil(3) * 4; + for (name, bytes) in [ + ("fs.readFile", limits.filesystem.max_read_file_bytes), + ("fs.writeFile", limits.filesystem.max_write_file_bytes), + ( + "transfer.chunk", + limits.filesystem.transfer.max_content_chunk_bytes, + ), + ("process.read", limits.process.max_read_bytes), + ("job.read", limits.job.max_read_bytes), + ] { + assert!(base64(bytes) < MAX_MESSAGE_BYTES * 3 / 4, "{name}"); + } + assert!(limits.max_concurrent_requests >= MIN_CONCURRENT_REQUESTS); + } #[tokio::test(flavor = "current_thread")] - async fn idle_report_counts_running_work_and_resets_on_activity() { + async fn initialize_reports_the_instance_cwd_and_capability_families() { let (_temp, runtime) = runtime(); - tokio::time::sleep(std::time::Duration::from_millis(30)).await; - let quiet = runtime.idle_report().await; - assert!(quiet.is_quiescent()); - assert!(quiet.idle_for_ms >= 30, "idle_for_ms={}", quiet.idle_for_ms); - - runtime.touch_activity(); - let touched = runtime.idle_report().await; - assert!( - touched.idle_for_ms < 30, - "idle_for_ms={}", - touched.idle_for_ms + let response = runtime.initialize_response(); + assert_eq!(response.protocol_version, ENVD_PROTOCOL_VERSION); + assert_eq!(response.plane, Plane::Data); + assert_eq!(&response.daemon_instance_id, runtime.instance_id()); + assert_eq!( + response.default_cwd.as_str(), + runtime.config().cwd.to_str().unwrap() ); + let filesystem = response.capabilities.filesystem.expect("filesystem"); + assert!(filesystem.write); + assert!(response.capabilities.process.is_some()); + assert!(response.capabilities.job.is_some()); + assert_eq!(response.limits.job.max_request_jobs, 64); + } - let process_id = ProcessId::new("sleeper"); - runtime - .processes() - .start_process(StartProcessParams { - process_id: process_id.clone(), - argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), "sleep 30".to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(60_000), - tty: false, - }) - .await - .expect("start"); + #[tokio::test(flavor = "current_thread")] + async fn a_read_only_daemon_refuses_writes_before_any_side_effect() { + let (_temp, runtime) = super::test_support::runtime_with(|config| { + config.read_only_fs = true; + }); + let path = runtime.config().cwd.join("written"); + let error = handle_request( + &runtime, + &RequestContext::detached(), + "fs/writeFile", + json!({"path": path.to_str().unwrap(), "data": "aGk="}), + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::CapabilityUnavailable); + assert!(!path.exists()); + } + + #[tokio::test(flavor = "current_thread")] + async fn idle_report_holds_zero_while_a_process_runs_and_resets_on_use() { + let (_temp, runtime) = runtime(); + tokio::time::sleep(std::time::Duration::from_millis(30)).await; + let quiet: GetIdleResponse = + serde_json::from_value(call(&runtime, EnvGetIdle::NAME, json!({})).await).unwrap(); + assert!(quiet.idle_for_ms >= 30, "idle_for_ms={}", quiet.idle_for_ms); + assert_eq!(quiet.running_process_count, 0); + + call( + &runtime, + ProcessStart::NAME, + json!({ + "processId": "sleeper", + "command": {"argv": ["/bin/sh", "-c", "sleep 30"], "timeoutMs": 60000}, + }), + ) + .await; let busy = runtime.idle_report().await; - assert_eq!(busy.running_processes, 1); + assert_eq!(busy.running_process_count, 1); assert_eq!(busy.idle_for_ms, 0); - assert!(!busy.is_quiescent()); - - runtime - .processes() - .terminate_process(TerminateProcessParams { - process_id, - signal: ProcessSignal::Kill, - }) - .await - .expect("terminate"); + + call( + &runtime, + ProcessTerminate::NAME, + json!({"processId": "sleeper", "signal": "kill"}), + ) + .await; for _ in 0..100 { - if runtime.idle_report().await.running_processes == 0 { + if runtime.idle_report().await.running_process_count == 0 { break; } tokio::time::sleep(std::time::Duration::from_millis(20)).await; } let after = runtime.idle_report().await; - assert_eq!(after.running_processes, 0); + assert_eq!(after.running_process_count, 0); assert!(after.idle_for_ms < 5_000); } #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn idle_report_counts_leftover_groups_without_treating_them_as_busy() { + async fn a_leftover_process_group_holds_the_idle_clock_like_running_work() { let (_temp, runtime) = runtime(); - let process_id = ProcessId::new("service"); - runtime - .processes() - .start_process(StartProcessParams { - process_id: process_id.clone(), - argv: vec![ - "/bin/sh".to_owned(), - "-c".to_owned(), - "nohup sleep 30 >/dev/null 2>&1 & exit 0".to_owned(), - ], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(60_000), - tty: false, - }) - .await - .expect("start"); - let output = runtime - .processes() - .read_process(ReadProcessParams { - process_id: process_id.clone(), - after_seq: None, - max_bytes: None, - wait_ms: None, - }) - .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.leftover_processes.len(), 1); - - let report = runtime.idle_report().await; - assert_eq!(report.running_processes, 0); - assert_eq!(report.leftover_process_groups, 1); - assert!(report.is_quiescent(), "leftovers are not running work"); - - runtime - .processes() - .terminate_process(TerminateProcessParams { - process_id, - signal: ProcessSignal::Kill, - }) - .await - .expect("kill leftovers"); - for _ in 0..100 { - if runtime.idle_report().await.leftover_process_groups == 0 { + call( + &runtime, + ProcessStart::NAME, + json!({ + "processId": "service", + "command": { + "argv": ["/bin/sh", "-c", "nohup sleep 30 >/dev/null 2>&1 & echo $! > leftover.pid; exit 0"], + "timeoutMs": 60000, + }, + }), + ) + .await; + let mut held = runtime.idle_report().await; + for _ in 0..250 { + if held.running_process_count == 0 && held.leftover_process_group_count == 1 { break; } tokio::time::sleep(std::time::Duration::from_millis(20)).await; + held = runtime.idle_report().await; } - assert_eq!(runtime.idle_report().await.leftover_process_groups, 0); + assert_eq!(held.running_process_count, 0, "the root exited"); + assert_eq!(held.leftover_process_group_count, 1); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert_eq!( + runtime.idle_report().await.idle_for_ms, + 0, + "a leftover service keeps the environment awake" + ); + + let pid: i32 = std::fs::read_to_string(runtime.config().cwd.join("leftover.pid")) + .expect("pid file") + .trim() + .parse() + .expect("pid"); + let _ = rustix::process::kill_process( + rustix::process::Pid::from_raw(pid).expect("pid"), + rustix::process::Signal::KILL, + ); + let mut released = runtime.idle_report().await; + for _ in 0..250 { + if released.leftover_process_group_count == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + released = runtime.idle_report().await; + } + assert_eq!(released.leftover_process_group_count, 0); + tokio::time::sleep(std::time::Duration::from_millis(30)).await; + let idle = runtime.idle_report().await.idle_for_ms; + assert!( + (30..5_000).contains(&idle), + "the clock restarts when the last leftover ends: idle_for_ms={idle}" + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn observation_methods_leave_the_idle_clock_alone() { + let (_temp, runtime) = runtime(); + tokio::time::sleep(std::time::Duration::from_millis(40)).await; + call(&runtime, EnvGetIdle::NAME, json!({})).await; + let cwd = runtime.config().cwd.to_str().unwrap().to_owned(); + let _ = handle_request( + &runtime, + &RequestContext::detached(), + "fs/scan", + json!({"roots": [{"path": cwd, "maxDepth": 0}]}), + ) + .await; + assert!(runtime.idle_report().await.idle_for_ms >= 40); } } diff --git a/crates/environment-daemon/src/main.rs b/crates/environment-daemon/src/main.rs index 1cb0146a7..d38e4a4d0 100644 --- a/crates/environment-daemon/src/main.rs +++ b/crates/environment-daemon/src/main.rs @@ -1,33 +1,102 @@ +use std::process::ExitCode; + use clap::Parser as _; use environment_daemon::{ DaemonRuntime, config::{DaemonArgs, DaemonCommand}, - server, upgrade, + server::{self, DaemonExit}, + upgrade, }; -#[tokio::main(flavor = "current_thread")] -async fn main() -> anyhow::Result<()> { +/// Worker threads of the async runtime. Blocking work runs on the blocking +/// pool; the workers keep connections, heartbeats and idle reports moving +/// even if one of them is held up. +const WORKER_THREADS: usize = 2; + +fn main() -> ExitCode { + // Configuration is read, and secret-bearing variables are removed from + // the process environment, before the async runtime starts any thread. let _ = dotenvy::dotenv(); let args = DaemonArgs::parse(); if args.print_build { - println!( - "{}", - serde_json::to_string(&environment_daemon::build_info())? - ); - return Ok(()); + return match serde_json::to_string(&environment_daemon::build_facts()) { + Ok(json) => { + println!("{json}"); + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("lightspeed-envd: {error}"); + ExitCode::FAILURE + } + }; } if args.command == Some(DaemonCommand::Upgrade) { - let installed = upgrade::install(args.upgrade_request()?).await?; - println!( - "installed lightspeed-envd {} ({}, protocol {}) at {}", - installed.version, - installed.git_sha, - installed.protocol_version, - installed.path.display() - ); - return Ok(()); + return upgrade_command(&args); + } + let exit = match args.into_config() { + Ok(config) => serve(config), + Err(error) => DaemonExit::Configuration(error), + }; + if !matches!(exit, DaemonExit::Shutdown) { + eprintln!("lightspeed-envd {exit}"); + } + ExitCode::from(exit.status()) +} + +fn serve(config: environment_daemon::config::DaemonConfig) -> DaemonExit { + let runtime = match async_runtime() { + Ok(runtime) => runtime, + Err(error) => return DaemonExit::Failed(error.into()), + }; + runtime.block_on(async move { + let daemon = match DaemonRuntime::new(config) { + Ok(daemon) => daemon, + Err(error) => return DaemonExit::Configuration(error), + }; + if let Err(error) = server::spawn_signal_listener(&daemon) { + return DaemonExit::Failed(error); + } + server::run(daemon).await + }) +} + +fn upgrade_command(args: &DaemonArgs) -> ExitCode { + let request = match args.upgrade_request() { + Ok(request) => request, + Err(error) => { + eprintln!("lightspeed-envd upgrade: {error:#}"); + return ExitCode::from(2); + } + }; + let installed = match async_runtime() { + Ok(runtime) => runtime.block_on(upgrade::install(request)), + Err(error) => { + eprintln!("lightspeed-envd upgrade: {error}"); + return ExitCode::FAILURE; + } + }; + match installed { + Ok(installed) => { + println!( + "installed lightspeed-envd {} ({}, protocol {}) at {}", + installed.version, + installed.git_sha, + installed.protocol_version, + installed.path.display() + ); + ExitCode::SUCCESS + } + Err(error) => { + eprintln!("lightspeed-envd upgrade: {error}"); + ExitCode::FAILURE + } } - let config = args.into_config()?; - let runtime = DaemonRuntime::new(config)?; - server::run(runtime).await +} + +fn async_runtime() -> std::io::Result { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(WORKER_THREADS) + .thread_name("lightspeed-envd") + .enable_all() + .build() } diff --git a/crates/environment-daemon/src/process.rs b/crates/environment-daemon/src/process.rs index f1f8aa307..4979890e5 100644 --- a/crates/environment-daemon/src/process.rs +++ b/crates/environment-daemon/src/process.rs @@ -1,1970 +1,1332 @@ -//! Interactive process execution. +//! `process/*`: interactive processes on the shared execution primitive. //! //! Every command starts as the leader of its own process group. Its natural //! exit never sweeps the group: whatever the command left running keeps the -//! environment's lifetime, is sampled once at exit and reported to the -//! caller, and keeps somewhere to write because the output readers stay on -//! the pipes until end of file. Only a timeout, an explicit kill, or the -//! daemon's own cancellation paths kill the group. +//! environment's lifetime and is reported as leftovers, sampled when a read +//! returns the settled exit. Only a timeout or an explicit terminate or kill +//! signals the group. //! -//! Output is delivered through a daemon-owned cursor so a handle used from a -//! later connection continues where the previous read stopped, and unread -//! output is retained as a bounded head-and-tail buffer. +//! Output is one byte stream per process, read through a daemon-owned +//! cursor that advances only after a response was written, or at explicit +//! offsets that never consume. Retention is bounded per process, per exited +//! process and across exited processes; an evicted handle leaves a +//! tombstone that answers `notFound {expired}`. Spawn records let a +//! restarted daemon adopt the groups an earlier instance left running and +//! answer their handles with `notFound {daemonRestarted}`. use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, - io::{Read as _, Write as _}, - path::{Component, PathBuf}, - process::Stdio, - sync::{Arc, Mutex as StdMutex}, - time::Duration, + collections::{BTreeMap, VecDeque}, + fmt::Write as _, + path::PathBuf, + sync::{Arc, Mutex, MutexGuard}, + time::{Duration, Instant}, }; use environment_protocol::{ - data::process::{ - LeftoverProcess, ProcessOutputChunk, ProcessOutputStream, ProcessSignal, - ProcessTermination, ReadProcessParams, ReadProcessResponse, ResizeProcessParams, - ResizeProcessResponse, StartProcessParams, StartProcessResponse, TerminateProcessParams, - TerminateProcessResponse, WriteProcessParams, WriteProcessResponse, WriteProcessStatus, + data::{ + exec::{ + ExecInput, ExecState, LeftoverDisposition, OutputPage, OutputStream, PtySize, ReturnOn, + StoppedBy, + }, + handshake::ProcessLimits, + process::{ + ProcessSignal, ReadProcessParams, ReadProcessResponse, StartProcessParams, + StartProcessResponse, TerminateProcessParams, TerminateProcessResponse, + WriteProcessParams, WriteProcessResponse, + }, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::{ByteChunk, EnvironmentPath, ProcessId}, -}; -use portable_pty::{CommandBuilder, MasterPty, PtySize, native_pty_system}; -use tokio::{ - io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, - process::{Child, Command}, - sync::{Mutex, Notify}, - task::JoinHandle, - time::Instant, + error::{Capability, NotFoundReason, ProtocolError}, + ids::{DaemonInstanceId, ProcessId}, + limits::MAX_PAGE_CHUNKS, }; +use serde::{Deserialize, Serialize}; use crate::{ - process_group, - redaction::{redact_bytes, redactions_for_secret_env}, + dispatch::{Reply, RequestContext}, + exec::{ + AfterExit, ExecOptions, Execution, OutputSink, PreparedCommand, + log::OutputLog, + outcome::describe_exit, + spawn_record::{SpawnRecord, write_json_atomic}, + validate_command, + }, + filesystem::Confinement, + process_group::{self, Group}, }; -/// How long a finished process entry stays readable after the first read -/// that observed its exit. -#[cfg(not(test))] -pub(crate) const TERMINAL_PROCESS_RETENTION: Duration = Duration::from_secs(60); -#[cfg(test)] -pub(crate) const TERMINAL_PROCESS_RETENTION: Duration = Duration::from_millis(10); - -/// How often the exit watcher polls a running child. Readers only notify on -/// pipe events, and leftovers holding the pipes can suppress EOF, so exit -/// observation must not depend on them. -#[cfg(not(test))] -const EXIT_POLL_INTERVAL: Duration = Duration::from_millis(100); -#[cfg(test)] -const EXIT_POLL_INTERVAL: Duration = Duration::from_millis(10); - -/// Unread output retained per process. Beyond it the middle is dropped and -/// the next read reports how many bytes went missing; the oldest half and -/// the newest chunks are kept. -#[cfg(not(test))] -const RETAINED_OUTPUT_CAP: usize = 1024 * 1024; -#[cfg(test)] -const RETAINED_OUTPUT_CAP: usize = 16 * 1024; - -const READ_BUFFER_BYTES: usize = 8192; +/// How long a tombstone answers `notFound {expired}`. +const TOMBSTONE_RETENTION: Duration = Duration::from_secs(24 * 60 * 60); + +/// Most tombstones kept; the oldest go first. +const MAX_TOMBSTONES: usize = 4096; + +/// Everything a [`ProcessManager`] needs from the daemon. +pub struct ProcessConfig { + /// Canonical default `cwd`. + pub cwd: PathBuf, + /// Resolves every `cwd`. + pub confinement: Arc, + /// Spawn records live in `/processes`. + pub state_dir: PathBuf, + /// Daemon variables no child inherits. + pub scrubbed_env: Vec, + pub instance_id: DaemonInstanceId, + pub limits: ProcessLimits, + /// `limits.maxWaitMs`, the `waitMs` ceiling. + pub max_wait_ms: u64, + /// `capabilities.process.pty`. + pub pty_supported: bool, +} +/// The daemon's processes. Cheap to clone. #[derive(Clone)] pub struct ProcessManager { - cwd: PathBuf, - fs_root: PathBuf, - processes: Arc>>>, - /// Daemon configuration variables no child may inherit. - scrubbed_env: Arc>, - /// Process groups whose root exited while members were still alive. - /// Pruned lazily once the group is empty. - leftover_groups: Arc>>, -} - -struct ProcessEntry { - state: Mutex, - notify: Notify, - readers: Mutex>>, + inner: Arc, } -struct ProcessState { - child: ProcessChild, - pid: Option, - pgid: Option, - stdin: Option, - pty_master: Option>, - redactions: Vec>, - output: OutputBuffer, - /// Output readers still attached to a pipe or PTY. - readers_open: u32, - exited: bool, - exit_observed_at: Option, - exit_code: Option, - termination: Option, - leftover_processes: Vec, - failure: Option, - cleanup_scheduled: bool, - /// The root exited and the drain grace passed while leftovers still held - /// the pipes: readers keep reading so nothing blocks the leftovers, but - /// the bytes are dropped. - discarding: bool, +struct ManagerInner { + config: ProcessConfig, + records_dir: PathBuf, + table: Mutex, + /// Groups no live entry owns any more that still had members when last + /// looked at: groups of evicted entries and groups adopted from an + /// earlier daemon instance. Each remembers its spawn record, deleted + /// once the group is empty. + leftover_groups: Mutex>, } -enum ProcessChild { - Pipe(Child), - Pty(Box), +#[derive(Default)] +struct Table { + slots: BTreeMap, + /// Tombstoned ids in eviction order. + tombstones: VecDeque<(ProcessId, Instant)>, } -enum ProcessInput { - Pty(Box), +enum Slot { + /// A start is spawning under this id. + Reserved, + Live(Arc), + /// Evicted; remembers how it ended. + Tombstone(String), + /// A handle of an earlier daemon instance. + Restarted, } -/// Retained output between the cursor and the newest chunk. -#[derive(Default)] -struct OutputBuffer { - chunks: VecDeque, - retained_bytes: usize, - next_seq: u64, - delivered_seq: u64, +struct Entry { + execution: Execution, + record_path: PathBuf, + /// When a reply returning the settled exit with nothing left unread was + /// written. + fully_read_at: Mutex>, } -struct RetainedChunk { - chunk: ProcessOutputChunk, - /// Bytes dropped from the buffer immediately before this chunk. - omitted_before: u64, +/// A process's retained output and its daemon cursor. +struct ProcessOutput { + log: OutputLog, + cursor: u64, + cap: u64, } -impl OutputBuffer { - fn push(&mut self, stream: ProcessOutputStream, bytes: Vec) { - let seq = self.next_seq; - self.next_seq += 1; - self.retained_bytes += bytes.len(); - self.chunks.push_back(RetainedChunk { - chunk: ProcessOutputChunk { - seq, - stream, - chunk: ByteChunk::from(bytes), - }, - omitted_before: 0, - }); - self.enforce_cap(); +impl OutputSink for ProcessOutput { + fn append(&mut self, stream: OutputStream, data: &[u8]) { + self.log.append(stream, data); + self.log.enforce_cap(self.cap, self.cursor); } +} - fn unread_bytes(&self) -> usize { - self.retained_bytes - } +/// The file of one spawn record. +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct StoredSpawn { + process_id: ProcessId, + spawn: SpawnRecord, +} - /// Drops chunks from just past the head until the buffer fits. The head - /// is the oldest half of the cap; the newest chunk is never dropped. - fn enforce_cap(&mut self) { - if self.retained_bytes <= RETAINED_OUTPUT_CAP { - return; - } - let head_cap = RETAINED_OUTPUT_CAP / 2; - let mut head_bytes = 0usize; - let mut head_len = 0usize; - for retained in &self.chunks { - // The head never grows past an omission: the chunk that follows - // dropped bytes stays the boundary, so every later drop extends - // the same gap instead of opening a second one. Pipe reads on - // Linux deliver uneven chunks, where a small survivor would - // otherwise be absorbed into the head. - if retained.omitted_before > 0 { - break; - } - let len = retained.chunk.chunk.as_slice().len(); - if head_bytes + len > head_cap { - break; - } - head_bytes += len; - head_len += 1; - } - let mut omitted = 0u64; - while self.retained_bytes > RETAINED_OUTPUT_CAP && self.chunks.len() > head_len + 1 { - let Some(dropped) = self.chunks.remove(head_len) else { - break; +impl ProcessManager { + /// Reconciles the spawn records an earlier instance left: groups of + /// this boot that still run are adopted as leftovers, and their ids + /// answer `notFound {daemonRestarted}`; every other record is deleted. + pub fn new(config: ProcessConfig) -> anyhow::Result { + let records_dir = config.state_dir.join("processes"); + std::fs::create_dir_all(&records_dir)?; + let boot = process_group::boot_id(); + let mut table = Table::default(); + let mut leftover_groups = BTreeMap::new(); + for entry in std::fs::read_dir(&records_dir)? { + let path = entry?.path(); + let stored = std::fs::read(&path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let Some(stored) = stored else { + eprintln!( + "lightspeed-envd: removing unreadable spawn record {}", + path.display() + ); + let _ = std::fs::remove_file(&path); + continue; }; - let len = dropped.chunk.chunk.as_slice().len(); - self.retained_bytes -= len; - omitted += len as u64 + dropped.omitted_before; - } - if omitted > 0 - && let Some(next) = self.chunks.get_mut(head_len) - { - next.omitted_before += omitted; - } - } - - /// Delivers from the cursor, advancing it. Whole chunks are taken up to - /// `max_bytes`; when the very first chunk alone exceeds the budget it is - /// split and its remainder stays at the cursor under the same sequence. - fn deliver(&mut self, max_bytes: Option) -> (Vec, u64) { - let mut delivered = Vec::new(); - let mut omitted = 0u64; - let mut bytes = 0usize; - while let Some(front) = self.chunks.front_mut() { - let len = front.chunk.chunk.as_slice().len(); - if let Some(max_bytes) = max_bytes - && bytes + len > max_bytes - { - if delivered.is_empty() && max_bytes > 0 { - let (head, rest) = front.chunk.chunk.as_slice().split_at(max_bytes); - delivered.push(ProcessOutputChunk { - seq: front.chunk.seq, - stream: front.chunk.stream, - chunk: ByteChunk::from(head.to_vec()), - }); - omitted += front.omitted_before; - front.omitted_before = 0; - front.chunk.chunk = ByteChunk::from(rest.to_vec()); - self.retained_bytes -= max_bytes; - } - break; + if !stored.spawn.same_boot(boot.as_deref()) { + let _ = std::fs::remove_file(&path); + continue; } - let retained = self.chunks.pop_front().expect("front chunk exists"); - self.retained_bytes -= len; - bytes += len; - omitted += retained.omitted_before; - delivered.push(retained.chunk); - } - self.delivered_seq = self - .chunks - .front() - .map_or(self.next_seq, |front| front.chunk.seq); - (delivered, omitted) - } - - /// Re-reads retained chunks from a sequence number without moving the - /// cursor. Best effort: delivered chunks are gone. - fn reread(&self, after_seq: u64, max_bytes: Option) -> (Vec, u64) { - let mut chunks = Vec::new(); - let mut omitted = 0u64; - let mut bytes = 0usize; - for retained in self - .chunks - .iter() - .filter(|retained| retained.chunk.seq >= after_seq) - { - let slice = retained.chunk.chunk.as_slice(); - if let Some(max_bytes) = max_bytes { - if bytes >= max_bytes { - break; - } - let remaining = max_bytes - bytes; - if slice.len() > remaining { - chunks.push(ProcessOutputChunk { - seq: retained.chunk.seq, - stream: retained.chunk.stream, - chunk: ByteChunk::from(slice[..remaining].to_vec()), - }); - omitted += retained.omitted_before; - break; - } + table.slots.insert(stored.process_id, Slot::Restarted); + if stored.spawn.group_survives() { + leftover_groups.insert(stored.spawn.pgid, (stored.spawn.group(), path)); + } else { + let _ = std::fs::remove_file(&path); } - bytes += slice.len(); - omitted += retained.omitted_before; - chunks.push(retained.chunk.clone()); - } - (chunks, omitted) - } -} - -impl ProcessManager { - pub fn new(cwd: PathBuf, fs_root: PathBuf) -> Self { - Self { - cwd: normalize_path(cwd), - fs_root: normalize_path(fs_root), - processes: Arc::new(Mutex::new(BTreeMap::new())), - scrubbed_env: Arc::new(Vec::new()), - leftover_groups: Arc::new(StdMutex::new(BTreeSet::new())), } + Ok(Self { + inner: Arc::new(ManagerInner { + config, + records_dir, + table: Mutex::new(table), + leftover_groups: Mutex::new(leftover_groups), + }), + }) } - pub fn with_scrubbed_env(mut self, names: Vec) -> Self { - self.scrubbed_env = Arc::new(names); - self - } - - /// Processes started here that have not exited yet. + /// Processes whose root has not exited. pub async fn running_count(&self) -> u32 { - let entries: Vec> = - self.processes.lock().await.values().cloned().collect(); - let mut running = 0u32; - for entry in entries { - if !entry.state.lock().await.exited { - running += 1; - } - } - running + self.maintain(); + let table = self.table(); + let running = table + .slots + .values() + .filter(|slot| match slot { + Slot::Live(entry) => !entry.execution.with_state(|state| state.exited()), + _ => false, + }) + .count(); + u32::try_from(running).unwrap_or(u32::MAX) } - /// Process groups of finished commands that still have a live member. - /// Leftovers are not running work: a service waiting for requests is - /// idle, so this never feeds the quiescence check. + /// Process groups whose root exited while members still run, adopted + /// groups of an earlier instance included. They hold the idle clock + /// like running work. pub fn leftover_group_count(&self) -> u32 { - let mut groups = self - .leftover_groups - .lock() - .expect("leftover groups poisoned"); - groups.retain(|pgid| process_group::group_alive(*pgid)); - u32::try_from(groups.len()).unwrap_or(u32::MAX) + let live = { + let table = self.table(); + table + .slots + .values() + .filter(|slot| match slot { + Slot::Live(entry) => { + entry.execution.with_state(|state| state.exited()) + && entry.execution.group_alive() + } + _ => false, + }) + .count() + }; + let mut groups = lock(&self.inner.leftover_groups); + groups.retain(|_, (group, record)| { + let alive = group.alive(); + if !alive { + let _ = std::fs::remove_file(record); + } + alive + }); + u32::try_from(live + groups.len()).unwrap_or(u32::MAX) } - pub async fn start_process( + pub async fn start( &self, + _ctx: &RequestContext, params: StartProcessParams, - ) -> Result { - if params.argv.is_empty() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "process argv must not be empty", - )); - } - let cwd = params - .cwd - .as_ref() - .map(|path| self.resolve_cwd(path)) - .transpose()? - .unwrap_or_else(|| self.cwd.clone()); - for name in params.secret_env.keys() { - if params.env.contains_key(name) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - format!("process env collides with secret env: {name}"), + ) -> Result { + let StartProcessParams { + process_id, + command, + pty, + } = params; + let config = &self.inner.config; + validate_command(&command, "command")?; + if let Some(size) = pty { + validate_pty(size, &command.input)?; + if !config.pty_supported { + return Err(ProtocolError::capability_unavailable( + Capability::ProcessPty, + "this daemon cannot run processes under a PTY", )); } } + let cwd = match &command.cwd { + Some(path) => config.confinement.resolve_directory(path)?, + None => config.cwd.clone(), + }; + self.maintain(); + self.reserve(&process_id)?; - if params.tty { - return self.start_pty_process(params, cwd).await; - } - - let mut command = Command::new(¶ms.argv[0]); - command - .args(¶ms.argv[1..]) - .current_dir(&cwd) - .envs(params.env.iter()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - for name in self.scrubbed_env.iter() { - command.env_remove(name); - } - for (name, value) in ¶ms.secret_env { - command.env(name, value.expose()); - } - // Plain pipes never keep an input open: a one-shot payload is - // written and closed at start, otherwise the child reads EOF at - // once. Interactive input needs a PTY. - if params.stdin.is_some() { - command.stdin(Stdio::piped()); - } else { - command.stdin(Stdio::null()); - } - process_group::spawn_in_own_group(&mut command); - - let mut child = command.spawn().map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - format!("spawn process {:?}: {error}", params.argv), - ) - })?; - let pid = child.id(); - let stdout = child.stdout.take(); - let stderr = child.stderr.take(); - let stdin = child.stdin.take(); - if let Some(input) = params.stdin { - let Some(mut writer) = stdin else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - "process stdin was not available", - )); - }; - writer.write_all(input.as_slice()).await.map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - })?; - drop(writer); - } - - let process_id = params.process_id; - let redactions = redactions_for_secret_env(¶ms.secret_env); - let readers_open = u32::from(stdout.is_some()) + u32::from(stderr.is_some()); - let entry = Arc::new(ProcessEntry { - state: Mutex::new(ProcessState::new( - ProcessChild::Pipe(child), - pid, - pid, - None, - None, - redactions, - readers_open, - )), - notify: Notify::new(), - readers: Mutex::new(Vec::new()), - }); - self.insert_entry(&process_id, entry.clone()).await?; - - { - let mut readers = entry.readers.lock().await; - if let Some(stdout) = stdout { - readers.push(tokio::spawn(read_stream( - entry.clone(), - stdout, - ProcessOutputStream::Stdout, - ))); - } - if let Some(stderr) = stderr { - readers.push(tokio::spawn(read_stream( - entry.clone(), - stderr, - ProcessOutputStream::Stderr, - ))); + let options = ExecOptions { + timeout: command.timeout_ms.map(Duration::from_millis), + terminate_grace: Duration::from_millis(config.limits.terminate_grace_ms), + after_exit: AfterExit::Keep, + }; + let sink = ProcessOutput { + log: OutputLog::default(), + cursor: 0, + cap: config.limits.max_retained_output_bytes, + }; + let prepared = PreparedCommand::new(command, cwd, &config.scrubbed_env, pty); + let execution = match prepared.spawn(sink, options, config.limits.max_input_queue_bytes) { + Ok(execution) => execution, + Err(error) => { + self.release(&process_id); + return Err(error); } + }; + let pid = execution.pid(); + let record_path = self + .inner + .records_dir + .join(format!("{}.json", process_id.as_str())); + let entry = Arc::new(Entry { + execution: execution.clone(), + record_path: record_path.clone(), + fully_read_at: Mutex::new(None), + }); + self.table() + .slots + .insert(process_id.clone(), Slot::Live(entry)); + let stored = StoredSpawn { + process_id, + spawn: SpawnRecord::new(execution.group(), &config.instance_id), + }; + if let Err(error) = write_json_atomic(&record_path, &stored) { + eprintln!( + "lightspeed-envd: could not persist the spawn record of {}: {error}", + stored.process_id + ); } - self.spawn_watchers(entry, params.timeout_ms); - Ok(StartProcessResponse { process_id }) + execution.start_input(); + Ok(StartProcessResponse { pid }) } - async fn start_pty_process( + pub async fn read( &self, - params: StartProcessParams, - cwd: PathBuf, - ) -> Result { - let pty = native_pty_system() - .openpty(PtySize::default()) - .map_err(process_error("open PTY"))?; - let mut command = CommandBuilder::new(¶ms.argv[0]); - command.args(¶ms.argv[1..]); - command.cwd(&cwd); - for name in self.scrubbed_env.iter() { - command.env_remove(name); - } - for (name, value) in ¶ms.env { - command.env(name, value); - } - for (name, value) in ¶ms.secret_env { - command.env(name, value.expose()); - } - let child = pty - .slave - .spawn_command(command) - .map_err(process_error("spawn PTY process"))?; - // The PTY child is a session leader, so its pid is its group id. - let pid = child.process_id(); - let reader = pty - .master - .try_clone_reader() - .map_err(process_error("clone PTY reader"))?; - let mut stdin = pty - .master - .take_writer() - .map_err(process_error("open PTY writer"))?; - if let Some(input) = params.stdin.as_ref() { - stdin - .write_all(input.as_slice()) - .map_err(process_error("write initial PTY input"))?; - stdin.flush().map_err(process_error("flush PTY input"))?; + ctx: &RequestContext, + params: ReadProcessParams, + ) -> Result, ProtocolError> { + let config = &self.inner.config; + if params.max_bytes > config.limits.max_read_bytes { + return Err(ProtocolError::above_ceiling( + "maxBytes", + params.max_bytes, + config.limits.max_read_bytes, + )); } - let process_id = params.process_id; - let redactions = redactions_for_secret_env(¶ms.secret_env); - let entry = Arc::new(ProcessEntry { - state: Mutex::new(ProcessState::new( - ProcessChild::Pty(child), - pid, - pid, - Some(ProcessInput::Pty(stdin)), - Some(pty.master), - redactions, - 1, - )), - notify: Notify::new(), - readers: Mutex::new(Vec::new()), - }); - self.insert_entry(&process_id, entry.clone()).await?; - entry - .readers - .lock() - .await - .push(read_pty_stream(entry.clone(), reader)); - self.spawn_watchers(entry, params.timeout_ms); - Ok(StartProcessResponse { process_id }) - } - - async fn insert_entry( - &self, - process_id: &ProcessId, - entry: Arc, - ) -> Result<(), EnvironmentProtocolError> { - let mut processes = self.processes.lock().await; - if processes.contains_key(process_id.as_str()) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Conflict, - format!("process id already exists: {process_id}"), + if params.wait_ms > config.max_wait_ms { + return Err(ProtocolError::above_ceiling( + "waitMs", + params.wait_ms, + config.max_wait_ms, )); } - processes.insert(process_id.to_string(), entry); - Ok(()) - } - - fn spawn_watchers(&self, entry: Arc, timeout_ms: Option) { - if let Some(timeout_ms) = timeout_ms { - tokio::spawn(timeout_process( - entry.clone(), - Duration::from_millis(timeout_ms), + if params.after.is_some() && params.tail_bytes.is_some() { + return Err(ProtocolError::invalid_field( + "tailBytes", + "after and tailBytes exclude each other", )); } - tokio::spawn(watch_exit(entry, self.leftover_groups.clone())); - } - - pub async fn read_process( - &self, - params: ReadProcessParams, - ) -> Result { - let Some(entry) = self.entry(¶ms.process_id).await else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::NotFound, - format!("unknown process id: {}", params.process_id), + self.maintain(); + let entry = self.lookup(¶ms.process_id)?; + let execution = &entry.execution; + + let (start, end) = execution.with_state(|state| { + let output = &mut state.sink; + if let Some(ack) = params.ack { + output.cursor = output.cursor.max(ack.min(output.log.end())); + } + let end = output.log.end(); + let start = match (params.after, params.tail_bytes) { + (Some(after), _) => after, + (None, Some(tail)) => end.saturating_sub(tail), + (None, None) => output.cursor, + }; + (start, end) + }); + if start > end { + return Err(ProtocolError::invalid_field( + "after", + format!("after {start} is past the end of the output ({end})"), )); - }; - let deadline = params - .wait_ms - .map(|wait_ms| Instant::now() + Duration::from_millis(wait_ms)); + } + let deadline = tokio::time::Instant::now() + Duration::from_millis(params.wait_ms); + let mut changes = execution.subscribe(); loop { - let (response, schedule_cleanup, wake_at) = { - let mut state = entry.state.lock().await; - if let Some(pgid) = update_exit_status(&mut state)? { - self.observe_exit(&entry, pgid); - } - let now = Instant::now(); - let (done, drain_deadline) = exit_settled(&state, now); - let should_return = if params.wait_ms.is_some() { - done || deadline.is_some_and(|deadline| now >= deadline) - || params - .max_bytes - .is_some_and(|max_bytes| state.output.unread_bytes() >= max_bytes) - } else { - done - }; - if should_return { - let schedule_cleanup = state.exited && !state.cleanup_scheduled; - if schedule_cleanup { - state.cleanup_scheduled = true; + changes.borrow_and_update(); + let ready = execution.with_state(|state| { + state.settled() + || match params.return_on { + ReturnOn::AnyOutput => state.sink.log.end() > start, + ReturnOn::Window => state.sink.log.end() - start >= params.max_bytes, + ReturnOn::Exit => false, } - ( - Some(response_from_state( - &mut state, - params.after_seq, - params.max_bytes, - )), - schedule_cleanup, - None, - ) - } else { - let wake_at = match (deadline, drain_deadline) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }; - (None, false, wake_at) - } - }; - if let Some(response) = response { - if schedule_cleanup { - self.schedule_terminal_cleanup(params.process_id.clone(), entry.clone()); - } - return Ok(response); + }); + if ready { + break; } - - if let Some(wake_at) = wake_at { - tokio::select! { - _ = entry.notify.notified() => {} - _ = tokio::time::sleep_until(wake_at) => {} - } - } else { - entry.notify.notified().await; + tokio::select! { + () = ctx.cancel.cancelled() => return Err(ProtocolError::cancelled()), + () = tokio::time::sleep_until(deadline) => break, + changed = changes.changed() => if changed.is_err() { break }, } } - } - pub async fn write_process( - &self, - params: WriteProcessParams, - ) -> Result { - let Some(entry) = self.entry(¶ms.process_id).await else { - return Ok(WriteProcessResponse { - status: WriteProcessStatus::UnknownProcess, - }); - }; - let mut state = entry.state.lock().await; - if let Some(pgid) = update_exit_status(&mut state)? { - self.observe_exit(&entry, pgid); - } - let chunk = params.chunk.filter(|chunk| !chunk.as_slice().is_empty()); - // No input and nothing to close is a wait; the caller's read follows. - if chunk.is_none() && !params.close_stdin { - return Ok(WriteProcessResponse { - status: WriteProcessStatus::Accepted, - }); - } - if state.exited { - return Ok(WriteProcessResponse { - status: if chunk.is_some() { - WriteProcessStatus::StdinClosed - } else { - WriteProcessStatus::Accepted + let (mut response, settled) = execution.with_state(|state| { + let page = state + .sink + .log + .page(start, params.max_bytes, MAX_PAGE_CHUNKS); + let settled = state.settled(); + let response = ReadProcessResponse { + output: OutputPage { + chunks: page.chunks, + next: page.next, + end: state.sink.log.end(), + gaps: page.gaps, }, - }); - } - let Some(stdin) = state.stdin.as_mut() else { - return Ok(WriteProcessResponse { - status: if chunk.is_some() { - WriteProcessStatus::StdinClosed + state: if settled { + ExecState::Exited } else { - WriteProcessStatus::Accepted + ExecState::Running }, - }); - }; - if let Some(chunk) = chunk { - match stdin { - ProcessInput::Pty(stdin) => stdin - .write_all(chunk.as_slice()) - .and_then(|()| stdin.flush()), - } - .map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - })?; - } - if params.close_stdin { - state.stdin.take(); + exit: state.exit().filter(|_| settled).cloned(), + stopped_by: state.stopped_by().filter(|_| settled), + error: state.error().cloned(), + pid: execution.pid(), + leftovers: None, + }; + (response, settled) + }); + if settled { + response.leftovers = execution.sample_leftovers(LeftoverDisposition::Kept); } - Ok(WriteProcessResponse { - status: WriteProcessStatus::Accepted, - }) - } - pub async fn terminate_process( - &self, - params: TerminateProcessParams, - ) -> Result { - let Some(entry) = self.entry(¶ms.process_id).await else { - return Ok(TerminateProcessResponse { running: false }); - }; - let mut state = entry.state.lock().await; - if let Some(pgid) = update_exit_status(&mut state)? { - self.observe_exit(&entry, pgid); - } - if state.exited { - // The root is gone; the signal still reaches whatever it left in - // its group, which is how a handle stops the service it started. - if let Some(pgid) = state.pgid { - match params.signal { - ProcessSignal::Interrupt => { - process_group::interrupt_group(pgid); - } - ProcessSignal::Kill => { - process_group::kill_group(pgid); - } - } - } - return Ok(TerminateProcessResponse { running: false }); + let consuming = params.after.is_none() && params.tail_bytes.is_none(); + let complete = settled && response.output.next == response.output.end; + if !consuming && !complete { + return Ok(Reply::new(response)); } - match params.signal { - ProcessSignal::Interrupt => { - if let Some(pgid) = state.pgid { - process_group::interrupt_group(pgid); - } else { - interrupt_child(&mut state.child); - } + let next = response.output.next; + let entry = entry.clone(); + Ok(Reply::with_commit(response, move || { + if consuming { + entry.execution.with_state(|state| { + state.sink.cursor = state.sink.cursor.max(next); + }); } - ProcessSignal::Kill => { - if let Some(pgid) = state.pgid { - process_group::kill_group(pgid); - } - kill_child(&mut state.child).await?; - state.exited = true; - state.exit_observed_at = Some(Instant::now()); - state.exit_code = None; - state.termination = Some(ProcessTermination::Killed); - state.stdin.take(); - spawn_reader_drain(entry.clone()); - entry.notify.notify_waiters(); + if complete { + lock(&entry.fully_read_at).get_or_insert_with(Instant::now); } - } - Ok(TerminateProcessResponse { running: true }) + })) } - pub async fn resize_process( + pub async fn write( &self, - params: ResizeProcessParams, - ) -> Result { - let Some(entry) = self.entry(¶ms.process_id).await else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::NotFound, - format!("unknown process id: {}", params.process_id), - )); - }; - let state = entry.state.lock().await; - let Some(master) = state.pty_master.as_ref() else { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "process is not attached to a PTY", - )); - }; - master - .resize(PtySize { - rows: params.size.rows, - cols: params.size.cols, - pixel_width: 0, - pixel_height: 0, - }) - .map_err(process_error("resize PTY"))?; - Ok(ResizeProcessResponse {}) - } - - /// Records that a newly exited root left its group alive, for the idle - /// report, and starts the drain. The readers are left on the pipes; the - /// drain task switches them to discard mode once the grace passes. - fn observe_exit(&self, entry: &Arc, pgid: u32) { - record_leftover_group(pgid, &self.leftover_groups); - spawn_group_drain(entry.clone()); + _ctx: &RequestContext, + params: WriteProcessParams, + ) -> Result { + self.maintain(); + let entry = self.lookup(¶ms.process_id)?; + let accepted_bytes = entry + .execution + .offer_input(params.data.as_slice(), params.close_input)?; + Ok(WriteProcessResponse { accepted_bytes }) } - fn resolve_cwd(&self, path: &EnvironmentPath) -> Result { - let candidate = if path.is_absolute() { - PathBuf::from(path.as_str()) - } else if path.as_str() == "." { - self.cwd.clone() - } else { - self.cwd.join(path.as_str()) + pub async fn terminate( + &self, + _ctx: &RequestContext, + params: TerminateProcessParams, + ) -> Result { + self.maintain(); + let entry = self.lookup(¶ms.process_id)?; + let running = match params.signal { + ProcessSignal::Interrupt => entry.execution.interrupt(), + ProcessSignal::Terminate => entry.execution.stop(StoppedBy::Terminate), + ProcessSignal::Kill => entry.execution.stop(StoppedBy::Kill), }; - let normalized = normalize_path(candidate); - if !normalized.starts_with(&self.fs_root) { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Forbidden, - format!( - "process cwd is outside bridge fs root: {} (root {})", - normalized.display(), - self.fs_root.display() - ), - )); - } - Ok(normalized) - } - - async fn entry(&self, process_id: &ProcessId) -> Option> { - self.processes - .lock() - .await - .get(process_id.as_str()) - .cloned() - } - - /// Drops the entry after the retention period. Readers are never - /// aborted here: those a sweep abandoned are gone already, and those - /// draining a pipe a leftover still holds must stay on it until end of - /// file so the leftover never sees `EPIPE`. - fn schedule_terminal_cleanup(&self, process_id: ProcessId, entry: Arc) { - let processes = self.processes.clone(); - tokio::spawn(async move { - tokio::time::sleep(TERMINAL_PROCESS_RETENTION).await; - let mut processes = processes.lock().await; - if processes - .get(process_id.as_str()) - .is_some_and(|current| Arc::ptr_eq(current, &entry)) - { - processes.remove(process_id.as_str()); - } - }); + Ok(TerminateProcessResponse { running }) } -} -impl ProcessState { - fn new( - child: ProcessChild, - pid: Option, - pgid: Option, - stdin: Option, - pty_master: Option>, - redactions: Vec>, - readers_open: u32, - ) -> Self { - Self { - child, - pid, - pgid, - stdin, - pty_master, - redactions, - output: OutputBuffer::default(), - readers_open, - exited: false, - exit_observed_at: None, - exit_code: None, - termination: None, - leftover_processes: Vec::new(), - failure: None, - cleanup_scheduled: false, - discarding: false, + /// Claims `process_id` before any side effect: a running, retained, + /// tombstoned or earlier-instance id is `alreadyExists`. + fn reserve(&self, process_id: &ProcessId) -> Result<(), ProtocolError> { + let mut table = self.table(); + if table.slots.contains_key(process_id) { + return Err(ProtocolError::already_exists(format!( + "process id {process_id} is already in use" + ))); } + table.slots.insert(process_id.clone(), Slot::Reserved); + Ok(()) } - fn push_output(&mut self, stream: ProcessOutputStream, bytes: &[u8]) { - if self.discarding { - return; + fn release(&self, process_id: &ProcessId) { + let mut table = self.table(); + if matches!(table.slots.get(process_id), Some(Slot::Reserved)) { + table.slots.remove(process_id); } - let bytes = redact_bytes(bytes, &self.redactions); - self.output.push(stream, bytes); } - fn reader_closed(&mut self) { - self.readers_open = self.readers_open.saturating_sub(1); - } -} - -/// Whether a read may return for an exited root: the readers reached end of -/// file, or the drain grace since the exit passed and leftovers still hold -/// the pipes. Also returns when to wake up for the grace when it has not. -fn exit_settled(state: &ProcessState, now: Instant) -> (bool, Option) { - if !state.exited { - return (false, None); - } - if state.readers_open == 0 || state.discarding { - return (true, None); + fn lookup(&self, process_id: &ProcessId) -> Result, ProtocolError> { + match self.table().slots.get(process_id) { + Some(Slot::Live(entry)) => Ok(entry.clone()), + Some(Slot::Tombstone(ended)) => Err(ProtocolError::not_found_because( + NotFoundReason::Expired, + format!("process {process_id} {ended} and its output is no longer retained"), + )), + Some(Slot::Restarted) => Err(ProtocolError::not_found_because( + NotFoundReason::DaemonRestarted, + format!("process {process_id} belongs to an earlier daemon instance"), + )), + Some(Slot::Reserved) | None => Err(ProtocolError::not_found(format!( + "unknown process id: {process_id}" + ))), + } } - let Some(exit_observed_at) = state.exit_observed_at else { - return (true, None); - }; - let drain_deadline = exit_observed_at + process_group::OUTPUT_DRAIN_GRACE; - (now >= drain_deadline, Some(drain_deadline)) -} -async fn read_stream(entry: Arc, mut reader: R, stream: ProcessOutputStream) -where - R: AsyncRead + Unpin, -{ - let mut buffer = vec![0; READ_BUFFER_BYTES]; - loop { - match reader.read(&mut buffer).await { - Ok(0) => break, - Ok(read) => { - let mut state = entry.state.lock().await; - state.push_output(stream, &buffer[..read]); - entry.notify.notify_waiters(); + /// Evicts settled entries past their retention or beyond the exited + /// byte budget, oldest exit first, and prunes old tombstones. + fn maintain(&self) { + let limits = &self.inner.config.limits; + let now = Instant::now(); + let after_read = Duration::from_millis(limits.retention_after_exit_ms); + let unread = Duration::from_millis(limits.unread_retention_ms); + let mut table = self.table(); + + let mut settled = Vec::new(); + let mut evict = Vec::new(); + for (process_id, slot) in &table.slots { + let Slot::Live(entry) = slot else { + continue; + }; + let Some((settled_at, retained)) = entry.execution.with_state(|state| { + state + .settled_at() + .map(|at| (at, state.sink.log.retained_bytes())) + }) else { + continue; + }; + let read_at = *lock(&entry.fully_read_at); + let expired = + read_at.is_some_and(|at| now >= at + after_read) || now >= settled_at + unread; + if expired { + evict.push(process_id.clone()); + } else { + settled.push((settled_at, retained, process_id.clone())); } - Err(error) => { - let mut state = entry.state.lock().await; - if state.failure.is_none() && !state.exited { - state.failure = Some(error.to_string()); - } + } + settled.sort(); + let mut total: u64 = settled.iter().map(|(_, retained, _)| retained).sum(); + for (_, retained, process_id) in settled { + if total <= limits.max_exited_retained_bytes { break; } + total -= retained; + evict.push(process_id); } - } - entry.state.lock().await.reader_closed(); - entry.notify.notify_waiters(); -} -fn read_pty_stream( - entry: Arc, - mut reader: Box, -) -> JoinHandle<()> { - tokio::task::spawn_blocking(move || { - let handle = tokio::runtime::Handle::current(); - let mut buffer = vec![0; READ_BUFFER_BYTES]; - loop { - match reader.read(&mut buffer) { - Ok(0) => break, - Ok(read) => { - let entry = entry.clone(); - let bytes = buffer[..read].to_vec(); - handle.block_on(async move { - let mut state = entry.state.lock().await; - state.push_output(ProcessOutputStream::Pty, &bytes); - entry.notify.notify_waiters(); - }); - } - Err(error) => { - let entry = entry.clone(); - handle.block_on(async move { - let mut state = entry.state.lock().await; - if state.failure.is_none() && !state.exited { - state.failure = Some(error.to_string()); - } - }); - break; + for process_id in evict { + let Some(Slot::Live(entry)) = table.slots.remove(&process_id) else { + continue; + }; + let ended = entry.execution.with_state(|state| { + let mut ended = state + .exit() + .map_or_else(|| "ended".to_owned(), describe_exit); + if let Some(stopped_by) = state.stopped_by() { + let _ = write!(ended, " (stopped by {stopped_by:?})"); } + // Readers a leftover keeps on the pipes outlive the entry; + // they discard, and the retained output goes now. + state.sink.log = OutputLog::default(); + ended + }); + let group = entry.execution.group(); + if entry.execution.group_alive() { + lock(&self.inner.leftover_groups) + .insert(group.pgid, (group, entry.record_path.clone())); + } else { + let _ = std::fs::remove_file(&entry.record_path); } + table + .slots + .insert(process_id.clone(), Slot::Tombstone(ended)); + table.tombstones.push_back((process_id, now)); } - handle.block_on(async move { - entry.state.lock().await.reader_closed(); - entry.notify.notify_waiters(); - }); - }) -} -/// Observes the child's exit independently of pipe events so a silent child -/// whose leftovers hold the pipes open still completes promptly. -async fn watch_exit(entry: Arc, leftover_groups: Arc>>) { - loop { - tokio::time::sleep(EXIT_POLL_INTERVAL).await; - let done = { - let mut state = entry.state.lock().await; - if state.exited { - true - } else { - match update_exit_status(&mut state) { - Ok(Some(pgid)) => { - record_leftover_group(pgid, &leftover_groups); - spawn_group_drain(entry.clone()); - true - } - Ok(None) => state.exited, - Err(_) => true, - } + while let Some((_, evicted_at)) = table.tombstones.front() { + if table.tombstones.len() <= MAX_TOMBSTONES && now < *evicted_at + TOMBSTONE_RETENTION { + break; + } + let (process_id, _) = table.tombstones.pop_front().expect("front exists"); + if matches!(table.slots.get(&process_id), Some(Slot::Tombstone(_))) { + table.slots.remove(&process_id); } - }; - if done { - entry.notify.notify_waiters(); - return; } } -} -async fn timeout_process(entry: Arc, timeout: Duration) { - tokio::time::sleep(timeout).await; - { - let mut state = entry.state.lock().await; - if state.exited { - return; - } - if let Some(pgid) = state.pgid { - process_group::kill_group(pgid); - } - let _ = kill_child(&mut state.child).await; - state.exited = true; - state.exit_observed_at = Some(Instant::now()); - state.exit_code = None; - state.termination = Some(ProcessTermination::TimedOut); - state.stdin.take(); + fn table(&self) -> MutexGuard<'_, Table> { + lock(&self.inner.table) } - drain_or_abort_readers(&entry).await; - entry.notify.notify_waiters(); } -/// Polls the child for exit. When the exit is newly observed while members -/// of its process group are still alive, returns the group id so the caller -/// can record the leftovers. -fn update_exit_status(state: &mut ProcessState) -> Result, EnvironmentProtocolError> { - if state.exited { - return Ok(None); - } - let result = match &mut state.child { - ProcessChild::Pipe(child) => child - .try_wait() - .map(|status| status.map(|status| status.code())), - ProcessChild::Pty(child) => child - .try_wait() - .map(|status| status.map(|status| Some(status.exit_code() as i32))), - }; - match result { - Ok(Some(status)) => { - state.exited = true; - state.exit_observed_at = Some(Instant::now()); - state.exit_code = status; - state.stdin.take(); - Ok(state.pgid.filter(|pgid| process_group::group_alive(*pgid))) - } - Ok(None) => Ok(None), - Err(error) => { - state.failure = Some(error.to_string()); - Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - format!("poll process exit: {error}"), - )) - } - } +fn validate_pty(size: PtySize, input: &ExecInput) -> Result<(), ProtocolError> { + let valid = 1..=PtySize::MAX_DIMENSION; + if !valid.contains(&size.rows) || !valid.contains(&size.cols) { + return Err(ProtocolError::invalid_field( + "pty", + format!("PTY rows and cols must be 1 to {}", PtySize::MAX_DIMENSION), + )); + } + if matches!(input, ExecInput::Pipe {}) { + return Err(ProtocolError::invalid_field( + "command.input", + "a PTY is the input; pipe input cannot be combined with pty", + )); + } + Ok(()) } -fn record_leftover_group(pgid: u32, leftover_groups: &StdMutex>) { - leftover_groups +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex .lock() - .expect("leftover groups poisoned") - .insert(pgid); + .unwrap_or_else(|poisoned| poisoned.into_inner()) } -async fn kill_child(child: &mut ProcessChild) -> Result<(), EnvironmentProtocolError> { - let result = match child { - ProcessChild::Pipe(child) => child.kill().await, - ProcessChild::Pty(child) => child.kill(), - }; - result.map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - error.to_string(), - ) - }) -} +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; -fn interrupt_child(child: &mut ProcessChild) { - let pid = match child { - ProcessChild::Pipe(child) => child.id(), - ProcessChild::Pty(child) => child.process_id(), + use environment_protocol::{ + bytes::{ByteChunk, SecretString}, + data::exec::{CommandSpec, ExitStatus, GapReason, SignalName}, + error::{ErrorData, PreconditionReason, ProtocolErrorKind}, }; - #[cfg(unix)] - if let Some(pid) = pid - .and_then(|pid| i32::try_from(pid).ok()) - .and_then(rustix::process::Pid::from_raw) - { - let _ = rustix::process::kill_process(pid, rustix::process::Signal::INT); - } - #[cfg(not(unix))] - let _ = pid; -} -fn process_error( - context: &'static str, -) -> impl FnOnce(E) -> EnvironmentProtocolError { - move |error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::ProcessFailed, - format!("{context}: {error}"), - ) - } -} - -/// After a natural exit, gives the readers the drain grace to reach end of -/// file so bytes the root wrote just before exiting are kept, then leaves -/// them on the pipes in discard mode. The pipe's own end of file, when the -/// last leftover holding it exits, ends them. The group is never signalled. -fn spawn_group_drain(entry: Arc) { - tokio::spawn(async move { - tokio::time::sleep(process_group::OUTPUT_DRAIN_GRACE).await; - let mut state = entry.state.lock().await; - if state.readers_open > 0 { - state.discarding = true; - } - drop(state); - entry.notify.notify_waiters(); - }); -} - -fn spawn_reader_drain(entry: Arc) { - tokio::spawn(async move { - drain_or_abort_readers(&entry).await; - entry.notify.notify_waiters(); - }); -} + use super::*; -/// After the group was killed: waits the drain grace for the readers, then -/// abandons any still stuck on a pipe an escaped descendant holds. -async fn drain_or_abort_readers(entry: &ProcessEntry) { - let mut readers = std::mem::take(&mut *entry.readers.lock().await); - let deadline = Instant::now() + process_group::OUTPUT_DRAIN_GRACE; - for reader in &mut readers { - if tokio::time::timeout_at(deadline, &mut *reader) - .await - .is_err() - { - reader.abort(); + fn limits() -> ProcessLimits { + ProcessLimits { + max_read_bytes: 4 * 1024 * 1024, + max_retained_output_bytes: 4 * 1024 * 1024, + retention_after_exit_ms: 60_000, + unread_retention_ms: 7_200_000, + max_exited_retained_bytes: 64 * 1024 * 1024, + terminate_grace_ms: 200, + max_input_queue_bytes: 64 * 1024, } } - let mut state = entry.state.lock().await; - state.readers_open = 0; - state.discarding = true; -} -fn response_from_state( - state: &mut ProcessState, - after_seq: Option, - max_bytes: Option, -) -> ReadProcessResponse { - let (chunks, omitted_bytes, next_seq) = match after_seq { - None => { - let (chunks, omitted) = state.output.deliver(max_bytes); - (chunks, omitted, state.output.delivered_seq) - } - Some(after_seq) => { - let (chunks, omitted) = state.output.reread(after_seq, max_bytes); - let next_seq = chunks - .last() - .map_or(after_seq.max(state.output.delivered_seq), |chunk| { - chunk.seq + 1 - }); - (chunks, omitted, next_seq) - } - }; - // Leftovers are sampled when the caller looks, so the list reflects - // what is alive now rather than the instant the root exited, when a - // background child may still have been mid-exec. - if state.exited { - state.leftover_processes = match state.pgid.filter(|pgid| process_group::group_alive(*pgid)) - { - Some(pgid) => sample_group_members(pgid), - None => Vec::new(), - }; - } - ReadProcessResponse { - chunks, - next_seq, - exited: state.exited, - exit_code: state.exit_code, - closed: state.exited, - failure: state.failure.clone(), - termination: state.termination, - pid: state.pid, - omitted_bytes, - leftover_processes: state.leftover_processes.clone(), + struct Fixture { + _temp: tempfile::TempDir, + root: PathBuf, + state_dir: PathBuf, + manager: ProcessManager, } -} -fn normalize_path(path: PathBuf) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(std::path::MAIN_SEPARATOR.to_string()), - Component::CurDir => {} - Component::ParentDir => { - normalized.pop(); - } - Component::Normal(segment) => normalized.push(segment), + fn fixture_with(limits: ProcessLimits) -> Fixture { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path().canonicalize().expect("canonical root"); + let state_dir = root.join("state"); + let manager = manager_at(&root, &state_dir, limits, "envd_first"); + Fixture { + _temp: temp, + root, + state_dir, + manager, } } - normalized -} -/// Live, non-zombie members of a process group with their command lines, -/// best effort: `/proc` on Linux, libproc on macOS, empty elsewhere. -#[cfg(target_os = "linux")] -fn sample_group_members(pgid: u32) -> Vec { - let Ok(entries) = std::fs::read_dir("/proc") else { - return Vec::new(); - }; - let mut members = Vec::new(); - for entry in entries.flatten() { - let Some(pid) = entry - .file_name() - .to_str() - .and_then(|name| name.parse::().ok()) - else { - continue; - }; - let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) else { - continue; - }; - // "pid (comm) state ppid pgrp ..."; comm may contain spaces and - // parentheses, so split at the last closing parenthesis. - let Some(open) = stat.find('(') else { - continue; - }; - let Some(close) = stat.rfind(')') else { - continue; - }; - let mut fields = stat[close + 1..].split_whitespace(); - let state = fields.next(); - let _ppid = fields.next(); - let pgrp = fields.next().and_then(|value| value.parse::().ok()); - if pgrp != Some(pgid) || state == Some("Z") { - continue; - } - let command = std::fs::read(entry.path().join("cmdline")) - .ok() - .map(|bytes| { - bytes - .split(|byte| *byte == 0) - .filter(|part| !part.is_empty()) - .map(|part| String::from_utf8_lossy(part).into_owned()) - .collect::>() - .join(" ") - }) - .filter(|command| !command.is_empty()) - .unwrap_or_else(|| stat[open + 1..close].to_owned()); - members.push(LeftoverProcess { pid, command }); + fn fixture() -> Fixture { + fixture_with(limits()) + } + + fn manager_at( + root: &std::path::Path, + state_dir: &std::path::Path, + limits: ProcessLimits, + instance: &str, + ) -> ProcessManager { + let confinement = + Confinement::new(root.to_owned(), state_dir.to_owned()).expect("confinement"); + ProcessManager::new(ProcessConfig { + cwd: root.to_owned(), + confinement: Arc::new(confinement), + state_dir: state_dir.to_owned(), + scrubbed_env: Vec::new(), + instance_id: DaemonInstanceId::new(instance).unwrap(), + limits, + max_wait_ms: 60_000, + pty_supported: true, + }) + .expect("manager") } - members.sort_by_key(|member| member.pid); - members -} -#[cfg(target_os = "macos")] -fn sample_group_members(pgid: u32) -> Vec { - use std::{ffi::c_void, mem}; - - let Ok(group) = libc::pid_t::try_from(pgid) else { - return Vec::new(); - }; - let pid_size = mem::size_of::(); - // libproc's group listing returns pid counts, not bytes: with a null - // buffer the count of every process on the system, which is an upper - // bound for the group, and with a buffer the count it filled. - let upper_bound = unsafe { libc::proc_listpgrppids(group, std::ptr::null_mut(), 0) }; - if upper_bound <= 0 { - return Vec::new(); - } - let mut pids = vec![0 as libc::pid_t; upper_bound as usize + 16]; - let count = unsafe { - libc::proc_listpgrppids( - group, - pids.as_mut_ptr() as *mut c_void, - (pids.len() * pid_size) as libc::c_int, - ) - }; - if count <= 0 { - return Vec::new(); - } - pids.truncate(count as usize); - let mut members = Vec::new(); - for pid in pids { - if pid <= 0 { - continue; - } - let mut info: libc::proc_bsdinfo = unsafe { mem::zeroed() }; - let info_size = mem::size_of::() as libc::c_int; - let read = unsafe { - libc::proc_pidinfo( - pid, - libc::PROC_PIDTBSDINFO, - 0, - &mut info as *mut libc::proc_bsdinfo as *mut c_void, - info_size, - ) - }; - if read == info_size && info.pbi_status == libc::SZOMB { - continue; - } - let mut path = vec![0u8; 4096]; - let len = - unsafe { libc::proc_pidpath(pid, path.as_mut_ptr() as *mut c_void, path.len() as u32) }; - let command = if len > 0 { - String::from_utf8_lossy(&path[..len as usize]).into_owned() - } else { - String::new() - }; - members.push(LeftoverProcess { - pid: pid as u32, - command, - }); + fn ctx() -> RequestContext { + RequestContext::detached() } - members.sort_by_key(|member| member.pid); - members -} - -#[cfg(not(any(target_os = "linux", target_os = "macos")))] -fn sample_group_members(_pgid: u32) -> Vec { - Vec::new() -} - -#[cfg(test)] -mod tests { - use super::*; - use environment_protocol::shared::SecretString; - fn manager() -> (tempfile::TempDir, PathBuf, ProcessManager) { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().canonicalize().expect("canonical root"); - let manager = ProcessManager::new(root.clone(), root.clone()); - (temp, root, manager) + fn id(value: &str) -> ProcessId { + ProcessId::new(value).unwrap() } - fn params(process_id: &str, script: &str) -> StartProcessParams { + fn shell(process_id: &str, script: &str) -> StartProcessParams { StartProcessParams { - process_id: ProcessId::new(process_id), - argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(30_000), - tty: false, + process_id: id(process_id), + command: CommandSpec { + argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], + cwd: None, + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input: ExecInput::None {}, + timeout_ms: Some(30_000), + }, + pty: None, } } - fn read_params(process_id: &str, wait_ms: Option) -> ReadProcessParams { + fn read_params(process_id: &str, return_on: ReturnOn, wait_ms: u64) -> ReadProcessParams { ReadProcessParams { - process_id: ProcessId::new(process_id), - after_seq: None, - max_bytes: None, + process_id: id(process_id), + after: None, + tail_bytes: None, + ack: None, + max_bytes: 4 * 1024 * 1024, wait_ms, + return_on, } } - fn stream_bytes(response: &ReadProcessResponse, stream: ProcessOutputStream) -> Vec { - response - .chunks - .iter() - .filter(|chunk| chunk.stream == stream) - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) - .collect() + fn written(reply: Reply) -> ReadProcessResponse { + if let Some(hook) = reply.on_written { + hook(); + } + reply.value } - fn all_bytes(response: &ReadProcessResponse) -> Vec { + fn data(response: &ReadProcessResponse) -> Vec { response + .output .chunks .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) + .flat_map(|chunk| chunk.data.as_slice().to_vec()) .collect() } - #[tokio::test(flavor = "current_thread")] - async fn process_reports_stdout_stderr_exit_code_and_pid() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params("proc-1", "printf out; printf err >&2")) - .await - .expect("start"); - - let output = manager - .read_process(read_params("proc-1", None)) - .await - .expect("read"); - - assert!(output.exited); - assert_eq!(output.exit_code, Some(0)); - assert!(output.pid.is_some()); - assert_eq!(output.termination, None); - assert!(output.leftover_processes.is_empty()); - assert_eq!(stream_bytes(&output, ProcessOutputStream::Stdout), b"out"); - assert_eq!(stream_bytes(&output, ProcessOutputStream::Stderr), b"err"); + /// Cursor reads until the settled exit with nothing left unread. + async fn read_to_end( + manager: &ProcessManager, + process_id: &str, + ) -> (Vec, ReadProcessResponse) { + let mut bytes = Vec::new(); + loop { + let reply = manager + .read(&ctx(), read_params(process_id, ReturnOn::AnyOutput, 10_000)) + .await + .expect("read"); + let response = written(reply); + bytes.extend(data(&response)); + if response.state == ExecState::Exited && response.output.next == response.output.end { + return (bytes, response); + } + } } - #[tokio::test(flavor = "current_thread")] - async fn terminal_process_records_are_pruned_and_reusable() { - let (_temp, _root, manager) = manager(); - let params = params("proc-reused", "true"); + fn not_found_reason(error: &ProtocolError) -> Option { + match &error.data { + ErrorData::NotFound { reason } => *reason, + _ => panic!("expected notFound, got {error:?}"), + } + } - manager - .start_process(params.clone()) - .await - .expect("start first"); - let output = manager - .read_process(read_params("proc-reused", None)) - .await - .expect("read terminal output"); - assert!(output.exited); + #[cfg(unix)] + async fn wait_for_pid_file(path: &std::path::Path) -> i32 { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Ok(content) = std::fs::read_to_string(path) + && let Ok(pid) = content.trim().parse() + { + return pid; + } + assert!(Instant::now() < deadline, "pid file was not written"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } - tokio::time::sleep(TERMINAL_PROCESS_RETENTION * 2).await; + #[cfg(unix)] + fn process_alive(pid: i32) -> bool { + rustix::process::Pid::from_raw(pid) + .is_some_and(|pid| rustix::process::test_kill_process(pid).is_ok()) + } - manager - .start_process(params) - .await - .expect("start reused id"); + #[cfg(unix)] + fn kill(pid: i32) { + let _ = rustix::process::kill_process( + rustix::process::Pid::from_raw(pid).expect("pid"), + rustix::process::Signal::KILL, + ); } #[tokio::test(flavor = "current_thread")] - async fn plain_pipe_process_reads_eof_on_stdin_at_once() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params("proc-cat", "cat; echo done")) + async fn reports_both_streams_exit_code_and_pid() { + let fixture = fixture(); + let started = fixture + .manager + .start(&ctx(), shell("p1", "printf out; printf err >&2; exit 3")) .await .expect("start"); - let output = tokio::time::timeout( + let (bytes, response) = read_to_end(&fixture.manager, "p1").await; + assert_eq!(response.pid, started.pid); + assert_eq!(response.exit, Some(ExitStatus::Code { code: 3 })); + assert_eq!(response.stopped_by, None); + assert!(response.leftovers.is_none()); + // The two streams interleave in arrival order. + let mut sorted = bytes; + sorted.sort_unstable(); + let mut expected = b"outerr".to_vec(); + expected.sort_unstable(); + assert_eq!(sorted, expected); + assert_eq!(response.output.end, 6); + } + + #[tokio::test(flavor = "current_thread")] + async fn large_bytes_input_never_blocks_the_start() { + // More input than any pipe buffer, to a child that writes before it + // has read everything: the start must not wait for the input. + let fixture = fixture(); + let input: Vec = (0..1024 * 1024u32).map(|i| b'a' + (i % 26) as u8).collect(); + let mut params = shell("cat", "cat"); + params.command.input = ExecInput::Bytes { + data: ByteChunk::from(input.clone()), + }; + tokio::time::timeout( Duration::from_secs(5), - manager.read_process(read_params("proc-cat", None)), + fixture.manager.start(&ctx(), params), ) .await - .expect("cat exits without waiting for input") - .expect("read"); - assert!(output.exited); - assert_eq!( - stream_bytes(&output, ProcessOutputStream::Stdout), - b"done\n" - ); - - let write = manager - .write_process(WriteProcessParams { - process_id: ProcessId::new("proc-cat"), - chunk: Some(ByteChunk::from(b"late".as_slice())), - close_stdin: false, - }) - .await - .expect("write"); - assert_eq!(write.status, WriteProcessStatus::StdinClosed); + .expect("the start answers at once") + .expect("start"); + let (bytes, response) = read_to_end(&fixture.manager, "cat").await; + assert_eq!(response.exit, Some(ExitStatus::Code { code: 0 })); + assert_eq!(bytes, input, "every byte went through"); } #[tokio::test(flavor = "current_thread")] - async fn one_shot_stdin_is_written_and_closed_at_start() { - let (_temp, _root, manager) = manager(); - let mut params = params("proc-stdin", "cat"); - params.stdin = Some(ByteChunk::from(b"hello".as_slice())); - manager.start_process(params).await.expect("start"); - let output = manager - .read_process(read_params("proc-stdin", None)) + async fn a_large_pty_write_never_freezes_the_daemon() { + // A PTY child that never reads: a write must not block the runtime + // thread under the entry lock until the child is killed. + let fixture = fixture(); + let mut params = shell("pty", "sleep 30"); + params.pty = Some(PtySize { rows: 24, cols: 80 }); + fixture.manager.start(&ctx(), params).await.expect("start"); + let chunk = vec![b'x'; 16 * 1024]; + let mut accepted = 0; + for _ in 0..64 { + let write = tokio::time::timeout( + Duration::from_secs(1), + fixture.manager.write( + &ctx(), + WriteProcessParams { + process_id: id("pty"), + data: ByteChunk::from(chunk.clone()), + close_input: false, + }, + ), + ) .await - .expect("read"); - assert!(output.exited); - assert_eq!(stream_bytes(&output, ProcessOutputStream::Stdout), b"hello"); + .expect("a write answers without blocking") + .expect("write"); + accepted += write.accepted_bytes; + } + // Whether the terminal blocks or discards input a child never reads + // differs by platform; either way no write may exceed what was sent. + assert!(accepted <= 64 * 16 * 1024); + // The runtime thread is free, and a kill still works. + tokio::time::sleep(Duration::from_millis(10)).await; + let terminated = fixture + .manager + .terminate( + &ctx(), + TerminateProcessParams { + process_id: id("pty"), + signal: ProcessSignal::Kill, + }, + ) + .await + .expect("kill"); + assert!(terminated.running); + let (_, response) = read_to_end(&fixture.manager, "pty").await; + assert_eq!(response.stopped_by, Some(StoppedBy::Kill)); } #[tokio::test(flavor = "current_thread")] - async fn pty_process_accepts_input_echoes_and_resizes() { - let (_temp, _root, manager) = manager(); - let mut params = params("proc-pty", "read line; printf '%s' \"$line\""); - params.tty = true; - manager.start_process(params).await.expect("start PTY"); - manager - .resize_process(ResizeProcessParams { - process_id: ProcessId::new("proc-pty"), - size: environment_protocol::data::process::TerminalSize { - rows: 40, - cols: 120, - }, - }) - .await - .expect("resize PTY"); - let write = manager - .write_process(WriteProcessParams { - process_id: ProcessId::new("proc-pty"), - chunk: Some(ByteChunk::from(b"hello\n".as_slice())), - close_stdin: false, - }) - .await - .expect("write PTY"); - assert_eq!(write.status, WriteProcessStatus::Accepted); - let output = manager - .read_process(read_params("proc-pty", None)) - .await - .expect("read PTY"); - assert!(output.exited); - assert!( - output - .chunks - .iter() - .all(|chunk| chunk.stream == ProcessOutputStream::Pty) + async fn pipe_input_streams_until_closed() { + let fixture = fixture(); + let mut params = shell("pipe", "cat; echo done"); + params.command.input = ExecInput::Pipe {}; + fixture.manager.start(&ctx(), params).await.expect("start"); + let write = |data: &'static [u8], close_input| WriteProcessParams { + process_id: id("pipe"), + data: ByteChunk::from(data), + close_input, + }; + assert_eq!( + fixture + .manager + .write(&ctx(), write(b"hello ", false)) + .await + .unwrap() + .accepted_bytes, + 6 + ); + assert_eq!( + fixture + .manager + .write(&ctx(), write(b"world\n", true)) + .await + .unwrap() + .accepted_bytes, + 6 ); - assert!(String::from_utf8_lossy(&all_bytes(&output)).contains("hello")); + let (bytes, response) = read_to_end(&fixture.manager, "pipe").await; + assert_eq!(bytes, b"hello world\ndone\n"); + assert_eq!(response.exit, Some(ExitStatus::Code { code: 0 })); + let error = fixture + .manager + .write(&ctx(), write(b"late", false)) + .await + .unwrap_err(); + assert!(matches!( + error.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::InputClosed + } + )); } #[tokio::test(flavor = "current_thread")] - async fn wait_collects_for_the_whole_window_or_until_exit() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params( - "proc-window", - "for i in 1 2 3 4 5 6 7 8 9 10; do echo line$i; sleep 0.02; done", - )) + async fn the_cursor_moves_only_once_the_reply_is_written() { + let fixture = fixture(); + fixture + .manager + .start(&ctx(), shell("cursor", "printf first; sleep 30")) .await .expect("start"); - - let started = std::time::Instant::now(); - let output = manager - .read_process(read_params("proc-window", Some(10_000))) + let lost = fixture + .manager + .read(&ctx(), read_params("cursor", ReturnOn::AnyOutput, 5_000)) .await .expect("read"); - assert!(output.exited, "a long wait returns at exit"); - assert!(started.elapsed() < Duration::from_secs(5)); - let text = String::from_utf8_lossy(&all_bytes(&output)).into_owned(); - assert_eq!(text.lines().count(), 10, "every line arrives in one read"); - - manager - .start_process(params( - "proc-short", - "for i in 1 2 3 4 5 6 7 8 9 10; do echo line$i; sleep 0.05; done", - )) + assert_eq!(data(&lost.value), b"first"); + // The connection dropped before the frame went out: no hook runs. + drop(lost); + let again = written( + fixture + .manager + .read(&ctx(), read_params("cursor", ReturnOn::AnyOutput, 5_000)) + .await + .expect("read again"), + ); + assert_eq!(data(&again), b"first", "nothing was consumed"); + let after = fixture + .manager + .read(&ctx(), read_params("cursor", ReturnOn::AnyOutput, 0)) .await - .expect("start short"); - let started = std::time::Instant::now(); - let output = manager - .read_process(read_params("proc-short", Some(120))) + .expect("read after commit") + .value; + assert!(after.output.chunks.is_empty()); + assert_eq!(after.output.next, 5); + + // A cancelled wait answers aborted and consumes nothing either. + let cancelled = ctx(); + cancelled.cancel.cancel(); + let error = fixture + .manager + .read(&cancelled, read_params("cursor", ReturnOn::Exit, 5_000)) .await - .expect("read short"); - let elapsed = started.elapsed(); - assert!(!output.exited); - assert!( - elapsed >= Duration::from_millis(100) && elapsed < Duration::from_millis(400), - "returned at the window, not the first chunk: {elapsed:?}" - ); - let lines = String::from_utf8_lossy(&all_bytes(&output)).lines().count(); - assert!( - (1..10).contains(&lines), - "partial output at the window: {lines} lines" - ); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::Aborted); + fixture + .manager + .terminate( + &ctx(), + TerminateProcessParams { + process_id: id("cursor"), + signal: ProcessSignal::Kill, + }, + ) + .await + .expect("kill"); } #[tokio::test(flavor = "current_thread")] - async fn cursor_reads_return_disjoint_output_and_rereads_do_not_move_it() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params( - "proc-cursor", - "echo first; sleep 0.2; echo second; sleep 0.2; echo third", - )) + async fn the_end_of_a_large_output_survives_and_exit_waits_for_it() { + // A command that prints far more than is retained and then its + // summary: the read at exit shows the head, one gap and the end. + let fixture = fixture_with(ProcessLimits { + max_retained_output_bytes: 64 * 1024, + ..limits() + }); + fixture + .manager + .start( + &ctx(), + shell( + "build", + "i=0; while [ $i -lt 20000 ]; do echo line-$i; i=$((i+1)); done; echo 'FAILED: 3 tests'", + ), + ) .await .expect("start"); - - let first = manager - .read_process(read_params("proc-cursor", Some(80))) - .await - .expect("first read"); - assert_eq!(all_bytes(&first), b"first\n"); - assert!(!first.exited); - - let reread = manager - .read_process(ReadProcessParams { - process_id: ProcessId::new("proc-cursor"), - after_seq: Some(0), - max_bytes: None, - wait_ms: Some(0), - }) - .await - .expect("reread"); - assert!( - reread.chunks.is_empty(), - "delivered chunks are not retained" + let response = written( + fixture + .manager + .read(&ctx(), read_params("build", ReturnOn::Exit, 30_000)) + .await + .expect("read"), ); - - let rest = manager - .read_process(read_params("proc-cursor", Some(10_000))) - .await - .expect("second read"); - assert!(rest.exited); - assert_eq!(all_bytes(&rest), b"second\nthird\n"); + assert_eq!(response.state, ExecState::Exited); + assert_eq!( + response.output.next, response.output.end, + "nothing left unread" + ); + assert_eq!(response.output.gaps.len(), 1); + assert_eq!(response.output.gaps[0].reason, GapReason::Overflow); + let text = String::from_utf8(data(&response)).unwrap(); + assert!(text.starts_with("line-0\n")); assert!( - rest.chunks - .iter() - .all(|chunk| chunk.seq > first.chunks[0].seq), - "the second read never repeats the first" + text.ends_with("FAILED: 3 tests\n"), + "the summary at the end survives" ); + let gap = response.output.gaps[0].omitted_bytes; + assert_eq!(text.len() as u64 + gap, response.output.end); } #[tokio::test(flavor = "current_thread")] - async fn max_bytes_splits_the_first_chunk_and_keeps_the_remainder() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params("proc-split", "printf abcdefgh")) + async fn pages_at_explicit_offsets_never_consume_and_tail_reads_start_at_the_end() { + let fixture = fixture(); + fixture + .manager + .start(&ctx(), shell("pages", "printf abcdefgh")) .await .expect("start"); - let head = manager - .read_process(ReadProcessParams { - process_id: ProcessId::new("proc-split"), - after_seq: None, - max_bytes: Some(3), - wait_ms: None, - }) - .await - .expect("head"); - assert_eq!(all_bytes(&head), b"abc"); - let tail = manager - .read_process(ReadProcessParams { - process_id: ProcessId::new("proc-split"), - after_seq: None, - max_bytes: Some(100), - wait_ms: None, - }) - .await - .expect("tail"); - assert_eq!(all_bytes(&tail), b"defgh"); - assert_eq!(head.chunks[0].seq, tail.chunks[0].seq); + let mut params = read_params("pages", ReturnOn::Exit, 10_000); + // `after` past the output produced so far is refused before any + // wait, so settle the process first with a non-consuming read. + params.after = Some(0); + let settled = written(fixture.manager.read(&ctx(), params.clone()).await.unwrap()); + assert_eq!(settled.state, ExecState::Exited); + assert_eq!(settled.output.end, 8); + params.after = Some(2); + params.max_bytes = 3; + let page = written(fixture.manager.read(&ctx(), params.clone()).await.unwrap()); + assert_eq!(data(&page), b"cde"); + assert_eq!(page.output.next, 5); + params.after = None; + params.tail_bytes = Some(2); + let tail = written(fixture.manager.read(&ctx(), params.clone()).await.unwrap()); + assert_eq!(data(&tail), b"gh"); + params.tail_bytes = None; + params.after = Some(9); + let error = fixture.manager.read(&ctx(), params).await.unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + let (all, _) = read_to_end(&fixture.manager, "pages").await; + assert_eq!(all, b"abcdefgh", "offset reads left the cursor at 0"); } #[tokio::test(flavor = "current_thread")] - async fn unread_output_is_capped_to_head_and_tail() { - let (_temp, _root, manager) = manager(); - // 64 KiB of numbered lines against a 16 KiB test cap. - manager - .start_process(params( - "proc-cap", - "i=0; while [ $i -lt 4096 ]; do printf '%015d\\n' $i; i=$((i+1)); done", - )) + async fn a_duplicate_id_runs_nothing() { + let fixture = fixture(); + let script = "echo run >> runs.txt"; + fixture + .manager + .start(&ctx(), shell("dup", script)) .await .expect("start"); - let output = manager - .read_process(read_params("proc-cap", None)) + let error = fixture + .manager + .start(&ctx(), shell("dup", script)) .await - .expect("read"); - assert!(output.exited); - let delivered = all_bytes(&output); - assert!( - delivered.len() <= RETAINED_OUTPUT_CAP, - "{}", - delivered.len() - ); - assert!(output.omitted_bytes > 0); - assert_eq!( - delivered.len() as u64 + output.omitted_bytes, - 4096 * 16, - "head + omitted + tail account for every byte" - ); - let text = String::from_utf8_lossy(&delivered).into_owned(); - assert!(text.starts_with("000000000000000\n"), "head kept"); - assert!(text.ends_with("000000000004095\n"), "tail kept"); - let gaps = output - .chunks - .windows(2) - .filter(|pair| pair[1].seq != pair[0].seq + 1) - .count(); - assert_eq!(gaps, 1, "one gap where the middle was"); - - let again = manager - .read_process(read_params("proc-cap", Some(0))) - .await - .expect("second read"); - assert!(again.chunks.is_empty()); - assert_eq!(again.omitted_bytes, 0, "the omission is reported once"); - } - - #[test] - fn cap_keeps_one_contiguous_gap_when_chunk_sizes_are_uneven() { - // 16 KiB test cap, 8 KiB head. Seven 1000-byte chunks fill the head; - // the 1500-byte chunk after them is the first to be dropped, which - // leaves a 500-byte survivor right behind the head. A head recomputed - // purely by bytes would absorb that survivor on the next drop and - // open a second gap behind it. - let mut buffer = OutputBuffer::default(); - let mut push = |len: usize| buffer.push(ProcessOutputStream::Stdout, vec![b'x'; len]); - for _ in 0..7 { - push(1000); - } - push(1500); - push(500); - for _ in 0..5 { - push(1500); - } - push(1500); - let total = 7 * 1000 + 1500 + 500 + 5 * 1500 + 1500; - - let (chunks, omitted) = buffer.deliver(None); - - let delivered: usize = chunks - .iter() - .map(|chunk| chunk.chunk.as_slice().len()) - .sum(); - assert!(delivered <= RETAINED_OUTPUT_CAP, "{delivered}"); - assert!(omitted > 0); - assert_eq!(delivered as u64 + omitted, total as u64); - let gaps = chunks - .windows(2) - .filter(|pair| pair[1].seq != pair[0].seq + 1) - .count(); - assert_eq!(gaps, 1, "one contiguous gap after the head"); - assert_eq!(chunks[6].seq, 6, "the whole head survives"); + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::AlreadyExists); + read_to_end(&fixture.manager, "dup").await; + let runs = std::fs::read_to_string(fixture.root.join("runs.txt")).unwrap(); + assert_eq!(runs, "run\n", "the command ran once"); } #[tokio::test(flavor = "current_thread")] - async fn empty_write_is_a_wait_until_the_entry_is_pruned() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params("proc-wait", "echo bye; exit 7")) + async fn unread_exited_processes_expire_into_tombstones() { + let fixture = fixture_with(ProcessLimits { + unread_retention_ms: 50, + ..limits() + }); + fixture + .manager + .start(&ctx(), shell("old", "true")) .await .expect("start"); for _ in 0..200 { - if manager.running_count().await == 0 { + if fixture.manager.running_count().await == 0 { break; } tokio::time::sleep(Duration::from_millis(10)).await; } - let write = manager - .write_process(WriteProcessParams { - process_id: ProcessId::new("proc-wait"), - chunk: None, - close_stdin: false, - }) + tokio::time::sleep(process_group::OUTPUT_DRAIN_GRACE + Duration::from_millis(100)).await; + fixture.manager.running_count().await; + let error = fixture + .manager + .read(&ctx(), read_params("old", ReturnOn::Exit, 0)) .await - .expect("empty write"); - assert_eq!(write.status, WriteProcessStatus::Accepted); - let output = manager - .read_process(read_params("proc-wait", Some(1_000))) + .unwrap_err(); + assert_eq!(not_found_reason(&error), Some(NotFoundReason::Expired)); + let error = fixture + .manager + .start(&ctx(), shell("old", "true")) .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.exit_code, Some(7)); - assert_eq!(all_bytes(&output), b"bye\n"); - - tokio::time::sleep(TERMINAL_PROCESS_RETENTION * 3).await; - let write = manager - .write_process(WriteProcessParams { - process_id: ProcessId::new("proc-wait"), - chunk: None, - close_stdin: false, - }) - .await - .expect("late write"); - assert_eq!(write.status, WriteProcessStatus::UnknownProcess); - } - - #[cfg(unix)] - async fn wait_for_pid_file(path: &std::path::Path) -> i32 { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); - loop { - if let Ok(content) = std::fs::read_to_string(path) - && let Ok(pid) = content.trim().parse() - { - return pid; - } - assert!( - std::time::Instant::now() < deadline, - "pid file was not written" - ); - tokio::time::sleep(Duration::from_millis(20)).await; - } - } - - #[cfg(unix)] - fn process_alive(pid: i32) -> bool { - rustix::process::Pid::from_raw(pid) - .is_some_and(|pid| rustix::process::test_kill_process(pid).is_ok()) - } - - #[cfg(unix)] - async fn wait_for_process_gone(pid: i32) { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); - loop { - if !process_alive(pid) { - return; - } - assert!( - std::time::Instant::now() < deadline, - "descendant {pid} was not terminated" - ); - tokio::time::sleep(Duration::from_millis(20)).await; - } + .unwrap_err(); + assert_eq!( + error.kind(), + ProtocolErrorKind::AlreadyExists, + "a tombstoned id is not reused" + ); } - #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn kill_terminates_descendants_in_group() { - let (_temp, root, manager) = manager(); - manager - .start_process(params( - "proc-group", - "sleep 30 & echo $! > orphan.pid; sleep 30", - )) + async fn unknown_handles_are_not_found_on_every_method() { + let fixture = fixture(); + let read = fixture + .manager + .read(&ctx(), read_params("nope", ReturnOn::Exit, 0)) .await - .expect("start"); - - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - let terminated = manager - .terminate_process(TerminateProcessParams { - process_id: ProcessId::new("proc-group"), - signal: ProcessSignal::Kill, - }) - .await - .expect("terminate"); - assert!(terminated.running); - wait_for_process_gone(pid).await; - let output = manager - .read_process(read_params("proc-group", Some(1_000))) + .unwrap_err(); + assert_eq!(not_found_reason(&read), None); + let write = fixture + .manager + .write( + &ctx(), + WriteProcessParams { + process_id: id("nope"), + data: ByteChunk::default(), + close_input: true, + }, + ) .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.termination, Some(ProcessTermination::Killed)); - } - - #[cfg(unix)] - #[tokio::test(flavor = "current_thread")] - async fn timeout_terminates_descendants_in_group() { - let (_temp, root, manager) = manager(); - let mut params = params("proc-timeout", "sleep 30 & echo $! > orphan.pid; sleep 30"); - params.timeout_ms = Some(200); - manager.start_process(params).await.expect("start"); - - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - let output = manager - .read_process(read_params("proc-timeout", None)) + .unwrap_err(); + assert_eq!(write.kind(), ProtocolErrorKind::NotFound); + let terminate = fixture + .manager + .terminate( + &ctx(), + TerminateProcessParams { + process_id: id("nope"), + signal: ProcessSignal::Kill, + }, + ) .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.termination, Some(ProcessTermination::TimedOut)); - wait_for_process_gone(pid).await; + .unwrap_err(); + assert_eq!(terminate.kind(), ProtocolErrorKind::NotFound); } #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn interrupt_reaches_the_group_and_the_next_read_observes_the_trap() { - let (_temp, _root, manager) = manager(); - manager - .start_process(params( - "proc-int", - "trap 'echo caught; exit 3' INT; sleep 100 & wait $!", - )) + async fn signal_deaths_are_named_on_pipes_and_ptys() { + let fixture = fixture(); + fixture + .manager + .start(&ctx(), shell("segv", "kill -SEGV $$")) .await .expect("start"); - tokio::time::sleep(Duration::from_millis(200)).await; - let signalled = manager - .terminate_process(TerminateProcessParams { - process_id: ProcessId::new("proc-int"), - signal: ProcessSignal::Interrupt, - }) - .await - .expect("interrupt"); - assert!(signalled.running); - let output = manager - .read_process(read_params("proc-int", Some(5_000))) - .await - .expect("read"); - assert!(output.exited); - assert_eq!(output.exit_code, Some(3)); - assert_eq!(output.termination, None); - assert_eq!(all_bytes(&output), b"caught\n"); + let mut pty = shell("pty-term", "kill -TERM $$"); + pty.pty = Some(PtySize { rows: 24, cols: 80 }); + fixture.manager.start(&ctx(), pty).await.expect("start PTY"); + let (_, segv) = read_to_end(&fixture.manager, "segv").await; + assert!(matches!( + segv.exit, + Some(ExitStatus::Signal { ref signal, .. }) if signal == &SignalName::new("SIGSEGV").unwrap() + )); + let (_, term) = read_to_end(&fixture.manager, "pty-term").await; + assert_eq!( + term.exit, + Some(ExitStatus::Signal { + signal: SignalName::new("SIGTERM").unwrap(), + core_dumped: false, + }), + "a PTY signal death is not exit code 1" + ); } #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn process_exit_keeps_leftover_descendants_and_reports_them() { - let (_temp, root, manager) = manager(); - manager - .start_process(params( - "proc-leftover", - "nohup sleep 30 >/dev/null 2>&1 & echo $! > orphan.pid; exit 0", - )) + async fn terminate_escalates_to_kill_after_the_grace() { + let fixture = fixture(); + fixture + .manager + .start(&ctx(), shell("stubborn", "trap '' TERM; sleep 30")) .await .expect("start"); - - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - // Let the background child finish exec'ing so the sample names it. tokio::time::sleep(Duration::from_millis(100)).await; - let started = std::time::Instant::now(); - let output = manager - .read_process(read_params("proc-leftover", None)) + let terminated = fixture + .manager + .terminate( + &ctx(), + TerminateProcessParams { + process_id: id("stubborn"), + signal: ProcessSignal::Terminate, + }, + ) .await - .expect("read"); - assert!(started.elapsed() < Duration::from_secs(5)); - assert!(output.exited); - assert_eq!(output.exit_code, Some(0)); - assert!(process_alive(pid), "the leftover survives the call"); + .expect("terminate"); + assert!(terminated.running); + let (_, response) = read_to_end(&fixture.manager, "stubborn").await; + assert_eq!(response.stopped_by, Some(StoppedBy::Terminate)); assert_eq!( - output - .leftover_processes - .iter() - .map(|member| member.pid) - .collect::>(), - vec![pid as u32], - "{:?}", - output.leftover_processes - ); - assert!( - output.leftover_processes[0].command.contains("sleep"), - "{:?}", - output.leftover_processes - ); - assert_eq!(manager.leftover_group_count(), 1); - - tokio::time::sleep(TERMINAL_PROCESS_RETENTION * 3).await; - assert!( - process_alive(pid), - "the leftover survives the entry's pruning" + response.exit, + Some(ExitStatus::Signal { + signal: SignalName::new("SIGKILL").unwrap(), + core_dumped: false, + }) ); - let stale = manager - .read_process(read_params("proc-leftover", Some(0))) - .await; - assert!(stale.is_err(), "the entry was pruned"); + } - let _ = rustix::process::kill_process( - rustix::process::Pid::from_raw(pid).expect("valid child pid"), - rustix::process::Signal::KILL, - ); - wait_for_process_gone(pid).await; - assert_eq!(manager.leftover_group_count(), 0); + #[cfg(unix)] + #[tokio::test(flavor = "current_thread")] + async fn a_timeout_stops_the_group() { + let fixture = fixture(); + let mut params = shell("slow", "sleep 30 & echo $! > orphan.pid; sleep 30"); + params.command.timeout_ms = Some(200); + fixture.manager.start(&ctx(), params).await.expect("start"); + let orphan = wait_for_pid_file(&fixture.root.join("orphan.pid")).await; + let (_, response) = read_to_end(&fixture.manager, "slow").await; + assert_eq!(response.stopped_by, Some(StoppedBy::Timeout)); + for _ in 0..100 { + if !process_alive(orphan) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!(!process_alive(orphan), "the group was stopped"); } #[cfg(unix)] #[tokio::test(flavor = "current_thread")] - async fn leftover_writing_to_the_pipe_after_exit_is_neither_blocked_nor_killed() { - let (_temp, root, manager) = manager(); - manager - .start_process(params( - "proc-chatty", - "(while true; do echo x; sleep 0.01; done) & echo $! > orphan.pid; exit 0", - )) + async fn natural_exit_keeps_and_reports_leftovers() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + shell( + "service", + "nohup sleep 30 >/dev/null 2>&1 & echo $! > orphan.pid; exit 0", + ), + ) .await .expect("start"); - let entry = manager - .entry(&ProcessId::new("proc-chatty")) - .await - .expect("entry exists while running"); - let output = manager - .read_process(read_params("proc-chatty", None)) - .await - .expect("read"); - assert!(output.exited); - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - - // Past the drain grace the readers discard; nothing accumulates and - // the writer keeps running. The entry handle outlives the pruning - // that the read started. - tokio::time::sleep(process_group::OUTPUT_DRAIN_GRACE * 2).await; - let (discarding, retained) = { - let state = entry.state.lock().await; - (state.discarding, state.output.unread_bytes()) - }; - assert!(discarding); + let orphan = wait_for_pid_file(&fixture.root.join("orphan.pid")).await; tokio::time::sleep(Duration::from_millis(100)).await; - assert_eq!( - entry.state.lock().await.output.unread_bytes(), - retained, - "stored output does not grow after the drain grace" - ); - assert!(process_alive(pid), "the writer was not killed by EPIPE"); - - let _ = rustix::process::kill_process( - rustix::process::Pid::from_raw(pid).expect("valid child pid"), - rustix::process::Signal::KILL, - ); - wait_for_process_gone(pid).await; + let (_, response) = read_to_end(&fixture.manager, "service").await; + let leftovers = response.leftovers.expect("leftovers reported"); + assert_eq!(leftovers.disposition, LeftoverDisposition::Kept); + assert_eq!(leftovers.processes[0].pid, orphan as u32); + assert!(process_alive(orphan), "never swept on natural exit"); + assert_eq!(fixture.manager.leftover_group_count(), 1); + assert_eq!(fixture.manager.running_count().await, 0); + kill(orphan); } - #[cfg(unix)] + #[cfg(any(target_os = "linux", target_os = "macos"))] #[tokio::test(flavor = "current_thread")] - async fn kill_after_exit_stops_the_leftover_group() { - let (_temp, root, manager) = manager(); - manager - .start_process(params( - "proc-service", - "nohup sleep 30 >/dev/null 2>&1 & echo $! > orphan.pid; exit 0", - )) + async fn a_restarted_daemon_adopts_surviving_groups() { + let fixture = fixture(); + fixture + .manager + .start( + &ctx(), + shell( + "survivor", + "nohup sleep 30 >/dev/null 2>&1 & echo $! > orphan.pid; exit 0", + ), + ) .await .expect("start"); - let pid = wait_for_pid_file(&root.join("orphan.pid")).await; - assert!(process_alive(pid)); - for _ in 0..200 { - if manager.running_count().await == 0 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert_eq!(manager.leftover_group_count(), 1); - - let terminated = manager - .terminate_process(TerminateProcessParams { - process_id: ProcessId::new("proc-service"), - signal: ProcessSignal::Kill, - }) + let orphan = wait_for_pid_file(&fixture.root.join("orphan.pid")).await; + fixture + .manager + .start(&ctx(), shell("gone", "true")) .await - .expect("kill"); - assert!(!terminated.running, "the root had already exited"); - wait_for_process_gone(pid).await; - assert_eq!(manager.leftover_group_count(), 0); - let output = manager - .read_process(read_params("proc-service", Some(0))) + .expect("start"); + read_to_end(&fixture.manager, "survivor").await; + read_to_end(&fixture.manager, "gone").await; + + let restarted = manager_at(&fixture.root, &fixture.state_dir, limits(), "envd_second"); + let error = restarted + .read(&ctx(), read_params("survivor", ReturnOn::Exit, 0)) .await - .expect("first read after the kill"); - assert!(output.exited); - assert!( - output.leftover_processes.is_empty(), - "a dead group is no longer reported" + .unwrap_err(); + assert_eq!( + not_found_reason(&error), + Some(NotFoundReason::DaemonRestarted) + ); + assert_eq!( + restarted.leftover_group_count(), + 1, + "the surviving group is adopted" ); + assert!(process_alive(orphan), "adopted, never killed"); + kill(orphan); + for _ in 0..100 { + if restarted.leftover_group_count() == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!(restarted.leftover_group_count(), 0); + let records = std::fs::read_dir(fixture.state_dir.join("processes")) + .unwrap() + .count(); + assert_eq!(records, 0, "records of dead groups are deleted"); } #[tokio::test(flavor = "current_thread")] - async fn process_injects_secret_env_and_redacts_output() { - let (_temp, _root, manager) = manager(); - let mut params = params("proc-secret", "printf \"$SECRET_TOKEN\""); - params.secret_env = BTreeMap::from([( - "SECRET_TOKEN".to_owned(), - SecretString::new("super-secret-token"), - )]); - manager.start_process(params).await.expect("start"); - - let output = manager - .read_process(read_params("proc-secret", None)) - .await - .expect("read"); + async fn secrets_are_redacted_and_ptys_need_the_capability() { + let fixture = fixture(); + let mut params = shell("secret", "printf \"$TOKEN\""); + params.command.secret_env = + BTreeMap::from([("TOKEN".to_owned(), SecretString::new("hunter2"))]); + fixture.manager.start(&ctx(), params).await.expect("start"); + let (bytes, _) = read_to_end(&fixture.manager, "secret").await; + assert_eq!(bytes, b""); + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let confinement = Confinement::new(root.clone(), root.join("state")).unwrap(); + let without_pty = ProcessManager::new(ProcessConfig { + cwd: root.clone(), + confinement: Arc::new(confinement), + state_dir: root.join("state"), + scrubbed_env: Vec::new(), + instance_id: DaemonInstanceId::new("envd_nopty").unwrap(), + limits: limits(), + max_wait_ms: 1_000, + pty_supported: false, + }) + .unwrap(); + let mut pty = shell("pty", "true"); + pty.pty = Some(PtySize { rows: 24, cols: 80 }); + let error = without_pty.start(&ctx(), pty).await.unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::CapabilityUnavailable); + let mut wait = read_params("pty", ReturnOn::Exit, 1_001); + wait.max_bytes = 0; + let error = without_pty.read(&ctx(), wait).await.unwrap_err(); assert_eq!( - stream_bytes(&output, ProcessOutputStream::Stdout), - b"" + error.kind(), + ProtocolErrorKind::InvalidParams, + "waitMs above the ceiling" ); } } diff --git a/crates/environment-daemon/src/process_group.rs b/crates/environment-daemon/src/process_group.rs index 787d5e4c2..610366673 100644 --- a/crates/environment-daemon/src/process_group.rs +++ b/crates/environment-daemon/src/process_group.rs @@ -1,11 +1,11 @@ //! Unix process-group lifecycle shared by job and process execution. //! -//! Job and process commands start in their own process group so that -//! descendants a script leaves behind can be signalled as a unit. Jobs sweep -//! the group when the root exits; processes sweep it only on timeout, kill, -//! or cancellation, and otherwise let leftovers live with the environment. A -//! descendant can still escape the group (`setsid`), so callers must never -//! make completion depend on the group being fully swept. +//! Every command starts as the leader of its own process group, so the +//! descendants a script leaves behind can be signalled as a unit. Jobs +//! sweep the group when the root exits; processes signal it only on +//! timeout or an explicit terminate or kill, and otherwise let leftovers +//! live with the environment. A descendant can still escape the group +//! (`setsid`), so completion never depends on the group being empty. use std::time::Duration; @@ -14,27 +14,35 @@ use tokio::process::Command; #[cfg(unix)] use rustix::process::{Pid, Signal, kill_process_group, test_kill_process_group}; -/// How long a terminated group gets between SIGTERM and SIGKILL. -#[cfg(not(test))] -pub(crate) const GROUP_TERM_GRACE: Duration = Duration::from_secs(1); -#[cfg(test)] -pub(crate) const GROUP_TERM_GRACE: Duration = Duration::from_millis(50); - -/// How long remaining pipe output is drained after the root process exits -/// before the job or process completes without it. +/// How long remaining output is drained after the root process exits +/// before the execution settles without it. #[cfg(not(test))] pub(crate) const OUTPUT_DRAIN_GRACE: Duration = Duration::from_secs(2); #[cfg(test)] pub(crate) const OUTPUT_DRAIN_GRACE: Duration = Duration::from_millis(200); +/// How often a terminating group is checked for members during its grace. +const GRACE_POLL_INTERVAL: Duration = Duration::from_millis(20); + +/// A signal the daemon sends to a whole process group. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum GroupSignal { + Interrupt, + Terminate, + Kill, +} + /// Places the spawned command in its own process group (the child becomes /// the group leader). pub(crate) fn spawn_in_own_group(command: &mut Command) { #[cfg(unix)] command.process_group(0); + #[cfg(not(unix))] + let _ = command; } -/// True when at least one process is left in the group. +/// True when at least one process is left in the group. Says nothing about +/// whose group it is; see [`Group::alive`]. pub(crate) fn group_alive(pgid: u32) -> bool { #[cfg(unix)] return group_pid(pgid).is_some_and(|pid| test_kill_process_group(pid).is_ok()); @@ -45,60 +53,302 @@ pub(crate) fn group_alive(pgid: u32) -> bool { } } -/// Immediately kills every process left in the group. -pub(crate) fn kill_group(pgid: u32) -> bool { +/// Sends `signal` to every process in the group. Returns true when the +/// group had at least one member to signal. +pub(crate) fn signal_group(pgid: u32, signal: GroupSignal) -> bool { #[cfg(unix)] - return signal_group(pgid, Signal::KILL); + { + let signal = match signal { + GroupSignal::Interrupt => Signal::INT, + GroupSignal::Terminate => Signal::TERM, + GroupSignal::Kill => Signal::KILL, + }; + group_pid(pgid).is_some_and(|pid| kill_process_group(pid, signal).is_ok()) + } #[cfg(not(unix))] { - let _ = pgid; + let _ = (pgid, signal); false } } -/// Terminates every process left in the group: SIGTERM, a bounded grace -/// period, then SIGKILL. Returns true when any process had to be signalled. -pub(crate) async fn sweep_group(pgid: u32) -> bool { - #[cfg(unix)] - { - if !signal_group(pgid, Signal::TERM) { +/// One spawned process group, named by its id and its leader's start time. +/// +/// A group id stays reserved while the group has members, so a live group +/// whose leader is gone is still this group. Once the group empties, the id +/// may name a later, unrelated group whose leader got the same pid: a live +/// process with the group's pid and another start time means the group is +/// gone. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Group { + pub pgid: u32, + pub leader_start_time: Option, +} + +impl Group { + /// The group led by `pgid`, a live process (a just spawned root). + pub fn capture(pgid: u32) -> Self { + Self { + pgid, + leader_start_time: process_start_time(pgid), + } + } + + /// The group still has members, and they are this group's. + pub fn alive(&self) -> bool { + if !group_alive(self.pgid) { return false; } - tokio::time::sleep(GROUP_TERM_GRACE).await; - signal_group(pgid, Signal::KILL); + match process_start_time(self.pgid) { + None => true, + Some(started) => self.leader_start_time == Some(started), + } + } + + /// Sends `signal` to the group while it is alive. Returns true when it + /// had a member to signal. + pub fn signal(&self, signal: GroupSignal) -> bool { + self.alive() && signal_group(self.pgid, signal) + } + + /// Terminates every process left in the group: `SIGTERM`, up to + /// `grace` for the members to exit, then `SIGKILL` to whatever remains. + /// Returns true when the group had members to signal. + pub async fn terminate(self, grace: Duration) -> bool { + if !self.signal(GroupSignal::Terminate) { + return false; + } + self.kill_after_grace(grace).await; true } - #[cfg(not(unix))] - { - let _ = pgid; - false + + /// Waits up to `grace` for the group to empty, then kills what is left. + pub async fn kill_after_grace(self, grace: Duration) { + let deadline = tokio::time::Instant::now() + grace; + while self.alive() { + if tokio::time::Instant::now() >= deadline { + self.signal(GroupSignal::Kill); + return; + } + tokio::time::sleep(GRACE_POLL_INTERVAL.min(grace)).await; + } } } #[cfg(unix)] fn group_pid(pgid: u32) -> Option { - // Group IDs 0 and 1 have special kill semantics rather than naming a child group. + // Group ids 0 and 1 have special kill semantics rather than naming a + // child group. let pgid = i32::try_from(pgid).ok().filter(|pid| *pid > 1)?; Pid::from_raw(pgid) } -#[cfg(unix)] -fn signal_group(pgid: u32, signal: Signal) -> bool { - group_pid(pgid).is_some_and(|pid| kill_process_group(pid, signal).is_ok()) +/// Sends `SIGINT` to one process group by id, such as a terminal's +/// foreground group. Returns true when it had a member to signal. +pub(crate) fn interrupt_foreground(pgid: u32) -> bool { + signal_group(pgid, GroupSignal::Interrupt) } -/// Sends `SIGINT` to every process left in the group. Returns true when the -/// group had at least one member to signal. -pub(crate) fn interrupt_group(pgid: u32) -> bool { - #[cfg(unix)] - return signal_group(pgid, Signal::INT); - #[cfg(not(unix))] +/// This boot of the machine, so spawn records of an earlier boot are never +/// mistaken for live groups: Linux's boot id, macOS's boot time. +pub(crate) fn boot_id() -> Option { + #[cfg(target_os = "linux")] { - let _ = pgid; - false + std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .ok() + .map(|id| id.trim().to_owned()) + .filter(|id| !id.is_empty()) + } + #[cfg(target_os = "macos")] + { + let mut boot = libc::timeval { + tv_sec: 0, + tv_usec: 0, + }; + let mut size = std::mem::size_of::(); + // SAFETY: the name is NUL-terminated and the buffer is a timeval of + // the size passed. + let status = unsafe { + libc::sysctlbyname( + c"kern.boottime".as_ptr(), + (&mut boot as *mut libc::timeval).cast(), + &mut size, + std::ptr::null_mut(), + 0, + ) + }; + (status == 0 && boot.tv_sec > 0) + .then(|| format!("boottime:{}.{:06}", boot.tv_sec, boot.tv_usec)) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + None + } +} + +/// When process `pid` started, in a platform unit that only needs to be +/// stable for one boot: clock ticks since boot on Linux, microseconds since +/// the epoch on macOS. Tells a group leader apart from a later process that +/// reused its pid. +pub(crate) fn process_start_time(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // "pid (comm) state ppid ..."; comm may contain spaces and + // parentheses, so split after the last closing parenthesis. The + // start time is field 22, the 20th after comm. + let close = stat.rfind(')')?; + stat[close + 1..] + .split_whitespace() + .nth(19) + .and_then(|value| value.parse().ok()) + } + #[cfg(target_os = "macos")] + { + let info = bsd_info(pid)?; + Some(info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec) + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = pid; + None } } +/// Live, non-zombie members of a process group with their command lines, +/// lowest pid first. Best effort: `/proc` on Linux, libproc on macOS, +/// nothing elsewhere. +#[cfg(target_os = "linux")] +pub(crate) fn group_members(pgid: u32) -> Vec<(u32, Option)> { + let Ok(entries) = std::fs::read_dir("/proc") else { + return Vec::new(); + }; + let mut members = Vec::new(); + for entry in entries.flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.parse::().ok()) + else { + continue; + }; + let Ok(stat) = std::fs::read_to_string(entry.path().join("stat")) else { + continue; + }; + let (Some(open), Some(close)) = (stat.find('('), stat.rfind(')')) else { + continue; + }; + let mut fields = stat[close + 1..].split_whitespace(); + let state = fields.next(); + let _ppid = fields.next(); + let pgrp = fields.next().and_then(|value| value.parse::().ok()); + if pgrp != Some(pgid) || state == Some("Z") { + continue; + } + let command = std::fs::read(entry.path().join("cmdline")) + .ok() + .map(|bytes| { + bytes + .split(|byte| *byte == 0) + .filter(|part| !part.is_empty()) + .map(|part| String::from_utf8_lossy(part).into_owned()) + .collect::>() + .join(" ") + }) + .filter(|command| !command.is_empty()) + .or_else(|| { + let comm = stat.get(open + 1..close).unwrap_or_default(); + (!comm.is_empty()).then(|| comm.to_owned()) + }); + members.push((pid, command)); + } + members.sort_by_key(|(pid, _)| *pid); + members +} + +#[cfg(target_os = "macos")] +pub(crate) fn group_members(pgid: u32) -> Vec<(u32, Option)> { + use std::{ffi::c_void, mem}; + + let Ok(group) = libc::pid_t::try_from(pgid) else { + return Vec::new(); + }; + let pid_size = mem::size_of::(); + // libproc's group listing returns pid counts, not bytes: with a null + // buffer the count of every process on the system, which is an upper + // bound for the group, and with a buffer the count it filled. + // SAFETY: a null buffer of size 0 only queries the count. + let upper_bound = unsafe { libc::proc_listpgrppids(group, std::ptr::null_mut(), 0) }; + if upper_bound <= 0 { + return Vec::new(); + } + let mut pids = vec![0 as libc::pid_t; upper_bound as usize + 16]; + // SAFETY: the buffer holds `pids.len()` pids and its byte size is passed. + let count = unsafe { + libc::proc_listpgrppids( + group, + pids.as_mut_ptr() as *mut c_void, + (pids.len() * pid_size) as libc::c_int, + ) + }; + if count <= 0 { + return Vec::new(); + } + pids.truncate(count as usize); + let mut members = Vec::new(); + for pid in pids { + let Ok(pid) = u32::try_from(pid) else { + continue; + }; + if pid == 0 { + continue; + } + if bsd_info(pid).is_some_and(|info| info.pbi_status == libc::SZOMB) { + continue; + } + let mut path = vec![0u8; 4096]; + // SAFETY: the buffer is 4096 bytes and its length is passed. + let len = unsafe { + libc::proc_pidpath( + pid as libc::c_int, + path.as_mut_ptr() as *mut c_void, + path.len() as u32, + ) + }; + let command = + (len > 0).then(|| String::from_utf8_lossy(&path[..len as usize]).into_owned()); + members.push((pid, command)); + } + members.sort_by_key(|(pid, _)| *pid); + members +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub(crate) fn group_members(_pgid: u32) -> Vec<(u32, Option)> { + Vec::new() +} + +#[cfg(target_os = "macos")] +fn bsd_info(pid: u32) -> Option { + use std::{ffi::c_void, mem}; + + let pid = libc::c_int::try_from(pid).ok()?; + // SAFETY: proc_bsdinfo is plain data; zeroed is a valid value. + let mut info: libc::proc_bsdinfo = unsafe { mem::zeroed() }; + let size = mem::size_of::() as libc::c_int; + // SAFETY: the buffer is a proc_bsdinfo of the size passed. + let read = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 0, + &mut info as *mut libc::proc_bsdinfo as *mut c_void, + size, + ) + }; + (read == size).then_some(info) +} + #[cfg(all(test, unix))] mod tests { use super::*; @@ -107,7 +357,38 @@ mod tests { fn process_groups_reject_special_and_out_of_range_ids() { for pgid in [0, 1, i32::MAX as u32 + 1, u32::MAX] { assert!(group_pid(pgid).is_none()); + assert!(!signal_group(pgid, GroupSignal::Kill)); } assert_eq!(group_pid(42), Pid::from_raw(42)); } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn this_process_has_a_start_time_and_the_machine_a_boot_id() { + assert!(process_start_time(std::process::id()).is_some()); + assert!(boot_id().is_some()); + } + + #[tokio::test(flavor = "current_thread")] + async fn terminate_escalates_to_kill_when_the_group_ignores_sigterm() { + let mut command = Command::new("/bin/sh"); + command + .args(["-c", "trap '' TERM; sleep 30"]) + .stdout(std::process::Stdio::null()); + spawn_in_own_group(&mut command); + let mut child = command.spawn().expect("spawn"); + let pgid = child.id().expect("pid"); + // Give the shell time to install its trap. + tokio::time::sleep(Duration::from_millis(100)).await; + let started = std::time::Instant::now(); + assert!( + Group::capture(pgid) + .terminate(Duration::from_millis(200)) + .await + ); + let status = child.wait().await.expect("wait"); + assert!(started.elapsed() >= Duration::from_millis(150)); + use std::os::unix::process::ExitStatusExt as _; + assert_eq!(status.signal(), Some(Signal::KILL.as_raw())); + } } diff --git a/crates/environment-daemon/src/redaction.rs b/crates/environment-daemon/src/redaction.rs index 47d6bde65..b921bd459 100644 --- a/crates/environment-daemon/src/redaction.rs +++ b/crates/environment-daemon/src/redaction.rs @@ -1,6 +1,6 @@ //! Shared byte redaction for captured process and job output. -use environment_protocol::shared::SecretString; +use environment_protocol::bytes::SecretString; use std::collections::BTreeMap; pub(crate) fn redactions_for_secret_env( diff --git a/crates/environment-daemon/src/registration.rs b/crates/environment-daemon/src/registration.rs index 23b230f8c..7e99a13d5 100644 --- a/crates/environment-daemon/src/registration.rs +++ b/crates/environment-daemon/src/registration.rs @@ -1,316 +1,684 @@ //! Outbound registration: dial the environment gateway, prove the daemon -//! identity, and serve reverse-dialed data connections for as long as the -//! control connection lives. +//! identity, and serve reverse-dialed streams for as long as the control +//! connection lives. //! -//! The daemon never chooses an identity mode and never retries a terminal -//! rejection: a closed environment, a refused key, or a bad signature ends -//! the process with a non-zero exit so an orchestrator sees it. +//! The control connection runs the shared JSON-RPC envelope with the daemon +//! as the client (the frame rules and liveness rule are in +//! `environment_protocol::registration`). The daemon reconnects with +//! backoff after disconnects, heartbeat timeouts and retryable refusals, +//! and ends the process on everything else, each with its own exit status +//! ([`DaemonExit`]): a deliberate shutdown says goodbye first, a terminal +//! refusal or an unsupported gateway protocol is `Refused`, and a +//! `registration/closed` notification ends it as `EnvironmentClosed` or +//! `Superseded`. `EnvironmentClosed`, from that notification or an +//! `environmentClosed` admission refusal, ends only this loop: the server +//! exits or parks, as `--on-closed` says. use std::{ - path::Path, - process::Command, + path::{Path, PathBuf}, sync::Arc, time::{Duration, Instant}, }; use anyhow::{Context as _, Result, anyhow, bail}; use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + bytes::SecretString, + error::{ErrorData, PreconditionReason, ProtocolError}, + jsonrpc::{self, Id, Message as RpcMessage, Notification, Request, Response, ResponseError}, + limits::HEARTBEAT_MISSED_INTERVALS, + manifest::{ + RegistrationClosed, RegistrationGoodbye, RegistrationOpenStream, RegistrationRegister, + }, registration::{ - DaemonControlMessage, GatewayControlMessage, RegisterParams, RegistrationReceipt, - RegistrationRejectionCode, decode_hex, + ChallengeParams, ClosedParams, ClosedReason, GoodbyeParams, GoodbyeReason, + OpenStreamParams, OpenStreamResponse, RegisterParams, RegisterResponse, + RegistrationReceipt, StreamSpec, challenge_protocol_version, }, - shared::{CURRENT_PROTOCOL_VERSION, SecretString}, }; -use futures_util::{SinkExt as _, StreamExt as _}; +use futures_util::{SinkExt as _, StreamExt as _, stream::SplitSink, stream::SplitStream}; +use tokio::sync::mpsc; use tokio_tungstenite::{ Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config, tungstenite::{ Message, client::IntoClientRequest as _, - http::{HeaderValue, header::AUTHORIZATION}, + http::{HeaderValue, header::AUTHORIZATION, header::USER_AGENT}, }, }; use crate::{ DaemonRuntime, - config::{REEXEC_REGISTRATION_KEY_ENV, RegistrationConfig}, + config::{ + AUTO_UPGRADE_ATTEMPTED_ENV, GatewayUrls, REEXEC_REGISTRATION_KEY_ENV, RegistrationConfig, + RestartInvocation, + }, identity::DaemonIdentity, - server::{run_data_connection, tracing_line}, + server::{DaemonExit, run_data_connection, tracing_line, websocket_config}, }; -const AUTO_UPGRADE_ATTEMPTED_ENV: &str = "LIGHTSPEED_ENVD_AUTO_UPGRADE_ATTEMPTED"; - -/// How long the daemon waits for the gateway's challenge and verdict. +/// Bound on TCP connect, TLS and the WebSocket upgrade of the control +/// connection. +const DIAL_TIMEOUT: Duration = Duration::from_secs(20); +/// How long the daemon waits for the gateway's first frame, and then for +/// the register response. const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(20); +/// Bound on dialing one reverse stream, upgrade included. +const STREAM_DIAL_TIMEOUT: Duration = Duration::from_secs(10); +/// How long a shutting-down daemon spends on goodbye and the close frame. +const GOODBYE_TIMEOUT: Duration = Duration::from_secs(2); +/// Shortest heartbeat interval the liveness deadline is derived from. +const MIN_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(1); const INITIAL_BACKOFF: Duration = Duration::from_secs(1); const MAX_BACKOFF: Duration = Duration::from_secs(60); /// A control session that lasted this long resets the reconnect backoff. const STABLE_SESSION: Duration = Duration::from_secs(30); +/// The id of the daemon's only request on a control connection. +const REGISTER_ID: i64 = 1; + +type ControlSocket = WebSocketStream>; +type ControlWriter = SplitSink; +type ControlReader = SplitStream; /// Why one control session ended. +#[derive(Debug)] enum SessionEnd { - /// The gateway or network went away; reconnect with backoff. + /// The gateway or network went away, the liveness deadline passed, or + /// the gateway broke the frame rules; reconnect with backoff. Disconnected(String), - /// The gateway refused the daemon for a transient reason. - Retryable(RegistrationRejectionCode, String), + /// The gateway refused admission for a transient reason; reconnect no + /// sooner than its hint. + Retry { + reason: String, + retry_after: Option, + }, + /// The gateway speaks this other protocol version. + ProtocolMismatch(u32), + /// The gateway's first frame is unreadable: an incompatible gateway + /// whose protocol only its discovery document can tell. + IncompatibleGateway(String), + /// End the process. + Exit(DaemonExit), } -/// Registration failure the daemon must not retry. -#[derive(Debug, thiserror::Error)] -#[error("registration rejected ({code:?}): {message}")] -pub struct TerminalRejection { - pub code: RegistrationRejectionCode, - pub message: String, +/// Register with the configured gateway and serve it until the process +/// must end. +pub async fn run_outbound(runtime: DaemonRuntime, registration: RegistrationConfig) -> DaemonExit { + let outbound = match Outbound::new(runtime, registration) { + Ok(outbound) => outbound, + Err(error) => return DaemonExit::Configuration(error), + }; + outbound.run().await } -pub async fn run_outbound(runtime: DaemonRuntime, registration: RegistrationConfig) -> Result<()> { - let identity = DaemonIdentity::load_or_create(&runtime.config().state_dir) - .context("load daemon identity")?; - tracing_line(&format!( - "registering with {} as daemon public key {}", - registration.gateway_url, - identity.public_key_hex() - )); - let tls = TlsSettings::from_config(registration.ca_file.as_deref())?; - let mut backoff = INITIAL_BACKOFF; - loop { - let started = Instant::now(); - match control_session(&runtime, ®istration, &identity, &tls).await { - Ok(SessionEnd::Disconnected(reason)) => { - tracing_line(&format!("control connection ended: {reason}")); +struct Outbound { + runtime: DaemonRuntime, + registration: RegistrationConfig, + urls: GatewayUrls, + identity: DaemonIdentity, + tls: TlsSettings, + /// Held only until the first admission. + registration_key: Option, +} + +impl Outbound { + fn new(runtime: DaemonRuntime, registration: RegistrationConfig) -> Result { + let urls = registration.validate()?; + if let Some(path) = ®istration.receipt_path { + check_receipt_path(path)?; + } + let identity = DaemonIdentity::load_or_create(&runtime.config().state_dir) + .context("load daemon identity")?; + let tls = TlsSettings::from_config(registration.ca_file.as_deref())?; + let registration_key = registration.registration_key.clone(); + Ok(Self { + runtime, + registration, + urls, + identity, + tls, + registration_key, + }) + } + + async fn run(mut self) -> DaemonExit { + tracing_line(&format!( + "registering with {} as daemon public key {}", + self.urls.connect, + self.identity.public_key() + )); + let shutdown = self.runtime.shutdown().clone(); + let mut backoff = INITIAL_BACKOFF; + loop { + let started = Instant::now(); + let mut retry_after = None; + match self.session().await { + SessionEnd::Exit(exit) => return exit, + SessionEnd::Disconnected(reason) => { + tracing_line(&format!("control connection ended: {reason}")); + } + SessionEnd::Retry { + reason, + retry_after: hint, + } => { + tracing_line(&format!("registration deferred: {reason}")); + retry_after = hint; + } + SessionEnd::ProtocolMismatch(gateway_protocol) => { + match self.upgrade(Some(gateway_protocol), None).await { + SessionEnd::Exit(exit) => return exit, + SessionEnd::Retry { reason, .. } | SessionEnd::Disconnected(reason) => { + tracing_line(&reason); + } + SessionEnd::ProtocolMismatch(_) | SessionEnd::IncompatibleGateway(_) => {} + } + } + SessionEnd::IncompatibleGateway(reason) => { + match self.upgrade(None, Some(reason)).await { + SessionEnd::Exit(exit) => return exit, + SessionEnd::Retry { reason, .. } | SessionEnd::Disconnected(reason) => { + tracing_line(&reason); + } + SessionEnd::ProtocolMismatch(_) | SessionEnd::IncompatibleGateway(_) => {} + } + } } - Ok(SessionEnd::Retryable(code, message)) => { - tracing_line(&format!("registration deferred ({code:?}): {message}")); + if started.elapsed() >= STABLE_SESSION { + backoff = INITIAL_BACKOFF; } - Err(error) => { - if let Some(rejection) = error.downcast_ref::() { - tracing_line(&format!("{rejection}; not retrying")); - return Err(error); - } - tracing_line(&format!("control connection failed: {error:#}")); + let delay = jittered(backoff).max(retry_after.unwrap_or_default()); + tokio::select! { + _ = shutdown.cancelled() => return DaemonExit::Shutdown, + _ = tokio::time::sleep(delay) => {} } + backoff = (backoff * 2).min(MAX_BACKOFF); } - if started.elapsed() >= STABLE_SESSION { - backoff = INITIAL_BACKOFF; - } - tokio::time::sleep(jittered(backoff)).await; - backoff = (backoff * 2).min(MAX_BACKOFF); } -} -async fn control_session( - runtime: &DaemonRuntime, - registration: &RegistrationConfig, - identity: &DaemonIdentity, - tls: &TlsSettings, -) -> Result { - let request = registration - .gateway_url - .as_str() - .into_client_request() - .context("gateway URL")?; - let (socket, _) = connect_async_tls_with_config(request, None, false, Some(tls.connector())) - .await - .map_err(|error| describe_connect_error(®istration.gateway_url, error))?; - let (mut writer, mut reader) = socket.split(); - - let challenge = tokio::time::timeout(HANDSHAKE_TIMEOUT, next_control_message(&mut reader)) - .await - .context("waiting for gateway challenge")??; - let nonce = match challenge { - Some(GatewayControlMessage::Challenge { - protocol_version, - nonce, - }) => { - if protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(protocol_mismatch_error(registration, protocol_version).await); + /// One control connection, from dial to its end. + async fn session(&mut self) -> SessionEnd { + let shutdown = self.runtime.shutdown().clone(); + let dialed = tokio::select! { + _ = shutdown.cancelled() => return SessionEnd::Exit(DaemonExit::Shutdown), + dialed = tokio::time::timeout( + DIAL_TIMEOUT, + dial(&self.urls.connect, None, &self.tls), + ) => dialed, + }; + let socket = match dialed { + Ok(Ok(socket)) => socket, + Ok(Err(error)) => return SessionEnd::Disconnected(format!("{error:#}")), + Err(_) => { + return SessionEnd::Disconnected(format!( + "connecting to {} timed out after {} s", + self.urls.connect, + DIAL_TIMEOUT.as_secs() + )); } - decode_hex(&nonce).ok_or_else(|| anyhow!("gateway challenge nonce is not hex"))? + }; + let (mut writer, mut reader) = socket.split(); + let end = self.admitted_session(&mut writer, &mut reader).await; + if !matches!(end, SessionEnd::Exit(DaemonExit::Shutdown)) { + let _ = tokio::time::timeout(GOODBYE_TIMEOUT, writer.close()).await; } - Some(GatewayControlMessage::Rejected { code, message }) => { - return rejection(code, message); + end + } + + async fn admitted_session( + &mut self, + writer: &mut ControlWriter, + reader: &mut ControlReader, + ) -> SessionEnd { + let shutdown = self.runtime.shutdown().clone(); + + // 1. The frozen first frame: its version is read before anything + // else, so a gateway of any version can tell this daemon it is + // stale. + let first = tokio::select! { + _ = shutdown.cancelled() => return SessionEnd::Exit(DaemonExit::Shutdown), + first = tokio::time::timeout(HANDSHAKE_TIMEOUT, next_text(reader)) => first, + }; + let first = match first { + Ok(Ok(text)) => text, + Ok(Err(end)) => return end, + Err(_) => return SessionEnd::Disconnected("no challenge from the gateway".to_owned()), + }; + let gateway_protocol = match challenge_protocol_version(&first) { + Ok(version) => version, + Err(error) => { + return SessionEnd::IncompatibleGateway(format!( + "{} sent an unreadable first frame ({error})", + self.urls.connect + )); + } + }; + if gateway_protocol != ENVD_PROTOCOL_VERSION { + return SessionEnd::ProtocolMismatch(gateway_protocol); } - Some(other) => bail!("unexpected gateway message before challenge: {other:?}"), - None => { - return Ok(SessionEnd::Disconnected( - "closed before challenge".to_owned(), - )); + let challenge = match decode_challenge(&first) { + Ok(challenge) => challenge, + Err(error) => return SessionEnd::Disconnected(format!("{error:#}")), + }; + + // 2. Register, signing the nonce bound to the dialed origin. + let params = RegisterParams { + protocol_version: ENVD_PROTOCOL_VERSION, + registration_key: self.registration_key.clone(), + daemon_public_key: self.identity.public_key(), + signature: self + .identity + .sign_challenge(&challenge.nonce, &self.urls.origin), + display_name: self.registration.display_name.clone(), + metadata: self.registration.metadata.clone(), + implementation: self.runtime.server_info().clone(), + }; + let request = + match Request::new(Id::Number(REGISTER_ID), RegistrationRegister::NAME, ¶ms) { + Ok(request) => request, + Err(error) => { + return SessionEnd::Exit(DaemonExit::Failed(anyhow!( + "encode register: {error}" + ))); + } + }; + if let Err(error) = writer.send(Message::Text(request.to_text().into())).await { + return SessionEnd::Disconnected(format!("send register: {error}")); } - }; + let response = tokio::select! { + _ = shutdown.cancelled() => return SessionEnd::Exit(DaemonExit::Shutdown), + response = tokio::time::timeout(HANDSHAKE_TIMEOUT, register_response(reader)) => response, + }; + let response = match response { + Ok(Ok(response)) => response, + Ok(Err(end)) => return end, + Err(_) => { + return SessionEnd::Disconnected( + "no register response from the gateway".to_owned(), + ); + } + }; + let admitted = match response.decode::() { + Ok(admitted) => admitted, + Err(ResponseError::Failed(error)) => return refusal(error), + Err(ResponseError::Undecodable(message)) => { + return SessionEnd::Disconnected(format!( + "the register response breaks the contract: {message}" + )); + } + }; - let register = DaemonControlMessage::Register(RegisterParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - registration_key: registration.registration_key.clone(), - daemon_public_key: identity.public_key_hex(), - signature: identity.sign_challenge(&nonce), - display_name: registration.display_name.clone(), - metadata: registration.metadata.clone(), - implementation: runtime.implementation(), - }); - writer - .send(Message::Text(serde_json::to_string(®ister)?.into())) - .await - .context("send register")?; - - let verdict = tokio::time::timeout(HANDSHAKE_TIMEOUT, next_control_message(&mut reader)) - .await - .context("waiting for gateway verdict")??; - let (receipt, heartbeat_interval_ms) = match verdict { - Some(GatewayControlMessage::Accepted { - receipt, - heartbeat_interval_ms, - }) => (receipt, heartbeat_interval_ms), - Some(GatewayControlMessage::Rejected { code, message }) => { - return rejection(code, message); - } - Some(other) => bail!("unexpected gateway message after register: {other:?}"), - None => return Ok(SessionEnd::Disconnected("closed before verdict".to_owned())), - }; - announce_receipt(&receipt, registration.receipt_path.as_deref())?; - let _ = heartbeat_interval_ms; + // 3. Admitted: the key has done its job. + self.registration_key = None; + if let Err(error) = + announce_receipt(&admitted.receipt, self.registration.receipt_path.as_deref()) + { + return SessionEnd::Exit(DaemonExit::Configuration(error)); + } + let interval = + Duration::from_millis(admitted.heartbeat_interval_ms).max(MIN_HEARTBEAT_INTERVAL); + let liveness = interval * HEARTBEAT_MISSED_INTERVALS; + self.serve_admitted(writer, reader, liveness).await + } - loop { - let message = match reader.next().await { - Some(Ok(message)) => message, - Some(Err(error)) => return Ok(SessionEnd::Disconnected(error.to_string())), - None => return Ok(SessionEnd::Disconnected("gateway closed".to_owned())), - }; - match message { - Message::Text(text) => match serde_json::from_str::(&text) { - Ok(GatewayControlMessage::OpenData { token, data_url }) => { - let runtime = runtime.clone(); - let tls = tls.clone(); - tokio::spawn(async move { - if let Err(error) = - serve_data_connection(runtime, &data_url, token, &tls).await - { - tracing_line(&format!("data connection failed: {error:#}")); - } - }); + /// The admitted control connection: answer `registration/openStream`, + /// obey `registration/closed`, and treat `liveness` without any frame + /// as a dead connection. + async fn serve_admitted( + &self, + writer: &mut ControlWriter, + reader: &mut ControlReader, + liveness: Duration, + ) -> SessionEnd { + let shutdown = self.runtime.shutdown().clone(); + let (answers, mut queued) = mpsc::channel::(64); + let mut deadline = tokio::time::Instant::now() + liveness; + loop { + tokio::select! { + _ = shutdown.cancelled() => { + say_goodbye(writer).await; + return SessionEnd::Exit(DaemonExit::Shutdown); } - Ok(GatewayControlMessage::Rejected { code, message }) => { - return rejection(code, message); + Some(text) = queued.recv() => { + if let Err(error) = writer.send(Message::Text(text.into())).await { + return SessionEnd::Disconnected(format!("send to the gateway: {error}")); + } } - Ok(other) => tracing_line(&format!("ignoring gateway message {other:?}")), - Err(error) => tracing_line(&format!("ignoring undecodable gateway frame: {error}")), - }, - Message::Ping(bytes) => { - if writer.send(Message::Pong(bytes)).await.is_err() { - return Ok(SessionEnd::Disconnected("pong failed".to_owned())); + incoming = tokio::time::timeout_at(deadline, reader.next()) => { + let Ok(incoming) = incoming else { + return SessionEnd::Disconnected(format!( + "no frame from the gateway for {} ms (heartbeat timeout)", + liveness.as_millis() + )); + }; + deadline = tokio::time::Instant::now() + liveness; + let message = match incoming { + Some(Ok(message)) => message, + Some(Err(error)) => return SessionEnd::Disconnected(error.to_string()), + None => return SessionEnd::Disconnected("gateway closed".to_owned()), + }; + match message { + Message::Text(text) => { + if let Some(end) = self.on_admitted_text(text.as_str(), &answers) { + return end; + } + } + Message::Binary(_) => { + return SessionEnd::Disconnected( + "the gateway sent a binary frame".to_owned(), + ); + } + Message::Close(frame) => { + return SessionEnd::Disconnected( + frame + .map(|frame| frame.reason.to_string()) + .filter(|reason| !reason.is_empty()) + .unwrap_or_else(|| "gateway closed".to_owned()), + ); + } + Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => {} + } + } + } + } + } + + /// Handle one text frame after admission; `Some` ends the session. + fn on_admitted_text(&self, text: &str, answers: &mpsc::Sender) -> Option { + match jsonrpc::parse(text) { + Err(rejection) => Some(SessionEnd::Disconnected(format!( + "the gateway sent an invalid frame: {}", + rejection.error + ))), + Ok(RpcMessage::Request(request)) => { + if request.method == RegistrationOpenStream::NAME { + match request.params::() { + Ok(params) => self.open_stream(request.id, params, answers.clone()), + Err(error) => answer(answers, Response::failure(Some(request.id), error)), + } + } else { + answer( + answers, + Response::failure( + Some(request.id), + ProtocolError::method_not_found(&request.method), + ), + ); } + None } - Message::Close(frame) => { - let _ = writer.flush().await; - return Ok(SessionEnd::Disconnected( - frame - .map(|frame| frame.reason.to_string()) - .unwrap_or_else(|| "gateway closed".to_owned()), + Ok(RpcMessage::Notification(notification)) => self.on_notification(notification), + Ok(RpcMessage::Response(response)) => { + tracing_line(&format!( + "ignoring a gateway response to id {}", + response + .id + .map_or_else(|| "null".to_owned(), |id| id.to_string()) )); + None } - Message::Binary(_) | Message::Pong(_) | Message::Frame(_) => {} } } -} -async fn protocol_mismatch_error( - registration: &RegistrationConfig, - gateway_protocol: u32, -) -> anyhow::Error { - let base = format!( - "gateway speaks protocol {gateway_protocol}; this daemon speaks {CURRENT_PROTOCOL_VERSION}; discovery document: {}", - registration.discovery_url - ); - let Some(restart) = ®istration.auto_upgrade else { - return TerminalRejection { - code: RegistrationRejectionCode::UnsupportedProtocol, - message: format!( - "{base}; run `lightspeed-envd upgrade` or set LIGHTSPEED_ENVD_AUTO_UPGRADE=1" - ), + fn on_notification(&self, notification: Notification) -> Option { + if notification.method != RegistrationClosed::NAME { + tracing_line(&format!( + "ignoring gateway notification {}", + notification.method + )); + return None; } - .into(); - }; - if std::env::var_os(AUTO_UPGRADE_ATTEMPTED_ENV).is_some() { - return TerminalRejection { - code: RegistrationRejectionCode::UnsupportedProtocol, - message: format!( - "{base}; automatic upgrade was already attempted by this process lineage" + let params = match notification.params::() { + Ok(params) => params, + Err(error) => { + tracing_line(&format!( + "ignoring undecodable registration/closed: {error}" + )); + return None; + } + }; + if let Some(message) = ¶ms.message { + tracing_line(&format!("gateway closed the session: {}", message.as_str())); + } + Some(SessionEnd::Exit(match params.reason { + ClosedReason::Superseded => DaemonExit::Superseded { + daemon_public_key: self.identity.public_key().to_hex(), + }, + ClosedReason::EnvironmentClosed => DaemonExit::EnvironmentClosed, + })) + } + + /// Dial the stream the gateway asked for on its own task and answer + /// once it is upgraded (or failed), then serve it. + fn open_stream(&self, id: Id, params: OpenStreamParams, answers: mpsc::Sender) { + let OpenStreamParams { token, stream } = params; + let StreamSpec::DataProtocol {} = stream; + let runtime = self.runtime.clone(); + let stream_url = self.urls.stream.clone(); + let tls = self.tls.clone(); + tokio::spawn(async move { + let dialed = + tokio::time::timeout(STREAM_DIAL_TIMEOUT, dial(&stream_url, Some(&token), &tls)) + .await; + let socket = match dialed { + Ok(Ok(socket)) => socket, + Ok(Err(error)) => { + tracing_line(&format!("stream dial failed: {error:#}")); + let error = ProtocolError::unavailable(format!("dial {stream_url}: {error:#}")); + let _ = answers + .send(Response::failure(Some(id), error).to_text()) + .await; + return; + } + Err(_) => { + let error = ProtocolError::unavailable(format!( + "dialing {stream_url} timed out after {} s", + STREAM_DIAL_TIMEOUT.as_secs() + )); + let _ = answers + .send(Response::failure(Some(id), error).to_text()) + .await; + return; + } + }; + let _ = answers + .send(Response::success(id, &OpenStreamResponse {}).to_text()) + .await; + drop(answers); + run_data_connection(&runtime, socket).await; + }); + } + + /// Install the gateway's build and re-execute it, or say why not. + /// Install the gateway's envd build and re-execute it: the build of + /// `gateway_protocol` when the challenge named one, otherwise (an + /// unreadable first frame, explained by `unreadable`) whatever protocol + /// discovery publishes, provided it is not this daemon's own. Each + /// lineage tries once per target protocol, and once after an + /// unreadable frame. + async fn upgrade( + &self, + gateway_protocol: Option, + unreadable: Option, + ) -> SessionEnd { + let base = match (gateway_protocol, &unreadable) { + (Some(gateway_protocol), _) => mismatch_summary(&self.registration, gateway_protocol), + (None, reason) => format!( + "{}; it is not an environment gateway this daemon can register with; discovery document: {}", + reason.as_deref().unwrap_or("the gateway is incompatible"), + self.registration.discovery_url ), + }; + if let Some(refusal) = upgrade_refusal(&self.registration, gateway_protocol) { + return SessionEnd::Exit(DaemonExit::Refused(format!("{base}; {refusal}"))); } - .into(); + let Some(restart) = &self.registration.auto_upgrade else { + return SessionEnd::Exit(DaemonExit::Refused(base)); + }; + tracing_line(&format!( + "{base}; automatic upgrade enabled, installing the gateway's envd build" + )); + let request = crate::upgrade::UpgradeRequest { + discovery_url: self.registration.discovery_url.clone(), + ca_file: self.registration.ca_file.clone(), + install_path: restart.executable.clone(), + target: release_info::TARGET.to_owned(), + expected_protocol: gateway_protocol, + other_than: gateway_protocol.is_none().then_some(ENVD_PROTOCOL_VERSION), + }; + let installed = tokio::select! { + _ = self.runtime.shutdown().cancelled() => { + return SessionEnd::Exit(DaemonExit::Shutdown); + } + installed = crate::upgrade::install(request) => installed, + }; + let installed = match installed { + Ok(installed) => installed, + Err(error) if error.is_transient() => { + return SessionEnd::Retry { + reason: format!("{base}; automatic upgrade will retry: {error}"), + retry_after: None, + }; + } + Err(error) => { + return SessionEnd::Exit(DaemonExit::Refused(format!( + "{base}; automatic upgrade failed: {error}" + ))); + } + }; + tracing_line(&format!( + "installed lightspeed-envd {} ({}, protocol {}); re-executing {}", + installed.version, + installed.git_sha, + installed.protocol_version, + restart.executable.display() + )); + let error = exec_replacement( + restart, + installed.protocol_version, + self.registration_key.as_ref(), + ); + SessionEnd::Exit(DaemonExit::Refused(format!( + "{base}; installed the replacement but could not re-execute it: {error}" + ))) } +} - tracing_line(&format!( - "{base}; automatic upgrade enabled, installing the gateway's envd build" - )); - let request = crate::upgrade::UpgradeRequest { - discovery_url: registration.discovery_url.clone(), - ca_file: registration.ca_file.clone(), - install_path: restart.executable.clone(), - target: release_info::TARGET.to_owned(), - expected_protocol: Some(gateway_protocol), - }; - let installed = match crate::upgrade::install(request).await { - Ok(installed) => installed, - Err(error) => { - return TerminalRejection { - code: RegistrationRejectionCode::UnsupportedProtocol, - message: format!("{base}; automatic upgrade failed: {error:#}"), +/// The challenge params, decoded tolerantly: the frame is frozen and may +/// carry members this daemon does not know. +fn decode_challenge(frame: &str) -> Result { + let value: serde_json::Value = serde_json::from_str(frame).context("decode challenge")?; + let params = value + .get("params") + .cloned() + .ok_or_else(|| anyhow!("challenge has no params"))?; + serde_json::from_value(params).context("decode challenge params") +} + +/// Map a register error to what the daemon does next. +fn refusal(error: ProtocolError) -> SessionEnd { + match &error.data { + ErrorData::VersionMismatch { expected, .. } => SessionEnd::ProtocolMismatch(*expected), + ErrorData::FailedPrecondition { + reason: PreconditionReason::EnvironmentClosed, + } => SessionEnd::Exit(DaemonExit::EnvironmentClosed), + ErrorData::Busy { retry_after_ms, .. } | ErrorData::Unavailable { retry_after_ms } => { + SessionEnd::Retry { + reason: error.to_string(), + retry_after: retry_after_ms.map(Duration::from_millis), } - .into(); } - }; - tracing_line(&format!( - "installed lightspeed-envd {} ({}, protocol {}); re-executing {}", - installed.version, - installed.git_sha, - installed.protocol_version, - restart.executable.display() - )); - let error = exec_replacement( - &restart.executable, - &restart.arg0, - &restart.args, - registration.registration_key.as_ref(), - ); - TerminalRejection { - code: RegistrationRejectionCode::UnsupportedProtocol, - message: format!("{base}; installed the replacement but could not re-execute it: {error}"), + _ if error.is_retryable() => SessionEnd::Retry { + reason: error.to_string(), + retry_after: None, + }, + _ => SessionEnd::Exit(DaemonExit::Refused(error.to_string())), } - .into() +} + +fn mismatch_summary(registration: &RegistrationConfig, gateway_protocol: u32) -> String { + format!( + "gateway speaks protocol {gateway_protocol}; this daemon speaks {ENVD_PROTOCOL_VERSION}; discovery document: {}", + registration.discovery_url + ) +} + +/// Why no automatic upgrade runs for this gateway protocol (`None`: an +/// unreadable first frame), if it does not. +fn upgrade_refusal( + registration: &RegistrationConfig, + gateway_protocol: Option, +) -> Option { + if registration.auto_upgrade.is_none() { + return Some( + "run `lightspeed-envd upgrade` or set LIGHTSPEED_ENVD_AUTO_UPGRADE=1".to_owned(), + ); + } + match (gateway_protocol, registration.upgrade_attempted_for) { + (Some(gateway_protocol), Some(attempted)) if attempted == gateway_protocol => { + Some(format!( + "an automatic upgrade to protocol {gateway_protocol} already ran in this process lineage and the result still does not match" + )) + } + (None, Some(attempted)) => Some(format!( + "an automatic upgrade (to protocol {attempted}) already ran in this process lineage and the gateway's first frame is still unreadable" + )), + _ => None, + } +} + +/// Arguments of the restarted daemon: the original ones without any +/// registration key option, which travels only through the one-exec +/// variable and only while it is still needed. +fn replacement_args(args: &[std::ffi::OsString]) -> Vec { + let mut kept = Vec::with_capacity(args.len()); + let mut skip_value = false; + for arg in args { + if std::mem::take(&mut skip_value) { + continue; + } + let text = arg.to_string_lossy(); + if text == "--registration-key" || text == "--registration-key-file" { + skip_value = true; + continue; + } + if text.starts_with("--registration-key=") || text.starts_with("--registration-key-file=") { + continue; + } + kept.push(arg.clone()); + } + kept } #[cfg(unix)] fn exec_replacement( - executable: &Path, - arg0: &std::ffi::OsStr, - args: &[std::ffi::OsString], + restart: &RestartInvocation, + gateway_protocol: u32, registration_key: Option<&SecretString>, ) -> std::io::Error { use std::os::unix::process::CommandExt as _; - let mut command = Command::new(executable); + let mut command = std::process::Command::new(&restart.executable); command - .arg0(arg0) - .args(args) - .env(AUTO_UPGRADE_ATTEMPTED_ENV, "1"); - // Configuration parsing deliberately removes key variables from the - // current process. Hand the already-held value across exactly this exec - // so a first-seen daemon can still register even if its key file was - // removed; the replacement consumes and scrubs this internal variable. + .arg0(&restart.arg0) + .args(replacement_args(&restart.args)) + .env(AUTO_UPGRADE_ATTEMPTED_ENV, gateway_protocol.to_string()) + .env_remove("LIGHTSPEED_ENVD_REGISTRATION_KEY") + .env_remove("LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE") + .env_remove(REEXEC_REGISTRATION_KEY_ENV); + // Configuration removed the key variables from this process. A daemon + // not yet admitted hands the held key across exactly this exec, so it + // can still register even if its key file was removed; an admitted one + // proves itself with its private key alone. if let Some(registration_key) = registration_key { - command - .env(REEXEC_REGISTRATION_KEY_ENV, registration_key.expose()) - .env_remove("LIGHTSPEED_ENVD_REGISTRATION_KEY") - .env_remove("LIGHTSPEED_ENVD_REGISTRATION_KEY_FILE"); + command.env(REEXEC_REGISTRATION_KEY_ENV, registration_key.expose()); } command.exec() } #[cfg(not(unix))] fn exec_replacement( - _executable: &Path, - _arg0: &std::ffi::OsStr, - _args: &[std::ffi::OsString], + _restart: &RestartInvocation, + _gateway_protocol: u32, _registration_key: Option<&SecretString>, ) -> std::io::Error { std::io::Error::new( @@ -319,59 +687,161 @@ fn exec_replacement( ) } +/// Send `registration/goodbye` and the close frame, within a short bound. +async fn say_goodbye(writer: &mut ControlWriter) { + let goodbye = Notification::new( + RegistrationGoodbye::NAME, + &GoodbyeParams { + reason: GoodbyeReason::Shutdown, + }, + ) + .expect("goodbye params encode"); + let _ = tokio::time::timeout(GOODBYE_TIMEOUT, async { + writer.send(Message::Text(goodbye.to_text().into())).await?; + writer.close().await + }) + .await; +} + +fn answer(answers: &mpsc::Sender, response: Response) { + if answers.try_send(response.to_text()).is_err() { + tracing_line("dropping an answer to the gateway: the control writer is backed up"); + } +} + +/// The next text frame before admission. Control frames are skipped; a +/// binary frame, a close or a transport error ends the session. +async fn next_text(reader: &mut ControlReader) -> Result { + loop { + match reader.next().await { + Some(Ok(Message::Text(text))) => return Ok(text.as_str().to_owned()), + Some(Ok(Message::Binary(_))) => { + return Err(SessionEnd::Disconnected( + "the gateway sent a binary frame".to_owned(), + )); + } + Some(Ok(Message::Close(_))) | None => { + return Err(SessionEnd::Disconnected( + "gateway closed before admission".to_owned(), + )); + } + Some(Ok(_)) => continue, + Some(Err(error)) => return Err(SessionEnd::Disconnected(error.to_string())), + } + } +} + +/// Wait for the register response. Until it arrives the gateway may send +/// nothing else: any request or notification breaks the frame rules. +async fn register_response(reader: &mut ControlReader) -> Result { + loop { + let text = next_text(reader).await?; + match jsonrpc::parse(&text) { + Ok(RpcMessage::Response(response)) if response.id == Some(Id::Number(REGISTER_ID)) => { + return Ok(response); + } + Ok(RpcMessage::Response(response)) => tracing_line(&format!( + "ignoring a gateway response to id {}", + response + .id + .map_or_else(|| "null".to_owned(), |id| id.to_string()) + )), + Ok(RpcMessage::Request(request)) => { + return Err(SessionEnd::Disconnected(format!( + "the gateway sent {} before the register response", + request.method + ))); + } + Ok(RpcMessage::Notification(notification)) => { + return Err(SessionEnd::Disconnected(format!( + "the gateway sent {} before the register response", + notification.method + ))); + } + Err(rejection) => { + return Err(SessionEnd::Disconnected(format!( + "the gateway sent an invalid frame: {}", + rejection.error + ))); + } + } + } +} + +/// Dial a gateway route with the protocol's message limits, optionally +/// presenting a bearer token. +async fn dial( + url: &str, + bearer: Option<&SecretString>, + tls: &TlsSettings, +) -> Result { + let mut request = url.into_client_request().context("gateway URL")?; + request.headers_mut().insert( + USER_AGENT, + HeaderValue::from_static(concat!("lightspeed-envd/", env!("CARGO_PKG_VERSION"))), + ); + if let Some(token) = bearer { + let mut value = + HeaderValue::from_str(&format!("Bearer {}", token.expose())).context("token header")?; + value.set_sensitive(true); + request.headers_mut().insert(AUTHORIZATION, value); + } + let (socket, _) = connect_async_tls_with_config( + request, + Some(websocket_config()), + false, + Some(tls.connector()), + ) + .await + .map_err(|error| describe_connect_error(url, error))?; + Ok(socket) +} + /// A gateway that answers the WebSocket upgrade with an HTTP status is /// reachable but not serving registration; say so instead of leaking a raw /// handshake error. Still retried, because a deploy can produce the same /// answer for a moment. fn describe_connect_error( - gateway_url: &str, + url: &str, error: tokio_tungstenite::tungstenite::Error, ) -> anyhow::Error { if let tokio_tungstenite::tungstenite::Error::Http(response) = &error { let status = response.status(); if status == 404 { return anyhow!( - "{gateway_url} does not serve environment registration (HTTP 404): point LIGHTSPEED_ENVD_GATEWAY_URL at a Lightspeed process running the environment-gateway role" + "{url} does not serve environment registration (HTTP 404): point LIGHTSPEED_ENVD_GATEWAY_URL at a Lightspeed process running the environment-gateway role" ); } - return anyhow!( - "environment gateway {gateway_url} answered HTTP {status} instead of upgrading" - ); + return anyhow!("environment gateway {url} answered HTTP {status} instead of upgrading"); } - anyhow::Error::new(error).context(format!("connect to environment gateway {gateway_url}")) + anyhow::Error::new(error).context(format!("connect to environment gateway {url}")) } -fn rejection(code: RegistrationRejectionCode, message: String) -> Result { - if code.is_terminal() { - Err(TerminalRejection { code, message }.into()) - } else { - Ok(SessionEnd::Retryable(code, message)) - } +/// Refuse to start when the receipt cannot be written: a daemon admitted +/// without its receipt would be unusable to whoever waits for it. +fn check_receipt_path(path: &Path) -> Result<()> { + let parent = receipt_parent(path); + std::fs::create_dir_all(&parent) + .with_context(|| format!("create receipt dir {}", parent.display()))?; + let probe = parent.join(format!( + ".{}.probe.{}", + path.file_name() + .map_or_else(|| "receipt".into(), |name| name.to_string_lossy()), + std::process::id() + )); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&probe) + .with_context(|| format!("receipt dir {} is not writable", parent.display()))?; + let _ = std::fs::remove_file(&probe); + Ok(()) } -async fn next_control_message( - reader: &mut futures_util::stream::SplitStream>, -) -> Result> -where - S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, -{ - loop { - match reader.next().await { - Some(Ok(Message::Text(text))) => { - return serde_json::from_str(&text) - .map(Some) - .context("decode gateway control message"); - } - Some(Ok(Message::Binary(bytes))) => { - return serde_json::from_slice(&bytes) - .map(Some) - .context("decode gateway control message"); - } - Some(Ok(Message::Close(_))) | None => return Ok(None), - Some(Ok(_)) => continue, - Some(Err(error)) => return Err(error.into()), - } - } +fn receipt_parent(path: &Path) -> PathBuf { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map_or_else(|| PathBuf::from("."), Path::to_path_buf) } /// Write the receipt atomically (when a path is configured) and log it once. @@ -379,43 +849,21 @@ where fn announce_receipt(receipt: &RegistrationReceipt, path: Option<&Path>) -> Result<()> { let json = serde_json::to_string(receipt)?; if let Some(path) = path { - if let Some(parent) = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - { - std::fs::create_dir_all(parent) - .with_context(|| format!("create receipt dir {}", parent.display()))?; - } + let parent = receipt_parent(path); + std::fs::create_dir_all(&parent) + .with_context(|| format!("create receipt dir {}", parent.display()))?; let temp = path.with_extension(format!("tmp.{}", std::process::id())); std::fs::write(&temp, format!("{json}\n")) .with_context(|| format!("write receipt {}", temp.display()))?; - std::fs::rename(&temp, path) - .with_context(|| format!("install receipt {}", path.display()))?; + if let Err(error) = std::fs::rename(&temp, path) { + let _ = std::fs::remove_file(&temp); + bail!("install receipt {}: {error}", path.display()); + } } tracing_line(&format!("registered {json}")); Ok(()) } -/// Dial the gateway data route for one waiting worker and run the ordinary -/// data protocol on it. The token is a one-time pairing credential issued -/// over the authenticated control connection. -async fn serve_data_connection( - runtime: DaemonRuntime, - data_url: &str, - token: SecretString, - tls: &TlsSettings, -) -> Result<()> { - let mut request = data_url.into_client_request().context("data URL")?; - request.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {}", token.expose())).context("token header")?, - ); - let (socket, _) = connect_async_tls_with_config(request, None, false, Some(tls.connector())) - .await - .context("connect data route")?; - run_data_connection(&runtime, socket).await -} - /// TLS trust for the gateway: the bundled WebPKI roots, plus an operator /// supplied PEM bundle for gateways behind a private CA. /// @@ -472,65 +920,678 @@ fn jittered(base: Duration) -> Duration { base.mul_f64(factor) } -#[allow(dead_code)] -type ControlSocket = WebSocketStream>; - #[cfg(test)] mod tests { + use std::sync::Mutex; + + use environment_protocol::{ + error::{BusyReason, PermissionReason}, + registration::{ChallengeNonce, signed_registration_message}, + routes::{CONNECT_PATH, DATA_PATH, canonical_origin}, + }; + use serde_json::{Value, json}; + use tokio::net::{TcpListener, TcpStream}; + use tokio_tungstenite::{ + accept_hdr_async_with_config, + tungstenite::handshake::server::{Request as HttpRequest, Response as HttpResponse}, + }; + use super::*; + use crate::{config::OnClosed, test_support::runtime_with}; + + type GatewaySocket = WebSocketStream; + + /// A scripted gateway on a loopback port. + struct FakeGateway { + listener: TcpListener, + connect_url: String, + } + + struct Accepted { + socket: GatewaySocket, + path: String, + authorization: Option, + } + + impl FakeGateway { + async fn start(prefix: &str) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let connect_url = format!( + "ws://{}{prefix}{CONNECT_PATH}", + listener.local_addr().unwrap() + ); + Self { + listener, + connect_url, + } + } + + // The callback's error type is fixed by tungstenite's `Callback`. + #[allow(clippy::result_large_err)] + async fn accept(&self) -> Accepted { + let (stream, _) = tokio::time::timeout(Duration::from_secs(10), self.listener.accept()) + .await + .expect("the daemon dials in time") + .unwrap(); + let seen = Arc::new(Mutex::new((String::new(), None))); + let record = seen.clone(); + let socket = accept_hdr_async_with_config( + stream, + move |request: &HttpRequest, response: HttpResponse| { + let authorization = request + .headers() + .get(AUTHORIZATION) + .map(|value| value.to_str().unwrap().to_owned()); + *record.lock().unwrap() = (request.uri().path().to_owned(), authorization); + Ok(response) + }, + Some(websocket_config()), + ) + .await + .unwrap(); + let (path, authorization) = seen.lock().unwrap().clone(); + Accepted { + socket, + path, + authorization, + } + } + } + + async fn send(socket: &mut GatewaySocket, frame: Value) { + socket + .send(Message::Text(frame.to_string().into())) + .await + .unwrap(); + } + + async fn recv(socket: &mut GatewaySocket) -> Value { + loop { + let frame = tokio::time::timeout(Duration::from_secs(10), socket.next()) + .await + .expect("a frame in time") + .expect("open") + .expect("frame"); + if let Message::Text(text) = frame { + return serde_json::from_str(text.as_str()).unwrap(); + } + } + } + + async fn challenge(socket: &mut GatewaySocket, nonce: [u8; 32]) { + send( + socket, + json!({ + "jsonrpc": "2.0", + "method": "registration/challenge", + "params": { + "protocolVersion": ENVD_PROTOCOL_VERSION, + "nonce": ChallengeNonce::from_bytes(nonce).to_hex(), + "addedLater": {"x": 1}, + }, + }), + ) + .await; + } + + fn receipt(admission: &str) -> Value { + json!({ + "environmentId": "environment_a", + "daemonId": "daemon_a", + "controlConnectionId": "connection_a", + "identityMode": "ephemeral", + "admission": admission, + }) + } + + async fn admit(socket: &mut GatewaySocket, register: &Value, heartbeat_ms: u64) { + send( + socket, + json!({ + "jsonrpc": "2.0", + "id": register["id"], + "result": {"receipt": receipt("created"), "heartbeatIntervalMs": heartbeat_ms}, + }), + ) + .await; + } + + async fn refuse(socket: &mut GatewaySocket, register: &Value, error: ProtocolError) { + send( + socket, + Response::failure( + Some(serde_json::from_value(register["id"].clone()).unwrap()), + error, + ) + .to_value(), + ) + .await; + } + + fn daemon( + gateway: &FakeGateway, + adjust: impl FnOnce(&mut RegistrationConfig), + ) -> (tempfile::TempDir, DaemonRuntime, RegistrationConfig) { + let mut registration = RegistrationConfig::new( + gateway.connect_url.clone(), + crate::upgrade::resolve_discovery_url(Some(&gateway.connect_url), None).unwrap(), + ); + registration.registration_key = Some(SecretString::new("lsrk_test")); + adjust(&mut registration); + let config_registration = registration.clone(); + let (temp, runtime) = runtime_with(move |config| { + config.registration = Some(config_registration); + }); + (temp, runtime, registration) + } + + #[tokio::test(flavor = "current_thread")] + async fn registers_with_an_origin_bound_signature_and_drops_the_key_once_admitted() { + let gateway = FakeGateway::start("/ls").await; + let receipt_dir = tempfile::tempdir().unwrap(); + let receipt_path = receipt_dir.path().join("out").join("registration.json"); + let (_temp, runtime, registration) = daemon(&gateway, |registration| { + registration.receipt_path = Some(receipt_path.clone()); + }); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + + let mut first = gateway.accept().await; + assert_eq!(first.path, format!("/ls{CONNECT_PATH}")); + let nonce = [5u8; 32]; + challenge(&mut first.socket, nonce).await; + let register = recv(&mut first.socket).await; + assert_eq!(register["method"], "registration/register"); + let params = RegisterParams::decode(®ister["params"]).expect("valid register"); + assert_eq!(params.protocol_version, ENVD_PROTOCOL_VERSION); + assert_eq!( + params.registration_key.as_ref().map(SecretString::expose), + Some("lsrk_test") + ); + let origin = canonical_origin(&gateway.connect_url).unwrap(); + let key = + ed25519_dalek::VerifyingKey::from_bytes(params.daemon_public_key.as_bytes()).unwrap(); + let signature = ed25519_dalek::Signature::from_bytes(params.signature.as_bytes()); + key.verify_strict( + &signed_registration_message(&ChallengeNonce::from_bytes(nonce), &origin), + &signature, + ) + .expect("the signature binds the nonce and the dialed origin"); + admit(&mut first.socket, ®ister, 30_000).await; + + // The receipt is written once admitted. + for _ in 0..200 { + if receipt_path.exists() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let written: RegistrationReceipt = + serde_json::from_str(&std::fs::read_to_string(&receipt_path).unwrap()).unwrap(); + assert_eq!(written.daemon_id.as_str(), "daemon_a"); + + // A reconnect proves the identity by its key alone. + drop(first); + let mut second = gateway.accept().await; + challenge(&mut second.socket, [6u8; 32]).await; + let again = recv(&mut second.socket).await; + let again = RegisterParams::decode(&again["params"]).unwrap(); + assert!(again.registration_key.is_none(), "the key is never resent"); + assert_eq!(again.daemon_public_key, params.daemon_public_key); + + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn open_stream_dials_the_derived_data_route_with_the_token() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [1u8; 32]).await; + let register = recv(&mut control.socket).await; + admit(&mut control.socket, ®ister, 30_000).await; + + send( + &mut control.socket, + json!({ + "jsonrpc": "2.0", + "id": "g1", + "method": "registration/openStream", + "params": {"token": "stream-token", "stream": {"kind": "dataProtocol"}}, + }), + ) + .await; + let mut stream = gateway.accept().await; + assert_eq!(stream.path, DATA_PATH); + assert_eq!(stream.authorization.as_deref(), Some("Bearer stream-token")); + let answer = recv(&mut control.socket).await; + assert_eq!(answer["id"], "g1"); + assert_eq!(answer["result"], json!({})); + + // The stream serves the data plane. + send( + &mut stream.socket, + json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { + "protocolVersion": ENVD_PROTOCOL_VERSION, "clientInfo": {"name": "gateway-test"}, + }}), + ) + .await; + let initialized = recv(&mut stream.socket).await; + assert_eq!(initialized["result"]["plane"], "data"); + + // An unknown gateway request is answered, and the session stays. + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "id": "g2", "method": "registration/somethingNew"}), + ) + .await; + let unknown: ProtocolError = + serde_json::from_value(recv(&mut control.socket).await["error"].clone()).unwrap(); + assert_eq!( + unknown.kind(), + environment_protocol::error::ProtocolErrorKind::MethodNotFound + ); + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_silent_gateway_is_dropped_after_three_heartbeat_intervals() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut first = gateway.accept().await; + challenge(&mut first.socket, [2u8; 32]).await; + let register = recv(&mut first.socket).await; + let admitted_at = Instant::now(); + admit(&mut first.socket, ®ister, 1_000).await; + // Keep the socket open but silent: only the deadline can end it. + let _held = first; + let _second = gateway.accept().await; + let elapsed = admitted_at.elapsed(); + assert!( + elapsed >= Duration::from_millis(2_900), + "reconnected after {elapsed:?}, before the deadline" + ); + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn closed_notifications_end_the_process_with_distinct_statuses() { + for (reason, status) in [("superseded", 5), ("environmentClosed", 4)] { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [3u8; 32]).await; + let register = recv(&mut control.socket).await; + admit(&mut control.socket, ®ister, 30_000).await; + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "method": "registration/closed", "params": { + "reason": reason, "message": "another process with this identity connected", + }}), + ) + .await; + let exit = tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), status, "{reason}: {exit}"); + } + } + + /// Close the session of a freshly dialed control connection: admit and + /// send `registration/closed`, or refuse admission, both as + /// `environmentClosed`. + async fn close_environment(gateway: &FakeGateway, by_notification: bool) -> Accepted { + let mut control = gateway.accept().await; + challenge(&mut control.socket, [9u8; 32]).await; + let register = recv(&mut control.socket).await; + if by_notification { + admit(&mut control.socket, ®ister, 30_000).await; + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "method": "registration/closed", "params": { + "reason": "environmentClosed", + }}), + ) + .await; + } else { + refuse( + &mut control.socket, + ®ister, + ProtocolError::failed_precondition(PreconditionReason::EnvironmentClosed, "closed"), + ) + .await; + } + control + } + + #[tokio::test(flavor = "current_thread")] + async fn a_closed_environment_exits_or_parks_as_configured() { + for by_notification in [true, false] { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, _) = daemon(&gateway, |_| {}); + let server = tokio::spawn(crate::server::run(runtime.clone())); + let _control = close_environment(&gateway, by_notification).await; + let exit = tokio::time::timeout(Duration::from_secs(10), server) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), 4, "exit by default: {exit}"); + + let gateway = FakeGateway::start("").await; + let (_temp, runtime, _) = daemon(&gateway, |registration| { + registration.on_closed = OnClosed::Park; + }); + let server = tokio::spawn(crate::server::run(runtime.clone())); + let _control = close_environment(&gateway, by_notification).await; + let redial = + tokio::time::timeout(Duration::from_millis(1_500), gateway.listener.accept()).await; + assert!(redial.is_err(), "a parked daemon never dials again"); + assert!(!server.is_finished(), "a parked daemon keeps running"); + runtime.shutdown().cancel(); + let exit = tokio::time::timeout(Duration::from_secs(10), server) + .await + .expect("the parked daemon stops on a signal") + .unwrap(); + assert_eq!(exit.status(), 0, "{exit}"); + } + } + + #[cfg(unix)] + #[tokio::test(flavor = "current_thread")] + async fn parking_also_stops_the_passive_listener() { + let gateway = FakeGateway::start("").await; + let socket_dir = tempfile::tempdir().unwrap(); + let socket = socket_dir.path().join("envd.sock"); + let mut registration = RegistrationConfig::new( + gateway.connect_url.clone(), + crate::upgrade::resolve_discovery_url(Some(&gateway.connect_url), None).unwrap(), + ); + registration.on_closed = OnClosed::Park; + let listen = socket.clone(); + let (_temp, runtime) = runtime_with(move |config| { + config.registration = Some(registration); + config.listen = Some(crate::config::ListenAddr::Unix(listen)); + }); + let server = tokio::spawn(crate::server::run(runtime.clone())); + let _control = close_environment(&gateway, true).await; + let mut stopped = false; + for _ in 0..500 { + if socket.exists() && tokio::net::UnixStream::connect(&socket).await.is_err() { + stopped = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(stopped, "the listener stopped accepting"); + assert!(!server.is_finished(), "a parked daemon keeps running"); + runtime.shutdown().cancel(); + let exit = tokio::time::timeout(Duration::from_secs(10), server) + .await + .expect("the parked daemon stops on a signal") + .unwrap(); + assert_eq!(exit.status(), 0, "{exit}"); + } + + #[tokio::test(flavor = "current_thread")] + async fn register_errors_are_terminal_unless_retryable() { + let terminal = [ + ( + ProtocolError::permission_denied( + Some(PermissionReason::InvalidRegistrationKey), + "unknown key", + ), + 3, + ), + ( + ProtocolError::failed_precondition(PreconditionReason::EnvironmentClosed, "closed"), + 4, + ), + ( + ProtocolError::invalid_request("implementation.version is too long"), + 3, + ), + ]; + for (error, status) in terminal { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [4u8; 32]).await; + let register = recv(&mut control.socket).await; + refuse(&mut control.socket, ®ister, error.clone()).await; + let exit = tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), status, "{error}"); + } + + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [4u8; 32]).await; + let register = recv(&mut control.socket).await; + refuse( + &mut control.socket, + ®ister, + ProtocolError::busy(Some(BusyReason::RateLimited), Some(1_500), "slow down"), + ) + .await; + let refused_at = Instant::now(); + let _again = gateway.accept().await; + assert!( + refused_at.elapsed() >= Duration::from_millis(1_400), + "the retry hint is honoured" + ); + assert!(!task.is_finished()); + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_gateway_request_before_admission_breaks_the_session() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [7u8; 32]).await; + let _register = recv(&mut control.socket).await; + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "id": "early", "method": "registration/openStream", "params": { + "token": "t", "stream": {"kind": "dataProtocol"}, + }}), + ) + .await; + let mut ended = false; + while let Ok(Some(frame)) = + tokio::time::timeout(Duration::from_secs(5), control.socket.next()).await + { + match frame { + Ok(Message::Close(_)) | Err(_) => { + ended = true; + break; + } + Ok(Message::Text(text)) => panic!("the daemon answered: {text}"), + Ok(_) => {} + } + } + assert!(ended, "the daemon closed the connection"); + let _reconnected = gateway.accept().await; + task.abort(); + } + + #[tokio::test(flavor = "current_thread")] + async fn shutdown_says_goodbye_and_exits_cleanly() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + challenge(&mut control.socket, [8u8; 32]).await; + let register = recv(&mut control.socket).await; + admit(&mut control.socket, ®ister, 30_000).await; + // Give the daemon a moment to enter its admitted loop. + tokio::time::sleep(Duration::from_millis(100)).await; + runtime.shutdown().cancel(); + let goodbye = recv(&mut control.socket).await; + assert_eq!(goodbye["method"], "registration/goodbye"); + assert_eq!(goodbye["params"], json!({"reason": "shutdown"})); + assert!(goodbye.get("id").is_none()); + let exit = tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_gateway_of_another_version_without_auto_upgrade_is_terminal() { + let gateway = FakeGateway::start("").await; + let (_temp, runtime, registration) = daemon(&gateway, |_| {}); + let discovery = registration.discovery_url.clone(); + let task = tokio::spawn(run_outbound(runtime.clone(), registration)); + let mut control = gateway.accept().await; + send( + &mut control.socket, + json!({"jsonrpc": "2.0", "method": "registration/challenge", "params": { + "protocolVersion": ENVD_PROTOCOL_VERSION + 1, "nonce": "not even hex", + }}), + ) + .await; + let exit = tokio::time::timeout(Duration::from_secs(10), task) + .await + .expect("the daemon ends") + .unwrap(); + assert_eq!(exit.status(), 3); + let DaemonExit::Refused(message) = exit else { + panic!("refused"); + }; + assert!(message.contains(&discovery)); + assert!(message.contains("lightspeed-envd upgrade")); + } + + #[test] + fn an_upgrade_runs_once_per_target_version() { + let mut registration = RegistrationConfig::new( + "wss://gateway.example/environment-gateway/connect".to_owned(), + "https://gateway.example/.well-known/lightspeed-envd".to_owned(), + ); + assert!( + upgrade_refusal(®istration, Some(4)).is_some(), + "not opted in" + ); + assert!( + upgrade_refusal(®istration, None).is_some(), + "not opted in" + ); + registration.auto_upgrade = Some(RestartInvocation { + executable: PathBuf::from("/usr/local/bin/lightspeed-envd"), + arg0: "lightspeed-envd".into(), + args: Vec::new(), + }); + assert!(upgrade_refusal(®istration, Some(4)).is_none()); + assert!( + upgrade_refusal(®istration, None).is_none(), + "an unreadable first frame upgrades to whatever discovery publishes" + ); + registration.upgrade_attempted_for = Some(4); + assert!(upgrade_refusal(®istration, Some(4)).is_some()); + assert!( + upgrade_refusal(®istration, Some(5)).is_none(), + "a later bump upgrades again" + ); + assert!( + upgrade_refusal(®istration, None).is_some(), + "a replacement that still cannot read the first frame stops" + ); + } + + #[test] + fn a_replacement_never_inherits_a_registration_key_argument() { + let args: Vec = [ + "--registration-key", + "lsrk_secret", + "--gateway-url", + "wss://g/environment-gateway/connect", + "--registration-key-file=/run/key", + "--registration-name", + "n", + ] + .into_iter() + .map(Into::into) + .collect(); + let kept: Vec = replacement_args(&args) + .into_iter() + .map(|arg| arg.into_string().unwrap()) + .collect(); + assert_eq!( + kept, + [ + "--gateway-url", + "wss://g/environment-gateway/connect", + "--registration-name", + "n" + ] + ); + } + + #[test] + fn retryable_refusals_carry_their_hint() { + let SessionEnd::Retry { retry_after, .. } = refusal(ProtocolError::busy( + Some(BusyReason::RateLimited), + Some(250), + "x", + )) else { + panic!("retry"); + }; + assert_eq!(retry_after, Some(Duration::from_millis(250))); + assert!(matches!( + refusal(ProtocolError::unavailable("store down")), + SessionEnd::Retry { .. } + )); + assert!(matches!( + refusal(ProtocolError::version_mismatch(4, 3)), + SessionEnd::ProtocolMismatch(4) + )); + assert!(matches!( + refusal(ProtocolError::expired("key expired")), + SessionEnd::Exit(DaemonExit::Refused(_)) + )); + } #[test] fn receipts_are_written_atomically_and_contain_only_ids() { let temp = tempfile::tempdir().expect("tempdir"); let path = temp.path().join("out").join("registration.json"); - let receipt = RegistrationReceipt { - environment_id: "environment_a".to_owned(), - incarnation_id: "incarnation_a".to_owned(), - daemon_id: "daemon_a".to_owned(), - connection_id: "connection_a".to_owned(), - identity_mode: "ephemeral".to_owned(), - }; + check_receipt_path(&path).expect("writable"); + let receipt: RegistrationReceipt = serde_json::from_value(receipt("reconnected")).unwrap(); announce_receipt(&receipt, Some(&path)).expect("write"); let written: RegistrationReceipt = serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("json"); assert_eq!(written, receipt); - assert!(std::fs::read_dir(path.parent().unwrap()).unwrap().count() == 1); + assert_eq!( + std::fs::read_dir(path.parent().unwrap()).unwrap().count(), + 1 + ); } + #[cfg(unix)] #[test] - fn terminal_rejections_stop_and_transient_ones_retry() { - assert!(rejection(RegistrationRejectionCode::EnvironmentClosed, "x".to_owned()).is_err()); - assert!(matches!( - rejection(RegistrationRejectionCode::CapacityExhausted, "x".to_owned()), - Ok(SessionEnd::Retryable( - RegistrationRejectionCode::CapacityExhausted, - _ - )) - )); - } - - #[tokio::test(flavor = "current_thread")] - async fn protocol_mismatch_without_auto_upgrade_names_discovery_and_command() { - let registration = RegistrationConfig { - gateway_url: "wss://gateway.example/environment-gateway/connect".to_owned(), - registration_key: None, - display_name: None, - metadata: Default::default(), - receipt_path: None, - ca_file: None, - discovery_url: "https://gateway.example/.well-known/lightspeed-envd".to_owned(), - auto_upgrade: None, - }; - let error = protocol_mismatch_error(®istration, CURRENT_PROTOCOL_VERSION + 1).await; - let rejection = error - .downcast_ref::() - .expect("terminal rejection"); - assert_eq!( - rejection.code, - RegistrationRejectionCode::UnsupportedProtocol - ); - assert!(rejection.message.contains(®istration.discovery_url)); - assert!(rejection.message.contains("lightspeed-envd upgrade")); + fn an_unwritable_receipt_dir_is_refused_at_startup() { + use std::os::unix::fs::PermissionsExt as _; + let temp = tempfile::tempdir().expect("tempdir"); + let locked = temp.path().join("locked"); + std::fs::create_dir(&locked).unwrap(); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o500)).unwrap(); + let refused = check_receipt_path(&locked.join("registration.json")); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700)).unwrap(); + // Root ignores directory permissions. + if !rustix::process::geteuid().is_root() { + assert!(refused.is_err()); + } } /// A self-signed EC certificate, valid 2026-2036, used only as PEM input. diff --git a/crates/environment-daemon/src/rpc.rs b/crates/environment-daemon/src/rpc.rs deleted file mode 100644 index 35d2a370b..000000000 --- a/crates/environment-daemon/src/rpc.rs +++ /dev/null @@ -1,82 +0,0 @@ -use environment_protocol::error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}; -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::{Value, json}; - -#[derive(Debug)] -pub struct RpcRequest { - pub id: Option, - pub method: Option, - pub params: Value, -} - -pub fn parse_request(value: Value) -> Result { - if !value.is_object() { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - "JSON-RPC request must be an object", - )); - } - let id = value.get("id").cloned(); - let method = value - .get("method") - .and_then(Value::as_str) - .map(ToOwned::to_owned); - let params = value.get("params").cloned().unwrap_or(Value::Null); - Ok(RpcRequest { id, method, params }) -} - -pub fn decode_params(params: Value) -> Result -where - T: DeserializeOwned, -{ - serde_json::from_value(params).map_err(|error| { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::InvalidRequest, - error.to_string(), - ) - }) -} - -pub fn encode_result(value: T) -> Result -where - T: Serialize, -{ - serde_json::to_value(value).map_err(|error| { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::Internal, error.to_string()) - }) -} - -pub fn success_response(id: Value, result: Value) -> Value { - json!({ - "jsonrpc": "2.0", - "id": id, - "result": result - }) -} - -pub fn error_response(id: Option, error: EnvironmentProtocolError) -> Value { - json!({ - "jsonrpc": "2.0", - "id": id.unwrap_or(Value::Null), - "error": error - }) -} - -pub fn method_not_found(method: &str) -> EnvironmentProtocolError { - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Unsupported, - format!("unsupported environment-protocol method: {method}"), - ) -} - -pub fn invalid_request(message: impl Into) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::InvalidRequest, message) -} - -pub fn not_found(message: impl Into) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::NotFound, message) -} - -pub fn unsupported(message: impl Into) -> EnvironmentProtocolError { - EnvironmentProtocolError::new(EnvironmentProtocolErrorCode::Unsupported, message) -} diff --git a/crates/environment-daemon/src/server.rs b/crates/environment-daemon/src/server.rs index 21db1d868..eac083ebf 100644 --- a/crates/environment-daemon/src/server.rs +++ b/crates/environment-daemon/src/server.rs @@ -1,44 +1,135 @@ -use anyhow::Context as _; -use environment_protocol::data::{ - methods::{FS_CAPTURE_METHOD, FS_MATERIALIZE_METHOD}, - transfer::*, +//! Serving the data plane: the passive listener and the connection loop +//! every data socket runs, accepted or reverse-dialed. +//! +//! Per connection, one reader parses frames with the shared envelope and +//! enforces the handshake; one writer task owns the socket sink. Requests +//! after `initialize` run as their own tasks, up to +//! `limits.maxConcurrentRequests` at once, so responses leave in completion +//! order and the reader never waits on the writer. When the reader ends, +//! every pending request of the connection is cancelled and its reply, +//! with any after-write hook, is dropped. + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, }; + +use anyhow::Context as _; use environment_protocol::{ - data::{ - fs::{ - CopyParams, CreateDirectoryParams, GetMetadataParams, GlobFilesParams, - ReadDirectoryParams, ReadFileParams, RemoveParams, SearchTextParams, WriteFileParams, - }, - handshake::{InitializeParams, InitializeResponse, InitializedParams}, - idle::IdleParams, - jobs::{CancelJobsParams, ListJobsParams, ReadJobsParams, StartJobsParams}, - methods::{ - ENV_IDLE_METHOD, FS_COPY_METHOD, FS_CREATE_DIRECTORY_METHOD, FS_GET_METADATA_METHOD, - FS_GLOB_FILES_METHOD, FS_READ_DIRECTORY_METHOD, FS_READ_FILE_METHOD, FS_REMOVE_METHOD, - FS_SEARCH_TEXT_METHOD, FS_WRITE_FILE_METHOD, - INITIALIZE_METHOD as DATA_INITIALIZE_METHOD, INITIALIZED_METHOD, JOB_CANCEL_METHOD, - JOB_LIST_METHOD, JOB_READ_METHOD, JOB_START_METHOD, PROCESS_READ_METHOD, - PROCESS_RESIZE_METHOD, PROCESS_START_METHOD, PROCESS_TERMINATE_METHOD, - PROCESS_WRITE_METHOD, - }, - process::{ - ReadProcessParams, ResizeProcessParams, StartProcessParams, TerminateProcessParams, - WriteProcessParams, - }, + ENVD_PROTOCOL_VERSION, + error::{BusyReason, ProtocolError}, + jsonrpc::{ + self, CANCEL_REQUEST_METHOD, CancelRequestParams, Id, Message as RpcMessage, Request, + Response, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::CURRENT_PROTOCOL_VERSION, + limits::MAX_MESSAGE_BYTES, + manifest::DataInitialize, }; -use futures_util::{SinkExt as _, StreamExt as _}; -use serde_json::Value; -use tokio::net::TcpListener; -use tokio::sync::{Semaphore, mpsc}; -use tokio_tungstenite::{accept_async, tungstenite::Message}; - -/// Serve the daemon until a transport fails terminally: the passive listener -/// when one is configured, the outbound registration loop when a gateway -/// URL is configured, or both. -pub async fn run(runtime: DaemonRuntime) -> anyhow::Result<()> { +use futures_util::{Sink, SinkExt as _, StreamExt as _}; +use tokio::{ + net::TcpListener, + sync::{Semaphore, mpsc}, +}; +use tokio_tungstenite::{ + WebSocketStream, accept_async_with_config, + tungstenite::{Message, protocol::WebSocketConfig}, +}; +use tokio_util::sync::CancellationToken; + +use crate::{ + DaemonRuntime, + config::{ListenAddr, OnClosed}, + dispatch::{self, ConnectionInfo, RequestContext, WrittenHook}, +}; + +/// How the daemon process ends. Each outcome has its own exit status so an +/// orchestrator can tell a deliberate stop from a refusal or a cloned +/// identity. +#[derive(Debug)] +pub enum DaemonExit { + /// `SIGTERM` or `SIGINT`; a registered daemon said goodbye first, and + /// a parked one had already stopped registering. + Shutdown, + /// Local configuration, state or receipt-file error. + Configuration(anyhow::Error), + /// The gateway refused registration for good, or it speaks another + /// protocol and no automatic upgrade succeeded. + Refused(String), + /// The gateway closed the identity's environment, and the daemon is + /// configured to exit rather than park. + EnvironmentClosed, + /// Another process with this daemon identity connected and won. + Superseded { daemon_public_key: String }, + /// An unexpected failure. + Failed(anyhow::Error), +} + +impl DaemonExit { + pub fn status(&self) -> u8 { + match self { + Self::Shutdown => 0, + Self::Failed(_) => 1, + Self::Configuration(_) => 2, + Self::Refused(_) => 3, + Self::EnvironmentClosed => 4, + Self::Superseded { .. } => 5, + } + } +} + +impl std::fmt::Display for DaemonExit { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Shutdown => formatter.write_str("shut down"), + Self::Configuration(error) => write!(formatter, "configuration error: {error:#}"), + Self::Refused(message) => write!(formatter, "registration refused: {message}"), + Self::EnvironmentClosed => formatter.write_str("the environment was closed"), + Self::Superseded { daemon_public_key } => write!( + formatter, + "superseded: another process with daemon public key {daemon_public_key} connected; \ + two daemons share one state directory" + ), + Self::Failed(error) => write!(formatter, "failed: {error:#}"), + } + } +} + +/// The WebSocket limits of every envd socket: one message is one frame of +/// at most `MAX_MESSAGE_BYTES`. +pub(crate) fn websocket_config() -> WebSocketConfig { + let limit = usize::try_from(MAX_MESSAGE_BYTES).unwrap_or(usize::MAX); + WebSocketConfig::default() + .max_message_size(Some(limit)) + .max_frame_size(Some(limit)) +} + +/// Serve the daemon until it stops: the passive listener when one is +/// configured, the outbound registration loop when a gateway URL is +/// configured, or both. A daemon configured to park once its environment +/// is closed stops both and waits for a deliberate stop instead of exiting. +pub async fn run(runtime: DaemonRuntime) -> DaemonExit { + let shutdown = runtime.shutdown().clone(); + let on_closed = runtime + .config() + .registration + .as_ref() + .map_or(OnClosed::Exit, |registration| registration.on_closed); + match (serve(runtime).await, on_closed) { + (DaemonExit::EnvironmentClosed, OnClosed::Park) => park(&shutdown).await, + (exit, _) => exit, + } +} + +/// Wait for `SIGTERM` or `SIGINT` after the registration loop and the +/// listener have ended. Data connections already open run until then. +async fn park(shutdown: &CancellationToken) -> DaemonExit { + tracing_line("the environment was closed; parked until SIGTERM or SIGINT"); + shutdown.cancelled().await; + DaemonExit::Shutdown +} + +async fn serve(runtime: DaemonRuntime) -> DaemonExit { let listen = runtime.config().listen.clone(); let registration = runtime.config().registration.clone(); match (listen, registration) { @@ -47,18 +138,51 @@ pub async fn run(runtime: DaemonRuntime) -> anyhow::Result<()> { crate::registration::run_outbound(runtime, registration).await } (Some(listen), Some(registration)) => { - let listener = tokio::spawn(run_listener(runtime.clone(), listen)); - let outbound = tokio::spawn(crate::registration::run_outbound(runtime, registration)); + let listener = run_listener(runtime.clone(), listen); + let outbound = crate::registration::run_outbound(runtime, registration); + tokio::pin!(listener, outbound); tokio::select! { - result = listener => result.context("listener task")?, - result = outbound => result.context("registration task")?, + exit = &mut listener => match exit { + // The registration loop says goodbye on the same token. + DaemonExit::Shutdown => outbound.await, + other => other, + }, + exit = &mut outbound => exit, } } - (None, None) => anyhow::bail!("configure a listener or a gateway URL"), + (None, None) => { + DaemonExit::Configuration(anyhow::anyhow!("configure a listener or a gateway URL")) + } } } -async fn run_listener(runtime: DaemonRuntime, listen: ListenAddr) -> anyhow::Result<()> { +/// Cancel the runtime's shutdown token on the first `SIGTERM` or `SIGINT`. +pub fn spawn_signal_listener(runtime: &DaemonRuntime) -> anyhow::Result<()> { + let shutdown = runtime.shutdown().clone(); + #[cfg(unix)] + { + use tokio::signal::unix::{SignalKind, signal}; + let mut terminate = signal(SignalKind::terminate()).context("install SIGTERM handler")?; + let mut interrupt = signal(SignalKind::interrupt()).context("install SIGINT handler")?; + tokio::spawn(async move { + tokio::select! { + _ = terminate.recv() => tracing_line("SIGTERM received; shutting down"), + _ = interrupt.recv() => tracing_line("SIGINT received; shutting down"), + } + shutdown.cancel(); + }); + } + #[cfg(not(unix))] + tokio::spawn(async move { + if tokio::signal::ctrl_c().await.is_ok() { + tracing_line("interrupt received; shutting down"); + shutdown.cancel(); + } + }); + Ok(()) +} + +async fn run_listener(runtime: DaemonRuntime, listen: ListenAddr) -> DaemonExit { match listen { ListenAddr::Tcp(addr) => run_tcp_listener(runtime, addr).await, #[cfg(unix)] @@ -66,31 +190,43 @@ async fn run_listener(runtime: DaemonRuntime, listen: ListenAddr) -> anyhow::Res } } -async fn run_tcp_listener( - runtime: DaemonRuntime, - listen: std::net::SocketAddr, -) -> anyhow::Result<()> { - let listener = TcpListener::bind(listen) +async fn run_tcp_listener(runtime: DaemonRuntime, listen: std::net::SocketAddr) -> DaemonExit { + let listener = match TcpListener::bind(listen) .await - .context("bind environment daemon listener")?; + .context("bind environment daemon listener") + { + Ok(listener) => listener, + Err(error) => return DaemonExit::Configuration(error), + }; tracing_line(&format!("listener ready on {listen}")); + let shutdown = runtime.shutdown().clone(); loop { - match listener.accept().await { - Ok((stream, peer)) => serve_accepted(runtime.clone(), stream, peer.to_string()), - Err(error) => accept_failed(error).await, + tokio::select! { + _ = shutdown.cancelled() => return DaemonExit::Shutdown, + accepted = listener.accept() => match accepted { + Ok((stream, peer)) => serve_accepted(runtime.clone(), stream, peer.to_string()), + Err(error) => accept_failed(error).await, + }, } } } #[cfg(unix)] -async fn run_unix_listener(runtime: DaemonRuntime, path: std::path::PathBuf) -> anyhow::Result<()> { - let listener = bind_unix_listener(&path)?; +async fn run_unix_listener(runtime: DaemonRuntime, path: std::path::PathBuf) -> DaemonExit { + let listener = match bind_unix_listener(&path) { + Ok(listener) => listener, + Err(error) => return DaemonExit::Configuration(error), + }; let label = ListenAddr::Unix(path).to_string(); tracing_line(&format!("listener ready on {label}")); + let shutdown = runtime.shutdown().clone(); loop { - match listener.accept().await { - Ok((stream, _)) => serve_accepted(runtime.clone(), stream, label.clone()), - Err(error) => accept_failed(error).await, + tokio::select! { + _ = shutdown.cancelled() => return DaemonExit::Shutdown, + accepted = listener.accept() => match accepted { + Ok((stream, _)) => serve_accepted(runtime.clone(), stream, label.clone()), + Err(error) => accept_failed(error).await, + }, } } } @@ -100,7 +236,7 @@ async fn run_unix_listener(runtime: DaemonRuntime, path: std::path::PathBuf) -> /// The pause stops a persistent error from spinning the loop. async fn accept_failed(error: std::io::Error) { tracing_line(&format!("accept failed: {error}")); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(100)).await; } /// Bind a Unix-socket listener only this daemon's user can connect to. A @@ -154,465 +290,840 @@ where S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, { tokio::spawn(async move { - match accept_async(stream).await { - Ok(socket) => { - if let Err(error) = run_data_connection(&runtime, socket).await { - eprintln!("lightspeed-envd connection {peer} failed: {error:#}"); - } - } - Err(error) => { - eprintln!("lightspeed-envd websocket handshake {peer} failed: {error}") - } + if let Err(error) = serve_stream(runtime, stream).await { + tracing_line(&format!("connection {peer} failed: {error:#}")); } }); } +/// Accept a WebSocket upgrade on an already connected byte stream and serve +/// the data plane on it until the connection ends. In-process callers pair +/// it with `DataClient::connect_initialized_over` on the other end of a +/// `tokio::io::duplex`. +pub async fn serve_stream(runtime: DaemonRuntime, stream: S) -> anyhow::Result<()> +where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, +{ + let socket = accept_async_with_config(stream, Some(websocket_config())) + .await + .context("websocket handshake")?; + run_data_connection(&runtime, socket).await; + Ok(()) +} + +/// One frame for the writer task: an encoded response and the hook to run +/// once it is on the socket. +struct Outgoing { + text: String, + on_written: Option, +} + +impl Outgoing { + fn response(response: Response) -> Self { + Self { + text: response.to_text(), + on_written: None, + } + } +} + +/// Pending requests of one connection by id, each with the token that +/// `$/cancelRequest` and the connection's close cancel. +type PendingRequests = Arc>>; + /// Serve the data protocol on one accepted or dialed socket. Outbound data /// sockets and passive connections run this same loop. -pub(crate) async fn run_data_connection( - runtime: &DaemonRuntime, - socket: tokio_tungstenite::WebSocketStream, -) -> anyhow::Result<()> +pub(crate) async fn run_data_connection(runtime: &DaemonRuntime, socket: WebSocketStream) where S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, { - let (mut writer, mut reader) = socket.split(); - let (responses, mut response_rx) = mpsc::channel::(64); - let concurrency = std::sync::Arc::new(Semaphore::new(32)); + let connection = Arc::new(ConnectionInfo { + connection_id: runtime.next_connection_id(), + closed: CancellationToken::new(), + }); + let max_concurrent = runtime.limits().max_concurrent_requests as usize; + // Handler replies are bounded by the request permits; the rest of the + // capacity holds the reader's own answers (framing errors, busy). A + // peer that fills it without reading is closed rather than buffered. + let (outgoing, outgoing_rx) = mpsc::channel::(max_concurrent + 64); + let (sink, mut stream) = socket.split(); + let writer = tokio::spawn(write_loop(sink, outgoing_rx, connection.closed.clone())); + let permits = Arc::new(Semaphore::new(max_concurrent)); + let pending: PendingRequests = Arc::default(); + let shutdown = runtime.shutdown().clone(); + let mut session = Session { + runtime, + connection: &connection, + outgoing: &outgoing, + permits: &permits, + pending: &pending, + initialized: false, + }; loop { - tokio::select! { - response = response_rx.recv() => { - let Some(response) = response else { return Ok(()) }; - writer.send(Message::Text(response.to_string().into())).await?; - } - message = reader.next() => { - let Some(message) = message else { return Ok(()) }; - match message? { - Message::Text(text) => dispatch_gateway_json(runtime, &responses, &concurrency, &text).await?, - Message::Binary(bytes) => { - let text = std::str::from_utf8(&bytes).context("binary provider frame is not UTF-8")?; - dispatch_gateway_json(runtime, &responses, &concurrency, text).await?; - } - Message::Ping(bytes) => writer.send(Message::Pong(bytes)).await?, - Message::Pong(_) | Message::Frame(_) => {} - Message::Close(_) => { - // Tungstenite queues the close reply while reading the - // peer's frame; flush it before dropping the socket. - writer.flush().await?; - return Ok(()) - }, - } + let frame = tokio::select! { + _ = connection.closed.cancelled() => break, + _ = shutdown.cancelled() => break, + frame = stream.next() => frame, + }; + let message = match frame { + Some(Ok(message)) => message, + Some(Err(error)) => { + tracing_line(&format!( + "connection {} ended: {error}", + connection.connection_id + )); + break; } + None => break, + }; + let keep_open = match message { + Message::Text(text) => session.on_text(text.as_str()), + Message::Binary(_) => session.answer(Response::failure( + None, + ProtocolError::invalid_request("binary frames are not part of the protocol"), + )), + // Tungstenite answers pings itself on the read path, ahead of + // anything the writer has queued. + Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => true, + Message::Close(_) => break, + }; + if !keep_open { + break; } } + connection.closed.cancel(); + pending + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .clear(); + drop(outgoing); + let _ = writer.await; } -use crate::{ - DaemonRuntime, - config::ListenAddr, - rpc::{ - decode_params, encode_result, error_response, invalid_request, method_not_found, - parse_request, success_response, - }, -}; +/// The reader's view of one connection. +struct Session<'a> { + runtime: &'a DaemonRuntime, + connection: &'a Arc, + outgoing: &'a mpsc::Sender, + permits: &'a Arc, + pending: &'a PendingRequests, + initialized: bool, +} -async fn dispatch_gateway_json( - runtime: &DaemonRuntime, - responses: &mpsc::Sender, - concurrency: &std::sync::Arc, - text: &str, -) -> anyhow::Result<()> { - let value: Value = serde_json::from_str(text)?; - let request = match parse_request(value) { - Ok(request) => request, - Err(error) => { - responses.send(error_response(None, error)).await?; - return Ok(()); +impl Session<'_> { + /// Handle one text frame; false closes the connection. + fn on_text(&mut self, text: &str) -> bool { + match jsonrpc::parse(text) { + Err(rejection) => match rejection.into_response() { + Some(response) => self.answer(response), + None => true, + }, + Ok(RpcMessage::Request(request)) => self.on_request(request), + Ok(RpcMessage::Notification(notification)) => { + if self.initialized + && notification.method == CANCEL_REQUEST_METHOD + && let Ok(params) = notification.params::() + { + self.cancel(¶ms.id); + } + true + } + // envd never sends requests, so no response can be one it + // waits for. + Ok(RpcMessage::Response(response)) => { + tracing_line(&format!( + "connection {} ignored a response to id {}", + self.connection.connection_id, + response + .id + .map_or_else(|| "null".to_owned(), |id| id.to_string()) + )); + true + } } - }; - let Some(id) = request.id else { - if request.method.as_deref() == Some(INITIALIZED_METHOD) { - let _ = decode_params::(request.params); + } + + fn on_request(&mut self, request: Request) -> bool { + if !self.initialized { + if request.method != DataInitialize::NAME { + return self.answer(Response::failure( + Some(request.id), + ProtocolError::not_initialized(), + )); + } + let result = initialize(self.runtime, &request); + self.initialized = result.is_ok(); + return self.answer(Response::from_result(request.id, result)); } - return Ok(()); - }; - let Ok(permit) = concurrency.clone().try_acquire_owned() else { - responses - .send(error_response( - Some(id), - EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Conflict, - "daemon request limit reached", + if request.method == DataInitialize::NAME { + return self.answer(Response::failure( + Some(request.id), + ProtocolError::invalid_request( + "initialize was already answered on this connection", ), - )) - .await?; - return Ok(()); - }; - let runtime = runtime.clone(); - let responses = responses.clone(); - tokio::spawn(async move { - let _permit = permit; - let result = match request.method.as_deref() { - Some(method) => handle_data(&runtime, method, request.params).await, - None => Err(invalid_request("request missing method")), - }; - let response = match result { - Ok(result) => success_response(id, result), - Err(error) => error_response(Some(id), error), - }; - let _ = responses.send(response).await; - }); - Ok(()) -} - -async fn handle_data( - runtime: &DaemonRuntime, - method: &str, - params: Value, -) -> Result { - // Handshakes and idle reports are observation, not use: only real - // filesystem/process/job requests count as activity. - if !matches!(method, DATA_INITIALIZE_METHOD | ENV_IDLE_METHOD) { - runtime.touch_activity(); - } - match method { - ENV_IDLE_METHOD => { - decode_params::(params)?; - encode_result(runtime.idle_report().await) + )); } - DATA_INITIALIZE_METHOD => { - let params = decode_params::(params)?; - if params.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(EnvironmentProtocolError::new( - EnvironmentProtocolErrorCode::Unsupported, + let Ok(permit) = self.permits.clone().try_acquire_owned() else { + return self.answer(Response::failure( + Some(request.id), + ProtocolError::busy( + Some(BusyReason::RequestLimit), + None, format!( - "unsupported data protocol version {}; expected {CURRENT_PROTOCOL_VERSION}", - params.protocol_version + "{} requests are already in flight on this connection", + self.runtime.limits().max_concurrent_requests ), + ), + )); + }; + let ctx = RequestContext::new(self.connection.clone()); + { + let mut pending = self + .pending + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if pending.contains_key(&request.id) { + drop(pending); + return self.answer(Response::failure( + Some(request.id), + ProtocolError::invalid_request("a request with this id is already in flight"), )); } - encode_result(InitializeResponse { - home_directory: std::env::var("HOME").ok(), - protocol_version: CURRENT_PROTOCOL_VERSION, - connection_id: runtime.next_connection_id(), - capabilities: runtime.capabilities(), - default_cwd: Some(runtime.config().cwd.to_string_lossy().into_owned()), - implementation: runtime.implementation(), - }) + pending.insert(request.id.clone(), ctx.cancel.clone()); + } + let runtime = self.runtime.clone(); + let outgoing = self.outgoing.clone(); + let pending = self.pending.clone(); + tokio::spawn(async move { + let _permit = permit; + let Request { id, method, params } = request; + let result = dispatch::handle_request(&runtime, &ctx, &method, params).await; + pending + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .remove(&id); + if ctx.connection.closed.is_cancelled() { + return; + } + let _ = outgoing.send(encode_reply(id, result)).await; + }); + true + } + + fn cancel(&self, id: &Id) { + let pending = self + .pending + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if let Some(token) = pending.get(id) { + token.cancel(); } - FS_READ_FILE_METHOD => encode_result( - runtime - .filesystem() - .read_file(decode_params::(params)?) - .await?, - ), - FS_WRITE_FILE_METHOD => encode_result( - runtime - .filesystem() - .write_file(decode_params::(params)?) - .await?, - ), - FS_CREATE_DIRECTORY_METHOD => encode_result( - runtime - .filesystem() - .create_directory(decode_params::(params)?) - .await?, - ), - FS_GET_METADATA_METHOD => encode_result( - runtime - .filesystem() - .get_metadata(decode_params::(params)?) - .await?, - ), - FS_READ_DIRECTORY_METHOD => encode_result( - runtime - .filesystem() - .read_directory(decode_params::(params)?) - .await?, - ), - FS_REMOVE_METHOD => encode_result( - runtime - .filesystem() - .remove(decode_params::(params)?) - .await?, - ), - environment_protocol::data::methods::FS_SCAN_METHOD => { - encode_result(runtime.filesystem().scan(decode_params(params)?).await?) + } + + /// Queue a reader-produced answer without waiting; false when the peer + /// stopped reading and the queue is full. + fn answer(&self, response: Response) -> bool { + match self.outgoing.try_send(Outgoing::response(response)) { + Ok(()) => true, + Err(mpsc::error::TrySendError::Full(_)) => { + tracing_line(&format!( + "connection {} closed: the peer does not read its responses", + self.connection.connection_id + )); + false + } + Err(mpsc::error::TrySendError::Closed(_)) => false, + } + } +} + +/// Encode a handler's result. A success whose encoding would break the +/// message limit becomes `payloadTooLarge`, and its after-write hook is +/// dropped with it: nothing the client never received may be committed. +fn encode_reply( + id: Id, + result: Result, ProtocolError>, +) -> Outgoing { + match result { + Ok(reply) => { + let response = Response { + id: Some(id), + result: Ok(reply.value), + }; + let text = response.to_value().to_string(); + if jsonrpc::check_message_size(&text).is_ok() { + Outgoing { + text, + on_written: reply.on_written, + } + } else { + Outgoing::response(response) + } + } + Err(error) => Outgoing::response(Response::failure(Some(id), error)), + } +} + +/// `initialize`: the version is compared before the strict decode, so a +/// client of another version learns that rather than which field it sent +/// that this daemon does not know. +fn initialize( + runtime: &DaemonRuntime, + request: &Request, +) -> Result { + if let Some(received) = request + .params + .get("protocolVersion") + .and_then(serde_json::Value::as_u64) + && received != u64::from(ENVD_PROTOCOL_VERSION) + { + return Err(ProtocolError::version_mismatch( + ENVD_PROTOCOL_VERSION, + u32::try_from(received).unwrap_or(u32::MAX), + )); + } + let _params: environment_protocol::data::handshake::InitializeParams = request.params()?; + Ok(runtime.initialize_response()) +} + +/// The connection's only writer: sends frames in the order handlers finish +/// and runs each after-write hook once its frame is on the socket. Stops, +/// dropping whatever is still queued, when the connection closes. +async fn write_loop( + mut sink: Si, + mut queue: mpsc::Receiver, + closed: CancellationToken, +) where + Si: Sink + Unpin, +{ + loop { + let next = tokio::select! { + biased; + _ = closed.cancelled() => break, + next = queue.recv() => next, + }; + let Some(Outgoing { text, on_written }) = next else { + break; + }; + let sent = tokio::select! { + biased; + _ = closed.cancelled() => break, + sent = sink.send(Message::Text(text.into())) => sent, + }; + match sent { + Ok(()) => { + if let Some(hook) = on_written { + hook(); + } + } + Err(_) => { + closed.cancel(); + break; + } } - environment_protocol::data::methods::FS_TRANSFER_METHOD => encode_result( - runtime - .filesystem() - .transfer(decode_params(params)?) - .await?, - ), - FS_CAPTURE_METHOD => encode_result( - runtime - .filesystem() - .capture(decode_params::(params)?) - .await?, - ), - FS_MATERIALIZE_METHOD => encode_result( - runtime - .filesystem() - .materialize(decode_params::(params)?) - .await?, - ), - FS_COPY_METHOD => encode_result( - runtime - .filesystem() - .copy(decode_params::(params)?) - .await?, - ), - FS_SEARCH_TEXT_METHOD => encode_result( - runtime - .filesystem() - .search_text(decode_params::(params)?) - .await?, - ), - FS_GLOB_FILES_METHOD => encode_result( - runtime - .filesystem() - .glob_files(decode_params::(params)?) - .await?, - ), - PROCESS_START_METHOD => encode_result( - runtime - .processes() - .start_process(decode_params::(params)?) - .await?, - ), - PROCESS_READ_METHOD => encode_result( - runtime - .processes() - .read_process(decode_params::(params)?) - .await?, - ), - PROCESS_WRITE_METHOD => encode_result( - runtime - .processes() - .write_process(decode_params::(params)?) - .await?, - ), - PROCESS_TERMINATE_METHOD => encode_result( - runtime - .processes() - .terminate_process(decode_params::(params)?) - .await?, - ), - PROCESS_RESIZE_METHOD => encode_result( - runtime - .processes() - .resize_process(decode_params::(params)?) - .await?, - ), - JOB_START_METHOD => encode_result( - runtime - .jobs() - .start_jobs(decode_params::(params)?) - .await?, - ), - JOB_LIST_METHOD => encode_result( - runtime - .jobs() - .list_jobs(decode_params::(params)?) - .await?, - ), - JOB_READ_METHOD => encode_result( - runtime - .jobs() - .read_jobs(decode_params::(params)?) - .await?, - ), - JOB_CANCEL_METHOD => encode_result( - runtime - .jobs() - .cancel_jobs(decode_params::(params)?) - .await?, - ), - other => Err(method_not_found(other)), } + // Sends our close frame, or the reply tungstenite queued for the peer's. + let _ = tokio::time::timeout(Duration::from_secs(5), sink.close()).await; } pub(crate) fn tracing_line(message: &str) { eprintln!("lightspeed-envd {message}"); } -#[cfg(all(test, unix))] -mod unix_listener_tests { - use std::os::unix::fs::PermissionsExt as _; +#[cfg(test)] +mod tests { + use std::time::Duration; - use environment_protocol::{data::handshake::InitializeParams, shared::EnvironmentScope}; - use tokio_tungstenite::client_async; + use environment_protocol::{ + error::{ErrorData, ProtocolErrorKind}, + handshake::ImplementationInfo, + }; + use serde_json::{Value, json}; + use tokio::io::DuplexStream; + use tokio_tungstenite::client_async_with_config; use super::*; - use crate::config::DaemonConfig; - - fn runtime(root: &std::path::Path, socket: &std::path::Path) -> DaemonRuntime { - DaemonRuntime::new(DaemonConfig { - listen: Some(ListenAddr::Unix(socket.to_path_buf())), - cwd: root.to_path_buf(), - fs_root: root.to_path_buf(), - state_dir: root.join("state"), - read_only_fs: false, - registration: None, - scrubbed_env: vec![], - }) - .unwrap() + use crate::test_support::runtime; + + type Client = WebSocketStream; + + async fn connect(runtime: &DaemonRuntime) -> Client { + let (client, server) = tokio::io::duplex(1 << 20); + tokio::spawn(serve_stream(runtime.clone(), server)); + client_async_with_config("ws://localhost/", client, Some(websocket_config())) + .await + .expect("upgrade") + .0 } - #[tokio::test(flavor = "current_thread")] - async fn serves_the_data_protocol_on_a_private_unix_socket() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); - // The parent does not exist yet: guest tmpfs paths start empty. - let socket = root.join("run").join("envd.sock"); - let server = tokio::spawn(run(runtime(&root, &socket))); - while !socket.exists() { - tokio::task::yield_now().await; - } - let mode = std::fs::metadata(&socket).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o600); - - let stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); - let (mut client, _) = client_async("ws://envd/", stream).await.unwrap(); - let params = InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "unix-listener-test".to_owned(), - scope: EnvironmentScope::Default, - resume_connection_id: None, - }; - let request = serde_json::json!({ - "jsonrpc": "2.0", - "id": 1, - "method": DATA_INITIALIZE_METHOD, - "params": params, - }); + async fn send(client: &mut Client, frame: Value) { client - .send(Message::Text(request.to_string().into())) + .send(Message::Text(frame.to_string().into())) .await - .unwrap(); - let Some(Ok(Message::Text(reply))) = client.next().await else { - panic!("expected an initialize response"); - }; - let reply: Value = serde_json::from_str(reply.as_str()).unwrap(); - assert_eq!(reply["id"], 1); - assert_eq!(reply["result"]["protocolVersion"], CURRENT_PROTOCOL_VERSION); - server.abort(); + .expect("send"); } - #[tokio::test(flavor = "current_thread")] - async fn stale_sockets_are_replaced_but_live_ones_and_files_are_not() { - let temp = tempfile::tempdir().unwrap(); - let socket = temp.path().join("envd.sock"); + async fn next_json(client: &mut Client) -> Value { + loop { + let frame = tokio::time::timeout(Duration::from_secs(10), client.next()) + .await + .expect("a frame in time") + .expect("open") + .expect("frame"); + if let Message::Text(text) = frame { + return serde_json::from_str(text.as_str()).expect("json"); + } + } + } - drop(std::os::unix::net::UnixListener::bind(&socket).unwrap()); - assert!(socket.exists(), "a dropped listener leaves its node behind"); - let listener = bind_unix_listener(&socket).expect("stale socket replaced"); + fn error_kind(response: &Value) -> ProtocolErrorKind { + let error: ProtocolError = + serde_json::from_value(response["error"].clone()).expect("an error response"); + error.kind() + } - assert!(bind_unix_listener(&socket).is_err(), "live listener kept"); - drop(listener); + fn initialize_params(version: u32) -> Value { + json!({ + "protocolVersion": version, + "clientInfo": {"name": "envd-server-test"}, + }) + } - let file = temp.path().join("not-a-socket"); - std::fs::write(&file, b"data").unwrap(); - assert!(bind_unix_listener(&file).is_err()); - assert_eq!(std::fs::read(&file).unwrap(), b"data"); + async fn initialized(runtime: &DaemonRuntime) -> Client { + let mut client = connect(runtime).await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": initialize_params(ENVD_PROTOCOL_VERSION)}), + ) + .await; + let response = next_json(&mut client).await; + assert_eq!(response["result"]["plane"], "data", "{response}"); + client } #[tokio::test(flavor = "current_thread")] - async fn a_live_socket_that_cannot_be_probed_is_left_in_place() { - let temp = tempfile::tempdir().unwrap(); - let socket = temp.path().join("envd.sock"); - let _live = std::os::unix::net::UnixListener::bind(&socket).unwrap(); - // Without write permission, connect fails with EACCES rather than - // ECONNREFUSED (root bypasses this and sees the live listener). - std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o000)).unwrap(); - - assert!(bind_unix_listener(&socket).is_err()); - assert!( - std::fs::symlink_metadata(&socket).is_ok(), - "live socket kept" + async fn the_handshake_is_enforced_and_the_connection_survives_every_refusal() { + let (_temp, runtime) = runtime(); + let mut client = connect(&runtime).await; + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 1, "method": "env/getIdle"}), + ) + .await; + let early = next_json(&mut client).await; + assert_eq!(early["id"], 1); + assert_eq!(error_kind(&early), ProtocolErrorKind::NotInitialized); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 2, "method": "initialize", "params": {"protocolVersion": ENVD_PROTOCOL_VERSION + 1, "clientInfo": {"name": "x"}, "fromTheFuture": true}}), + ) + .await; + let mismatch: ProtocolError = + serde_json::from_value(next_json(&mut client).await["error"].clone()).unwrap(); + assert_eq!( + mismatch.data, + ErrorData::VersionMismatch { + expected: ENVD_PROTOCOL_VERSION, + received: ENVD_PROTOCOL_VERSION + 1 + } ); - } -} -#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] -mod transfer_tests { - use super::*; - use crate::config::DaemonConfig; - use environment_protocol::shared::{ByteChunk, EnvironmentPath}; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 3, "method": "initialize", "params": initialize_params(ENVD_PROTOCOL_VERSION)}), + ) + .await; + let response = next_json(&mut client).await; + assert_eq!(response["id"], 3); + assert_eq!(response["result"]["protocolVersion"], ENVD_PROTOCOL_VERSION); + assert_eq!( + response["result"]["daemonInstanceId"], + runtime.instance_id().as_str() + ); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 4, "method": "initialize", "params": initialize_params(ENVD_PROTOCOL_VERSION)}), + ) + .await; + assert_eq!( + error_kind(&next_json(&mut client).await), + ProtocolErrorKind::InvalidRequest + ); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 5, "method": "env/getIdle"}), + ) + .await; + let idle = next_json(&mut client).await; + assert!(idle["result"]["idleForMs"].is_u64(), "{idle}"); + } #[tokio::test(flavor = "current_thread")] - async fn transfer_dispatch_and_read_only_capabilities() { - let temp = tempfile::tempdir().unwrap(); - let root = temp.path().to_path_buf(); - for read_only in [false, true] { - let runtime = DaemonRuntime::new(DaemonConfig { - listen: None, - cwd: root.clone(), - fs_root: root.clone(), - state_dir: root.join("state"), - read_only_fs: read_only, - registration: None, - scrubbed_env: vec![], - }) + async fn framing_errors_are_answered_with_null_ids_and_notifications_never() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + + client.send(Message::Text("not json".into())).await.unwrap(); + let parse = next_json(&mut client).await; + assert_eq!(parse["id"], Value::Null); + assert_eq!(error_kind(&parse), ProtocolErrorKind::ParseError); + + client + .send(Message::Binary(b"{}".to_vec().into())) + .await .unwrap(); - assert!(runtime.capabilities().filesystem_capture); - assert_eq!(runtime.capabilities().filesystem_materialize, !read_only); - let limits = TransferLimits { - max_entries: 1, - max_depth: 0, - max_file_bytes: 2, - max_total_bytes: 2, - // This checks dispatch and access, not storage latency. Allow - // shared-runner delays across the transfer's journal fsyncs. - max_duration_ms: MAX_TRANSFER_DURATION_MS, - }; - let entries = vec![TransferEntry { - path: "".into(), - content: TransferContent::File { - data: ByteChunk::from(vec![0, 255]), - executable: true, - }, - }]; - let request = MaterializeParams { - destination: EnvironmentPath::new("binary").unwrap(), - entries: entries.clone(), - limits, - on_existing: TransferOnExisting::Replace, - }; - let result = handle_data( + let binary = next_json(&mut client).await; + assert_eq!(binary["id"], Value::Null); + assert_eq!(error_kind(&binary), ProtocolErrorKind::InvalidRequest); + + // A malformed notification and an unknown one: no answer at all. + send( + &mut client, + json!({"jsonrpc": "2.0", "method": "$/cancelRequest", "params": [1]}), + ) + .await; + send( + &mut client, + json!({"jsonrpc": "2.0", "method": "x/unknown"}), + ) + .await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "after", "method": "nope/nothing"}), + ) + .await; + let next = next_json(&mut client).await; + assert_eq!(next["id"], "after"); + assert_eq!(error_kind(&next), ProtocolErrorKind::MethodNotFound); + } + + #[tokio::test(flavor = "current_thread")] + async fn params_are_decoded_strictly_and_name_the_field() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": 9, "method": "env/getIdle", "params": {"unexpected": 1}}), + ) + .await; + let response = next_json(&mut client).await; + assert_eq!(error_kind(&response), ProtocolErrorKind::InvalidParams); + } + + /// Every data request in the manifest reaches a handler: none falls + /// through to `methodNotFound`. + #[tokio::test(flavor = "current_thread")] + async fn every_manifest_request_is_dispatched() { + use environment_protocol::{ + handshake::Plane, + manifest::{self, Direction, MessageKind}, + }; + let (_temp, runtime) = runtime(); + for spec in manifest::plane_methods(Plane::Data).filter(|spec| { + spec.kind == MessageKind::Request && spec.direction == Direction::ClientToServer + }) { + let result = dispatch::handle_request( &runtime, - FS_MATERIALIZE_METHOD, - serde_json::to_value(request).unwrap(), + &RequestContext::detached(), + spec.name, + json!({"__unknownMember": true}), ) .await; - if read_only { - assert_eq!( - result.unwrap_err().code, - EnvironmentProtocolErrorCode::CapabilityUnavailable + if let Err(error) = result { + assert_ne!( + error.kind(), + ProtocolErrorKind::MethodNotFound, + "{} is not dispatched", + spec.name ); - } else { - assert_eq!(result.unwrap()["bytes"], 2); } - let captured: CaptureResponse = serde_json::from_value( - handle_data( - &runtime, - FS_CAPTURE_METHOD, - serde_json::to_value(CaptureParams { - source: EnvironmentPath::new("binary").unwrap(), - limits, - }) - .unwrap(), - ) - .await - .unwrap(), + } + } + + #[tokio::test(flavor = "current_thread")] + async fn requests_are_pipelined_and_abortable_waits_are_cancelled() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "start", "method": "process/start", "params": { + "processId": "quiet", + "command": {"argv": ["/bin/sh", "-c", "sleep 30"], "timeoutMs": 60000}, + }}), + ) + .await; + assert!(next_json(&mut client).await["result"]["pid"].is_u64()); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "wait", "method": "process/read", "params": { + "processId": "quiet", "maxBytes": 1024, "waitMs": 30000, "returnOn": "exit", + }}), + ) + .await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "idle", "method": "env/getIdle"}), + ) + .await; + let first = next_json(&mut client).await; + assert_eq!( + first["id"], "idle", + "a later request answers first: {first}" + ); + + send( + &mut client, + json!({"jsonrpc": "2.0", "method": "$/cancelRequest", "params": {"id": "wait"}}), + ) + .await; + let cancelled = next_json(&mut client).await; + assert_eq!(cancelled["id"], "wait"); + let error: ProtocolError = serde_json::from_value(cancelled["error"].clone()).unwrap(); + assert_eq!(error, ProtocolError::cancelled()); + + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "kill", "method": "process/terminate", "params": {"processId": "quiet", "signal": "kill"}}), + ) + .await; + assert_eq!(next_json(&mut client).await["id"], "kill"); + } + + #[tokio::test(flavor = "current_thread")] + async fn requests_beyond_the_concurrency_ceiling_are_busy() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "start", "method": "process/start", "params": { + "processId": "quiet", + "command": {"argv": ["/bin/sh", "-c", "sleep 30"], "timeoutMs": 60000}, + }}), + ) + .await; + next_json(&mut client).await; + let ceiling = runtime.limits().max_concurrent_requests; + for index in 0..=ceiling { + send( + &mut client, + json!({"jsonrpc": "2.0", "id": index, "method": "process/read", "params": { + "processId": "quiet", "after": 0, "maxBytes": 0, "waitMs": 30000, "returnOn": "exit", + }}), ) - .unwrap(); - assert_eq!(captured.entries, entries); + .await; } + let busy = next_json(&mut client).await; + assert_eq!(busy["id"], ceiling); + let error: ProtocolError = serde_json::from_value(busy["error"].clone()).unwrap(); + assert!(matches!( + error.data, + ErrorData::Busy { + reason: Some(BusyReason::RequestLimit), + .. + } + )); + send( + &mut client, + json!({"jsonrpc": "2.0", "method": "$/cancelRequest", "params": {"id": 0}}), + ) + .await; + assert_eq!(next_json(&mut client).await["id"], 0); + send( + &mut client, + json!({"jsonrpc": "2.0", "id": "kill", "method": "process/terminate", "params": {"processId": "quiet", "signal": "kill"}}), + ) + .await; } -} -#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] -#[path = "filesystem/transfer/tests.rs"] -mod streaming_transfer_tests; + /// A cursor read whose connection drops is cancelled and consumes + /// nothing: the next connection's cursor read still sees every byte. + #[tokio::test(flavor = "current_thread")] + async fn a_dropped_connection_consumes_no_output() { + let (_temp, runtime) = runtime(); + let mut first = initialized(&runtime).await; + send( + &mut first, + json!({"jsonrpc": "2.0", "id": 1, "method": "process/start", "params": { + "processId": "late", + "command": {"argv": ["/bin/sh", "-c", "printf early; sleep 1; printf late"], "timeoutMs": 60000}, + }}), + ) + .await; + next_json(&mut first).await; + send( + &mut first, + json!({"jsonrpc": "2.0", "id": 2, "method": "process/read", "params": { + "processId": "late", "maxBytes": 1024, "waitMs": 30000, "returnOn": "exit", + }}), + ) + .await; + tokio::time::sleep(Duration::from_millis(200)).await; + drop(first); + + tokio::time::sleep(Duration::from_millis(1500)).await; + let mut second = initialized(&runtime).await; + send( + &mut second, + json!({"jsonrpc": "2.0", "id": 3, "method": "process/read", "params": { + "processId": "late", "maxBytes": 1024, "waitMs": 10000, "returnOn": "exit", + }}), + ) + .await; + let page = next_json(&mut second).await; + let text: String = page["result"]["output"]["chunks"] + .as_array() + .expect("chunks") + .iter() + .map(|chunk| { + let bytes: environment_protocol::bytes::ByteChunk = + serde_json::from_value(chunk["data"].clone()).unwrap(); + String::from_utf8(bytes.into_inner()).unwrap() + }) + .collect(); + assert_eq!(text, "earlylate", "{page}"); + } + + #[tokio::test(flavor = "current_thread")] + async fn the_client_library_completes_the_handshake() { + let (_temp, runtime) = runtime(); + let (client_io, server_io) = tokio::io::duplex(1 << 20); + tokio::spawn(serve_stream(runtime.clone(), server_io)); + let options = environment_client::ConnectOptions::new( + ImplementationInfo::new("envd-server-test").unwrap(), + ); + let (client, initialize) = + environment_client::DataClient::connect_initialized_over(client_io, options) + .await + .expect("initialized"); + assert_eq!(&initialize.daemon_instance_id, runtime.instance_id()); + client + .get_idle(&environment_protocol::data::idle::GetIdleParams {}) + .await + .expect("idle"); + } + + #[tokio::test(flavor = "current_thread")] + async fn shutdown_ends_listeners_and_connections() { + let (_temp, runtime) = runtime(); + let mut client = initialized(&runtime).await; + runtime.shutdown().cancel(); + let ended = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match client.next().await { + None | Some(Err(_)) | Some(Ok(Message::Close(_))) => break, + Some(Ok(_)) => continue, + } + } + }) + .await; + assert!(ended.is_ok(), "the daemon closed the connection"); + } + + #[cfg(unix)] + mod unix_listener { + use std::os::unix::fs::PermissionsExt as _; + + use super::*; + use crate::test_support::runtime_with; + + #[tokio::test(flavor = "current_thread")] + async fn serves_the_data_protocol_on_a_private_unix_socket() { + let socket_dir = tempfile::tempdir().unwrap(); + // The parent does not exist yet: guest tmpfs paths start empty. + let socket = socket_dir.path().join("run").join("envd.sock"); + let (_temp, runtime) = runtime_with(|config| { + config.listen = Some(ListenAddr::Unix(socket.clone())); + }); + let server = tokio::spawn(run(runtime.clone())); + while !socket.exists() { + tokio::task::yield_now().await; + } + let mode = std::fs::metadata(&socket).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + + let stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); + let (mut client, _) = + client_async_with_config("ws://envd/", stream, Some(websocket_config())) + .await + .unwrap(); + client + .send(Message::Text( + json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": initialize_params(ENVD_PROTOCOL_VERSION)}) + .to_string() + .into(), + )) + .await + .unwrap(); + let Some(Ok(Message::Text(reply))) = client.next().await else { + panic!("expected an initialize response"); + }; + let reply: Value = serde_json::from_str(reply.as_str()).unwrap(); + assert_eq!(reply["id"], 1); + assert_eq!(reply["result"]["protocolVersion"], ENVD_PROTOCOL_VERSION); + + runtime.shutdown().cancel(); + let exit = tokio::time::timeout(Duration::from_secs(5), server) + .await + .expect("listener stops") + .unwrap(); + assert_eq!(exit.status(), 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn stale_sockets_are_replaced_but_live_ones_and_files_are_not() { + let temp = tempfile::tempdir().unwrap(); + let socket = temp.path().join("envd.sock"); + + drop(std::os::unix::net::UnixListener::bind(&socket).unwrap()); + assert!(socket.exists(), "a dropped listener leaves its node behind"); + let listener = bind_unix_listener(&socket).expect("stale socket replaced"); + + assert!(bind_unix_listener(&socket).is_err(), "live listener kept"); + drop(listener); + + let file = temp.path().join("not-a-socket"); + std::fs::write(&file, b"data").unwrap(); + assert!(bind_unix_listener(&file).is_err()); + assert_eq!(std::fs::read(&file).unwrap(), b"data"); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_live_socket_that_cannot_be_probed_is_left_in_place() { + let temp = tempfile::tempdir().unwrap(); + let socket = temp.path().join("envd.sock"); + let _live = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + // Without write permission, connect fails with EACCES rather than + // ECONNREFUSED (root bypasses this and sees the live listener). + std::fs::set_permissions(&socket, std::fs::Permissions::from_mode(0o000)).unwrap(); + + assert!(bind_unix_listener(&socket).is_err()); + assert!( + std::fs::symlink_metadata(&socket).is_ok(), + "live socket kept" + ); + } + } + + #[test] + fn exit_statuses_are_distinct() { + let statuses = [ + DaemonExit::Shutdown.status(), + DaemonExit::Failed(anyhow::anyhow!("x")).status(), + DaemonExit::Configuration(anyhow::anyhow!("x")).status(), + DaemonExit::Refused("x".to_owned()).status(), + DaemonExit::EnvironmentClosed.status(), + DaemonExit::Superseded { + daemon_public_key: "k".to_owned(), + } + .status(), + ]; + let unique: std::collections::BTreeSet<_> = statuses.iter().collect(); + assert_eq!(unique.len(), statuses.len()); + assert_eq!(statuses, [0, 1, 2, 3, 4, 5]); + } +} diff --git a/crates/environment-daemon/src/upgrade.rs b/crates/environment-daemon/src/upgrade.rs index 013539645..2735de706 100644 --- a/crates/environment-daemon/src/upgrade.rs +++ b/crates/environment-daemon/src/upgrade.rs @@ -6,21 +6,22 @@ //! before it can replace the running executable. use std::{ - collections::BTreeMap, io::{Read as _, Seek as _, Write as _}, net::IpAddr, path::{Path, PathBuf}, time::Duration, }; -use anyhow::{Context as _, Result, anyhow, bail}; +use anyhow::{Context as _, anyhow}; +use environment_protocol::{ + bootstrap::{ArtifactSha256, BuildFacts, DiscoveryArtifact, DiscoveryDocument}, + routes::{DISCOVERY_PATH, canonical_origin}, +}; use flate2::read::GzDecoder; use futures_util::StreamExt as _; use reqwest::Url; -use serde::Deserialize; use sha2::{Digest as _, Sha256}; -const DISCOVERY_PATH: &str = "/.well-known/lightspeed-envd"; const MAX_DISCOVERY_BYTES: usize = 1_048_576; const MAX_ARCHIVE_BYTES: u64 = 256 * 1024 * 1024; const MAX_BINARY_BYTES: u64 = 256 * 1024 * 1024; @@ -36,6 +37,10 @@ pub struct UpgradeRequest { /// challenge. Manual upgrades install whichever protocol the deployment /// currently publishes. pub expected_protocol: Option, + /// A protocol that installing would not change, refused as terminal: + /// an automatic upgrade after an unreadable first frame installs any + /// protocol but the daemon's own. + pub other_than: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -46,38 +51,61 @@ pub struct InstalledBuild { pub path: PathBuf, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct DiscoveryDocument { - version: String, - git_sha: String, - protocol_version: u32, - artifacts: BTreeMap, +/// Why an upgrade did not install anything. +#[derive(Debug, thiserror::Error)] +pub enum UpgradeError { + /// A network failure, timeout, HTTP 5xx or 429 while fetching the + /// discovery document or the archive: retry later. + #[error("{0:#}")] + Transient(anyhow::Error), + /// Everything else: an integrity failure (checksum, candidate build + /// facts, a discovery protocol other than the challenge's), a document + /// without this target, or a local install failure. Retrying cannot + /// help. + #[error("{0:#}")] + Terminal(anyhow::Error), +} + +impl UpgradeError { + pub fn is_transient(&self) -> bool { + matches!(self, Self::Transient(_)) + } +} + +type UpgradeResult = Result; + +fn terminal(error: impl Into) -> UpgradeError { + UpgradeError::Terminal(error.into()) } -#[derive(Debug, Deserialize)] -struct DiscoveryArtifact { - file: String, - sha256: String, - url: Option, +/// Classify a failed HTTP exchange: transport trouble and server-side +/// statuses are transient, a client-side status is terminal. +fn http_failure(error: reqwest::Error, context: String) -> UpgradeError { + let transient = match error.status() { + Some(status) => status.is_server_error() || status.as_u16() == 429, + None => true, + }; + let error = anyhow::Error::new(error).context(context); + if transient { + UpgradeError::Transient(error) + } else { + UpgradeError::Terminal(error) + } } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct CandidateBuild { - name: String, - version: String, - git_sha: String, - target: String, - protocol_version: u32, +macro_rules! fail { + ($($arg:tt)*) => { + return Err(terminal(anyhow!($($arg)*))) + }; } /// Resolve an explicit override or derive the deployment discovery endpoint -/// from an outbound WebSocket gateway URL. +/// from an outbound WebSocket gateway URL: its canonical origin, as HTTP, +/// with `DISCOVERY_PATH`. pub fn resolve_discovery_url( gateway_url: Option<&str>, override_url: Option<&str>, -) -> Result { +) -> anyhow::Result { if let Some(url) = override_url { validate_discovery_url(url)?; return Ok(url.to_owned()); @@ -85,123 +113,139 @@ pub fn resolve_discovery_url( let gateway_url = gateway_url.ok_or_else(|| { anyhow!("upgrade needs LIGHTSPEED_ENVD_GATEWAY_URL or LIGHTSPEED_ENVD_DISCOVERY_URL") })?; - let mut url = Url::parse(gateway_url).context("parse environment gateway URL")?; - let http_scheme = match url.scheme() { - "wss" => "https", - "ws" => "http", - scheme => bail!("cannot derive discovery URL from gateway scheme {scheme}"), - }; - url.set_scheme(http_scheme) - .map_err(|_| anyhow!("derive discovery URL scheme"))?; - url.set_path(DISCOVERY_PATH); - url.set_query(None); - url.set_fragment(None); - let resolved = url.to_string(); + let resolved = derive_discovery_url(gateway_url)?; validate_discovery_url(&resolved)?; Ok(resolved) } +/// The discovery endpoint a gateway URL implies, not yet checked for +/// transport security: `https` for `wss://`, plain `http` for `ws://`. +pub fn derive_discovery_url(gateway_url: &str) -> anyhow::Result { + if !gateway_url.starts_with("ws://") && !gateway_url.starts_with("wss://") { + anyhow::bail!("cannot derive a discovery URL from gateway URL {gateway_url}"); + } + let origin = canonical_origin(gateway_url).context("parse environment gateway URL")?; + Ok(match origin.as_str().strip_prefix("wss://") { + Some(authority) => format!("https://{authority}{DISCOVERY_PATH}"), + None => format!( + "http://{}{DISCOVERY_PATH}", + origin.as_str().trim_start_matches("ws://") + ), + }) +} + /// Discovery and archive downloads require HTTPS. Plain HTTP exists only for /// loopback development and the local automatic-upgrade acceptance fixture. -pub fn validate_discovery_url(value: &str) -> Result<()> { +pub fn validate_discovery_url(value: &str) -> anyhow::Result<()> { validate_download_url(value, "discovery URL").map(|_| ()) } -pub async fn install(request: UpgradeRequest) -> Result { +pub async fn install(request: UpgradeRequest) -> UpgradeResult { crate::install_crypto_provider(); - validate_discovery_url(&request.discovery_url)?; - let client = http_client(request.ca_file.as_deref()).await?; + validate_discovery_url(&request.discovery_url).map_err(terminal)?; + let client = http_client(request.ca_file.as_deref()) + .await + .map_err(terminal)?; let discovery = fetch_discovery(&client, &request.discovery_url).await?; if let Some(expected) = request.expected_protocol && discovery.protocol_version != expected { - bail!( + fail!( "discovery document {} advertises protocol {}, but the gateway challenged with protocol {expected}", request.discovery_url, discovery.protocol_version ); } - validate_git_sha(&discovery.git_sha, "discovery gitSha")?; - let artifact = discovery.artifacts.get(&request.target).ok_or_else(|| { - anyhow!( + if request.other_than == Some(discovery.protocol_version) { + fail!( + "discovery document {} advertises protocol {}, which this daemon already speaks", + request.discovery_url, + discovery.protocol_version + ); + } + validate_git_sha(&discovery.git_sha, "discovery gitSha").map_err(terminal)?; + let Some(artifact) = discovery.artifacts.get(&request.target) else { + fail!( "discovery document {} has no envd artifact for target {}", request.discovery_url, request.target - ) - })?; - validate_artifact(artifact)?; - let artifact_url = artifact.url.as_deref().ok_or_else(|| { - anyhow!( + ); + }; + validate_artifact(artifact).map_err(terminal)?; + let Some(artifact_url) = artifact.url.as_deref() else { + fail!( "deployment discovery document {} has no public URL for target {}", request.discovery_url, request.target - ) - })?; - let artifact_url = validate_download_url(artifact_url, "envd artifact URL")?; + ); + }; + let artifact_url = + validate_download_url(artifact_url, "envd artifact URL").map_err(terminal)?; if !artifact_url .path() .ends_with(&format!("/{}", artifact.file)) { - bail!("envd artifact URL does not end with its declared filename"); + fail!("envd artifact URL does not end with its declared filename"); } let (mut archive_file, actual_sha) = download_archive(&client, &artifact_url, MAX_ARCHIVE_BYTES).await?; if actual_sha != artifact.sha256 { - bail!( + fail!( "envd archive checksum mismatch: expected {}, downloaded {actual_sha}", artifact.sha256 ); } - let parent = request.install_path.parent().ok_or_else(|| { - anyhow!( + let manual = || { + manual_install_command( + artifact_url.as_str(), + &artifact.sha256.to_hex(), + &request.install_path, + ) + }; + let Some(parent) = request.install_path.parent() else { + fail!( "envd executable has no parent directory: {}", request.install_path.display() - ) - })?; + ); + }; let mut candidate = tempfile::Builder::new() .prefix(".lightspeed-envd-upgrade-") .tempfile_in(parent) .map_err(|error| { - anyhow!( + terminal(anyhow!( "cannot write next to {}: {error}\n{}", request.install_path.display(), - manual_install_command( - artifact_url.as_str(), - &artifact.sha256, - &request.install_path - ) - ) + manual() + )) })?; - extract_binary(&mut archive_file, candidate.as_file_mut())?; - make_executable(candidate.path())?; + extract_binary(&mut archive_file, candidate.as_file_mut()).map_err(terminal)?; + make_executable(candidate.path()).map_err(terminal)?; candidate .as_file_mut() .sync_all() - .context("sync candidate envd executable")?; + .context("sync candidate envd executable") + .map_err(terminal)?; // Close the write handle before executing the candidate: Linux refuses // to run a file that is still open for writing (ETXTBSY). The path stays // reserved and is removed on every failure path. let candidate = candidate.into_temp_path(); - let build = inspect_candidate(&candidate).await?; - verify_candidate(&build, &discovery, &request.target)?; + let build = inspect_candidate(&candidate).await.map_err(terminal)?; + verify_candidate(&build, &discovery, &request.target).map_err(terminal)?; candidate.persist(&request.install_path).map_err(|error| { - anyhow!( + terminal(anyhow!( "replace {} atomically: {}\n{}", request.install_path.display(), error.error, - manual_install_command( - artifact_url.as_str(), - &artifact.sha256, - &request.install_path - ) - ) + manual() + )) })?; std::fs::File::open(&request.install_path) .and_then(|installed| installed.sync_all()) - .context("sync installed envd executable")?; + .context("sync installed envd executable") + .map_err(terminal)?; Ok(InstalledBuild { version: build.version, @@ -211,7 +255,7 @@ pub async fn install(request: UpgradeRequest) -> Result { }) } -async fn http_client(ca_file: Option<&Path>) -> Result { +async fn http_client(ca_file: Option<&Path>) -> anyhow::Result { let mut builder = reqwest::Client::builder() .connect_timeout(Duration::from_secs(15)) .read_timeout(Duration::from_secs(30)) @@ -223,7 +267,7 @@ async fn http_client(ca_file: Option<&Path>) -> Result { let certificates = reqwest::tls::Certificate::from_pem_bundle(&pem) .with_context(|| format!("parse CA file {}", ca_file.display()))?; if certificates.is_empty() { - bail!("CA file {} holds no certificates", ca_file.display()); + anyhow::bail!("CA file {} holds no certificates", ca_file.display()); } for certificate in certificates { builder = builder.add_root_certificate(certificate); @@ -232,79 +276,91 @@ async fn http_client(ca_file: Option<&Path>) -> Result { builder.build().context("build envd upgrade HTTP client") } -async fn fetch_discovery(client: &reqwest::Client, url: &str) -> Result { +/// Fetch and decode the discovery document. It is part of the frozen +/// bootstrap surface: members this daemon does not know are ignored. +async fn fetch_discovery(client: &reqwest::Client, url: &str) -> UpgradeResult { + let context = || format!("fetch envd discovery document {url}"); let response = client .get(url) .send() .await - .with_context(|| format!("fetch envd discovery document {url}"))? - .error_for_status() - .with_context(|| format!("fetch envd discovery document {url}"))?; + .and_then(reqwest::Response::error_for_status) + .map_err(|error| http_failure(error, context()))?; if response .content_length() .is_some_and(|length| length > MAX_DISCOVERY_BYTES as u64) { - bail!("envd discovery document exceeds {MAX_DISCOVERY_BYTES} bytes"); + fail!("envd discovery document exceeds {MAX_DISCOVERY_BYTES} bytes"); } let mut bytes = Vec::new(); let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { - let chunk = chunk.context("read envd discovery document")?; + let chunk = chunk.map_err(|error| http_failure(error, context()))?; if bytes.len().saturating_add(chunk.len()) > MAX_DISCOVERY_BYTES { - bail!("envd discovery document exceeds {MAX_DISCOVERY_BYTES} bytes"); + fail!("envd discovery document exceeds {MAX_DISCOVERY_BYTES} bytes"); } bytes.extend_from_slice(&chunk); } - serde_json::from_slice(&bytes).context("decode envd discovery document") + serde_json::from_slice(&bytes) + .context("decode envd discovery document") + .map_err(terminal) } async fn download_archive( client: &reqwest::Client, url: &Url, max_bytes: u64, -) -> Result<(tempfile::NamedTempFile, String)> { +) -> UpgradeResult<(tempfile::NamedTempFile, ArtifactSha256)> { + let context = || format!("download envd archive {url}"); let response = client .get(url.clone()) .send() .await - .with_context(|| format!("download envd archive {url}"))? - .error_for_status() - .with_context(|| format!("download envd archive {url}"))?; + .and_then(reqwest::Response::error_for_status) + .map_err(|error| http_failure(error, context()))?; if response .content_length() .is_some_and(|length| length > max_bytes) { - bail!("envd archive exceeds {max_bytes} bytes"); + fail!("envd archive exceeds {max_bytes} bytes"); } - let mut file = tempfile::NamedTempFile::new().context("create temporary envd archive")?; + let mut file = tempfile::NamedTempFile::new() + .context("create temporary envd archive") + .map_err(terminal)?; let mut stream = response.bytes_stream(); let mut size = 0_u64; let mut sha = Sha256::new(); while let Some(chunk) = stream.next().await { - let chunk = chunk.context("read envd archive response")?; - size = size - .checked_add(chunk.len() as u64) - .ok_or_else(|| anyhow!("envd archive size overflow"))?; + let chunk = chunk.map_err(|error| http_failure(error, context()))?; + size = size.saturating_add(chunk.len() as u64); if size > max_bytes { - bail!("envd archive exceeds {max_bytes} bytes"); + fail!("envd archive exceeds {max_bytes} bytes"); } sha.update(&chunk); file.write_all(&chunk) - .context("write temporary envd archive")?; + .context("write temporary envd archive") + .map_err(terminal)?; } file.as_file_mut() .sync_all() - .context("sync temporary envd archive")?; + .context("sync temporary envd archive") + .map_err(terminal)?; file.as_file_mut() .rewind() - .context("rewind temporary envd archive")?; - Ok((file, hex::encode(sha.finalize()))) + .context("rewind temporary envd archive") + .map_err(terminal)?; + let digest: [u8; 32] = sha + .finalize() + .as_slice() + .try_into() + .expect("sha256 is 32 bytes"); + Ok((file, ArtifactSha256::from_bytes(digest))) } fn extract_binary( archive_file: &mut tempfile::NamedTempFile, destination: &mut std::fs::File, -) -> Result<()> { +) -> anyhow::Result<()> { archive_file .as_file_mut() .rewind() @@ -316,27 +372,27 @@ fn extract_binary( let mut entry = entry.context("read envd archive entry")?; let path = entry.path().context("read envd archive path")?; if path.as_ref() != Path::new("lightspeed-envd") { - bail!("envd archive contains unexpected entry {}", path.display()); + anyhow::bail!("envd archive contains unexpected entry {}", path.display()); } if found { - bail!("envd archive contains lightspeed-envd more than once"); + anyhow::bail!("envd archive contains lightspeed-envd more than once"); } if !entry.header().entry_type().is_file() { - bail!("envd archive entry lightspeed-envd is not a regular file"); + anyhow::bail!("envd archive entry lightspeed-envd is not a regular file"); } let size = entry.size(); if size == 0 || size > MAX_BINARY_BYTES { - bail!("envd executable has invalid size {size}"); + anyhow::bail!("envd executable has invalid size {size}"); } let copied = std::io::copy(&mut entry.by_ref().take(MAX_BINARY_BYTES + 1), destination) .context("extract envd executable")?; if copied != size { - bail!("envd executable size differs from its archive header"); + anyhow::bail!("envd executable size differs from its archive header"); } found = true; } if !found { - bail!("envd archive does not contain lightspeed-envd"); + anyhow::bail!("envd archive does not contain lightspeed-envd"); } destination .flush() @@ -344,18 +400,20 @@ fn extract_binary( } #[cfg(unix)] -fn make_executable(path: &Path) -> Result<()> { +fn make_executable(path: &Path) -> anyhow::Result<()> { use std::os::unix::fs::PermissionsExt as _; std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)) .with_context(|| format!("make candidate executable {}", path.display())) } #[cfg(not(unix))] -fn make_executable(_path: &Path) -> Result<()> { - bail!("envd self-upgrade is supported only on Unix") +fn make_executable(_path: &Path) -> anyhow::Result<()> { + anyhow::bail!("envd self-upgrade is supported only on Unix") } -async fn inspect_candidate(path: &Path) -> Result { +/// Run the candidate's `--print-build`, decoded as the frozen +/// [`BuildFacts`]. +async fn inspect_candidate(path: &Path) -> anyhow::Result { let output = tokio::time::timeout( CANDIDATE_TIMEOUT, tokio::process::Command::new(path) @@ -367,7 +425,7 @@ async fn inspect_candidate(path: &Path) -> Result { .context("candidate envd --print-build timed out")? .with_context(|| format!("run candidate envd {}", path.display()))?; if !output.status.success() { - bail!( + anyhow::bail!( "candidate envd --print-build exited with {}: {}", output.status, String::from_utf8_lossy(&output.stderr).trim() @@ -377,38 +435,39 @@ async fn inspect_candidate(path: &Path) -> Result { } fn verify_candidate( - build: &CandidateBuild, + build: &BuildFacts, discovery: &DiscoveryDocument, target: &str, -) -> Result<()> { - if build.name != "lightspeed-envd" { - bail!( - "candidate build name is {}, expected lightspeed-envd", - build.name +) -> anyhow::Result<()> { + if build.name != crate::DAEMON_NAME { + anyhow::bail!( + "candidate build name is {}, expected {}", + build.name, + crate::DAEMON_NAME ); } if build.version != discovery.version { - bail!( + anyhow::bail!( "candidate version {} differs from discovery version {}", build.version, discovery.version ); } if build.git_sha != discovery.git_sha { - bail!( + anyhow::bail!( "candidate gitSha {} differs from discovery gitSha {}", build.git_sha, discovery.git_sha ); } if build.target != target { - bail!( + anyhow::bail!( "candidate target {} differs from requested target {target}", build.target ); } if build.protocol_version != discovery.protocol_version { - bail!( + anyhow::bail!( "candidate protocol {} differs from discovery protocol {}", build.protocol_version, discovery.protocol_version @@ -417,55 +476,47 @@ fn verify_candidate( Ok(()) } -fn validate_artifact(artifact: &DiscoveryArtifact) -> Result<()> { +fn validate_artifact(artifact: &DiscoveryArtifact) -> anyhow::Result<()> { if artifact.file.is_empty() || Path::new(&artifact.file) .file_name() .and_then(|name| name.to_str()) != Some(artifact.file.as_str()) { - bail!("envd artifact has an unsafe filename"); + anyhow::bail!("envd artifact has an unsafe filename"); } if !artifact.file.ends_with(".tar.gz") { - bail!("envd artifact is not a .tar.gz archive"); - } - if artifact.sha256.len() != 64 - || !artifact - .sha256 - .bytes() - .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) - { - bail!("envd artifact has an invalid SHA-256 checksum"); + anyhow::bail!("envd artifact is not a .tar.gz archive"); } Ok(()) } -fn validate_git_sha(value: &str, label: &str) -> Result<()> { +fn validate_git_sha(value: &str, label: &str) -> anyhow::Result<()> { if value.len() != 40 || !value .bytes() .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) { - bail!("{label} must be a full lowercase hexadecimal commit"); + anyhow::bail!("{label} must be a full lowercase hexadecimal commit"); } Ok(()) } -fn validate_download_url(value: &str, label: &str) -> Result { +fn validate_download_url(value: &str, label: &str) -> anyhow::Result { let url = Url::parse(value).with_context(|| format!("parse {label}"))?; if !url.username().is_empty() || url.password().is_some() { - bail!("{label} must not contain credentials"); + anyhow::bail!("{label} must not contain credentials"); } if url.fragment().is_some() { - bail!("{label} must not contain a fragment"); + anyhow::bail!("{label} must not contain a fragment"); } match url.scheme() { "https" => {} "http" if is_loopback(&url) => {} - _ => bail!("{label} must use https (or http toward loopback)"), + _ => anyhow::bail!("{label} must use https (or http toward loopback)"), } if url.host_str().is_none() { - bail!("{label} has no host"); + anyhow::bail!("{label} has no host"); } Ok(url) } @@ -474,6 +525,7 @@ fn is_loopback(url: &Url) -> bool { let Some(host) = url.host_str() else { return false; }; + let host = host.trim_start_matches('[').trim_end_matches(']'); host == "localhost" || host .parse::() @@ -539,7 +591,7 @@ mod tests { release_info::TARGET ); let archive = envd_archive(script.as_bytes()); - let archive_sha = hex::encode(Sha256::digest(&archive)); + let archive_sha = environment_protocol::bytes::encode_hex(&Sha256::digest(&archive)); let (origin, server) = serve_upgrade_with_document(archive, |origin| { discovery_json(origin, &archive_sha, &git_sha, 3) }) @@ -551,6 +603,7 @@ mod tests { install_path: install_path.clone(), target: release_info::TARGET.to_owned(), expected_protocol: Some(3), + other_than: None, }) .await .expect("install"); @@ -581,10 +634,12 @@ mod tests { install_path: install_path.clone(), target: release_info::TARGET.to_owned(), expected_protocol: Some(3), + other_than: None, }) .await .expect_err("protocol mismatch"); server.abort(); + assert!(!error.is_transient()); assert!( error .to_string() @@ -596,6 +651,36 @@ mod tests { ); } + #[cfg(unix)] + #[tokio::test(flavor = "current_thread")] + async fn an_upgrade_to_the_daemons_own_protocol_is_refused() { + let temp = tempfile::tempdir().expect("tempdir"); + let install_path = temp.path().join("lightspeed-envd"); + std::fs::write(&install_path, b"old envd").expect("old executable"); + let git_sha = "d".repeat(40); + let archive = envd_archive(b"#!/bin/sh\nexit 1\n"); + let (origin, server) = serve_upgrade_with_document(archive, |origin| { + discovery_json(origin, &"0".repeat(64), &git_sha, 3) + }) + .await; + let error = install(UpgradeRequest { + discovery_url: format!("{origin}/envd.json"), + ca_file: None, + install_path: install_path.clone(), + target: release_info::TARGET.to_owned(), + expected_protocol: None, + other_than: Some(3), + }) + .await + .expect_err("own protocol"); + server.abort(); + assert!(!error.is_transient(), "reinstalling cannot help"); + assert_eq!( + std::fs::read(&install_path).expect("old executable"), + b"old envd" + ); + } + #[cfg(unix)] #[tokio::test(flavor = "current_thread")] async fn checksum_mismatch_leaves_the_executable_untouched() { @@ -614,10 +699,12 @@ mod tests { install_path: install_path.clone(), target: release_info::TARGET.to_owned(), expected_protocol: Some(4), + other_than: None, }) .await .expect_err("checksum mismatch"); server.await.expect("server"); + assert!(!error.is_transient(), "integrity failures are terminal"); assert!(error.to_string().contains("archive checksum mismatch")); assert_eq!( std::fs::read(&install_path).expect("old executable"), @@ -625,6 +712,51 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn server_errors_are_transient_and_client_errors_terminal() { + for (status, transient) in [(503, true), (429, true), (404, false)] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener"); + let origin = format!("http://{}", listener.local_addr().expect("address")); + let server = tokio::spawn(async move { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + let (mut socket, _) = listener.accept().await.expect("accept"); + let mut request = vec![0_u8; 4096]; + let _ = socket.read(&mut request).await; + let response = format!( + "HTTP/1.1 {status} Status\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + socket.write_all(response.as_bytes()).await.expect("write"); + }); + let temp = tempfile::tempdir().expect("tempdir"); + let error = install(UpgradeRequest { + discovery_url: format!("{origin}/envd.json"), + ca_file: None, + install_path: temp.path().join("lightspeed-envd"), + target: release_info::TARGET.to_owned(), + expected_protocol: Some(3), + other_than: None, + }) + .await + .expect_err("no document"); + server.await.expect("server"); + assert_eq!(error.is_transient(), transient, "HTTP {status}: {error}"); + } + + let unreachable = install(UpgradeRequest { + discovery_url: "http://127.0.0.1:9/envd.json".to_owned(), + ca_file: None, + install_path: PathBuf::from("/nonexistent/lightspeed-envd"), + target: release_info::TARGET.to_owned(), + expected_protocol: None, + other_than: None, + }) + .await + .expect_err("nothing listens"); + assert!(unreachable.is_transient(), "{unreachable}"); + } + #[cfg(unix)] fn envd_archive(bytes: &[u8]) -> Vec { use flate2::{Compression, write::GzEncoder}; diff --git a/crates/environment-daemon/tests/client_round_trip.rs b/crates/environment-daemon/tests/client_round_trip.rs new file mode 100644 index 000000000..e7991405e --- /dev/null +++ b/crates/environment-daemon/tests/client_round_trip.rs @@ -0,0 +1,497 @@ +//! envd's data plane driven end to end by its real consumers: the client +//! library over envd's own connection loop, the tools' data conformance +//! suite, the tools' transfer orchestration against a blob store, and the +//! builtin file tools over a remote environment. + +use std::{ + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, +}; + +use environment_client::{ClientError, ConnectOptions, DataClient}; +use environment_daemon::{ + DaemonRuntime, + config::DaemonConfig, + dispatch::{RequestContext, handle_request}, + server, +}; +use environment_protocol::{ + data::transfer::{TransferOnExisting, TransferPhase}, + handshake::ImplementationInfo, + jsonrpc::Response, + paths::EnvironmentPath, +}; +use harness::storage::BlobStore; +use serde_json::json; +use tools::{ + builtin::{ + BuiltinTool, BuiltinToolContext, BuiltinToolOperation as Op, BuiltinToolSurface as Surface, + }, + environment_protocol::{RemoteEnvironmentConnection, conformance}, + transfer, +}; + +fn daemon_config(root: &Path, state_dir: PathBuf) -> DaemonConfig { + DaemonConfig { + listen: None, + cwd: root.to_path_buf(), + fs_root: root.to_path_buf(), + state_dir, + read_only_fs: false, + registration: None, + scrubbed_env: Vec::new(), + } +} + +/// A daemon over a fresh root, its state dir beside the root, and an +/// initialized client connected to it through the real connection loop. +struct Daemon { + _temp: tempfile::TempDir, + base: PathBuf, + root: PathBuf, + client: DataClient, +} + +impl Daemon { + async fn start() -> Self { + let temp = tempfile::tempdir().expect("tempdir"); + let base = temp.path().canonicalize().expect("canonical temp"); + let root = base.join("root"); + std::fs::create_dir_all(&root).expect("root"); + let runtime = + DaemonRuntime::new(daemon_config(&root, base.join("state"))).expect("runtime"); + let (client_io, server_io) = tokio::io::duplex(1 << 20); + tokio::spawn(server::serve_stream(runtime, server_io)); + let options = + ConnectOptions::new(ImplementationInfo::new("envd-round-trip-test").expect("name")); + let (client, _) = DataClient::connect_initialized_over(client_io, options) + .await + .expect("initialized"); + Self { + _temp: temp, + base, + root, + client, + } + } + + fn path(&self, relative: &str) -> EnvironmentPath { + let path = if relative.is_empty() { + self.root.clone() + } else { + self.root.join(relative) + }; + EnvironmentPath::new(path.to_str().expect("UTF-8 temp path")).expect("environment path") + } +} + +#[tokio::test(flavor = "current_thread")] +async fn envd_passes_the_data_conformance_suite() { + let daemon = Daemon::start().await; + let outside = daemon.base.join("outside"); + std::fs::create_dir_all(&outside).expect("outside"); + let mut suite = conformance::DataConformance::new(daemon.client.clone(), daemon.path("")); + suite.outside_root = + Some(EnvironmentPath::new(outside.to_str().expect("UTF-8")).expect("outside path")); + let report = conformance::run(&suite).await; + if let Err(failure) = report.into_result() { + panic!("{failure}"); + } +} + +/// A VFS tree materialized into envd and captured back is the same tree; +/// a repeat reuses the bytes already there, and capturing into an empty +/// store moves them. +#[tokio::test(flavor = "current_thread")] +async fn vfs_trees_round_trip_through_materialize_and_capture() { + let daemon = Daemon::start().await; + let cas = tempfile::tempdir().expect("cas"); + let store = store_fs::FsBlobStore::open(cas.path()) + .await + .expect("store"); + // Larger than one content chunk, so content moves in several writes. + let size = 5 * 1024 * 1024 + 7; + let content: Vec = (0..size).map(|index| (index % 251) as u8).collect(); + let blob = store.put_bytes(content.clone()).await.expect("blob"); + let mut directory = vfs::VfsDirectory::default(); + directory.entries.insert( + "run".into(), + vfs::VfsEntry::File(vfs::VfsFile { + blob_ref: blob.clone(), + size_bytes: size as u64, + media_type: None, + executable: true, + }), + ); + directory.entries.insert( + "empty".into(), + vfs::VfsEntry::Directory(vfs::VfsDirectory::default()), + ); + let selection = vfs::VfsEntry::Directory(directory); + std::fs::write(daemon.root.join("sibling"), b"preserved").expect("sibling"); + + let first = transfer::materialize( + &daemon.client, + &store, + "first", + &selection, + daemon.path("selected"), + TransferOnExisting::Replace, + ) + .await + .expect("materialize"); + assert_eq!(first.phase, TransferPhase::Complete); + assert_eq!(first.total_bytes, size as u64); + assert_eq!(first.transferred_bytes, size as u64); + assert_eq!(first.reused_bytes, 0); + let run = daemon.root.join("selected/run"); + assert_eq!(std::fs::read(&run).expect("run"), content); + assert_eq!( + std::fs::metadata(&run).expect("run").permissions().mode() & 0o111, + 0o111 + ); + assert!(daemon.root.join("selected/empty").is_dir()); + + let again = transfer::materialize( + &daemon.client, + &store, + "again", + &selection, + daemon.path("selected"), + TransferOnExisting::Replace, + ) + .await + .expect("materialize again"); + assert_eq!(again.transferred_bytes, 0); + assert_eq!(again.reused_bytes, size as u64); + + let captured = transfer::capture( + &daemon.client, + &store, + None, + "capture-known", + daemon.path("selected"), + ) + .await + .expect("capture"); + assert_eq!(captured.entry, selection); + assert_eq!(captured.status.transferred_bytes, 0); + + let empty_cas = tempfile::tempdir().expect("empty cas"); + let empty_store = store_fs::FsBlobStore::open(empty_cas.path()) + .await + .expect("empty store"); + let captured = transfer::capture( + &daemon.client, + &empty_store, + None, + "capture-missing", + daemon.path("selected"), + ) + .await + .expect("capture into an empty store"); + assert_eq!(captured.entry, selection); + assert_eq!(captured.status.transferred_bytes, size as u64); + assert!(empty_store.has_blob(&blob).await.expect("has blob")); + assert_eq!( + std::fs::read(daemon.root.join("sibling")).expect("sibling"), + b"preserved" + ); +} + +/// A client whose every request is dispatched straight into `runtime`, +/// recording the largest frame either way, and able to lose the response +/// of the next `transfer/commit` after the daemon acted on it. +fn in_process_client(runtime: DaemonRuntime) -> (DataClient, Arc, Arc) { + let largest = Arc::new(AtomicUsize::new(0)); + let lose_commit = Arc::new(AtomicBool::new(false)); + let initialize = runtime.initialize_response(); + let client = DataClient::in_process(initialize, { + let largest = largest.clone(); + let lose_commit = lose_commit.clone(); + move |request| { + let runtime = runtime.clone(); + let largest = largest.clone(); + let lose_commit = lose_commit.clone(); + Box::pin(async move { + largest.fetch_max(request.to_text().len(), Ordering::Relaxed); + let result = handle_request( + &runtime, + &RequestContext::detached(), + &request.method, + request.params.clone(), + ) + .await; + let response = match result { + Ok(reply) => { + let response = Response::success(request.id.clone(), &reply.value); + if let Some(on_written) = reply.on_written { + on_written(); + } + response + } + Err(error) => Response::failure(Some(request.id.clone()), error), + }; + if request.method == "transfer/commit" && lose_commit.swap(false, Ordering::Relaxed) + { + return Err(ClientError::Transport("response lost".to_owned())); + } + largest.fetch_max(response.to_text().len(), Ordering::Relaxed); + Ok(response) + }) + } + }); + (client, largest, lose_commit) +} + +/// The builtin `write_file` (from a blob reference) and `reference` tools +/// move large binary files through bounded transfer chunks on every tool +/// surface; a retried operation answers its first receipt, even after the +/// source changed, the file was edited, or the commit response was lost. +#[tokio::test(flavor = "current_thread")] +async fn blob_file_tools_stream_and_keep_receipts_across_retries() { + for surface in [ + Surface::Canonical, + Surface::CodexLike, + Surface::ClaudeCodeLike, + ] { + let temp = tempfile::tempdir().expect("tempdir"); + let base = temp.path().canonicalize().expect("canonical temp"); + let root = base.join("root"); + std::fs::create_dir_all(&root).expect("root"); + let runtime = + DaemonRuntime::new(daemon_config(&root, base.join("state"))).expect("runtime"); + let cas = tempfile::tempdir().expect("cas"); + let store = Arc::new( + store_fs::FsBlobStore::open(cas.path()) + .await + .expect("store"), + ); + // Larger than the daemon's content chunk and its whole-file read ceiling. + let size = 9 * 1024 * 1024 + 7; + let content = store.put_bytes(vec![0x83; size]).await.expect("blob"); + let (client, largest, lose_commit) = in_process_client(runtime); + let (_, context) = RemoteEnvironmentConnection::new(client) + .with_cwd(tools::fs::FsPath::new(root.to_str().expect("UTF-8")).expect("cwd")) + .into_contexts(store.clone()); + let context = context.with_environment_id("test-env"); + let key = if surface == Surface::ClaudeCodeLike { + "file_path" + } else { + "path" + }; + let write = BuiltinTool::environment(Op::WriteFile, surface); + let capture = BuiltinTool::environment(Op::Reference, surface); + + let writer = context.clone().with_operation_id("write-binary"); + let output = write + .invoke_json( + BuiltinToolContext::Environment(&writer), + json!({key: "nested/source.bin", "content_ref": content}), + ) + .await + .expect("write from reference"); + assert_eq!(output.output_json["bytes_written"], size); + assert_eq!(output.output_json["receipt"]["total_bytes"], size); + let source = root.join("nested/source.bin"); + assert_eq!( + std::fs::metadata(&source).expect("written").len(), + size as u64 + ); + + let capture_cas = tempfile::tempdir().expect("capture cas"); + let capture_store = Arc::new( + store_fs::FsBlobStore::open(capture_cas.path()) + .await + .expect("capture store"), + ); + let mut capture_context = context.clone().with_operation_id("capture-binary"); + capture_context.blobs = capture_store.clone(); + let captured = capture + .invoke_json( + BuiltinToolContext::Environment(&capture_context), + json!({"path": "nested/source.bin"}), + ) + .await + .expect("capture"); + assert_eq!(captured.output_json["content_ref"], content.to_string()); + assert_eq!(captured.output_json["byte_len"], size); + assert_eq!(captured.output_json["source"]["id"], "test-env"); + assert_eq!(captured.attachments.len(), 1); + assert_eq!( + capture_store + .read_blob_range(&content, size as u64 - 7, 7) + .await + .expect("captured tail"), + vec![0x83; 7] + ); + + std::fs::write(&source, b"changed live file").expect("change source"); + let changed = capture + .invoke_json( + BuiltinToolContext::Environment(&capture_context), + json!({"path": "nested/source.bin"}), + ) + .await + .expect("retry after change"); + assert_eq!( + changed.output_json, captured.output_json, + "a retried capture answers the version it first captured" + ); + std::fs::remove_file(&source).expect("remove source"); + let deleted = capture + .invoke_json( + BuiltinToolContext::Environment(&capture_context), + json!({"path": "nested/source.bin"}), + ) + .await + .expect("retry after delete"); + assert_eq!( + deleted.output_json, captured.output_json, + "a completed capture never revisits the live path" + ); + + // A lost commit response leaves the write's receipt to answer the retry. + let result = root.join("result.bin"); + std::fs::write(&result, b"original").expect("result"); + std::fs::set_permissions(&result, std::fs::Permissions::from_mode(0o751)).expect("mode"); + let writer = context.clone().with_operation_id("lost-write-receipt"); + lose_commit.store(true, Ordering::Relaxed); + assert!( + write + .invoke_json( + BuiltinToolContext::Environment(&writer), + json!({key: "result.bin", "content_ref": content}), + ) + .await + .is_err() + ); + assert_eq!( + std::fs::metadata(&result) + .expect("result") + .permissions() + .mode() + & 0o777, + 0o751 + ); + std::fs::write(&result, b"later local edit").expect("edit"); + let retry = write + .invoke_json( + BuiltinToolContext::Environment(&writer), + json!({key: "result.bin", "content_ref": content}), + ) + .await + .expect("retry the write"); + assert_eq!(retry.output_json["receipt"]["total_bytes"], size); + assert_eq!(std::fs::read(&result).expect("result"), b"later local edit"); + + std::fs::create_dir(root.join("directory")).expect("directory"); + std::fs::write(root.join("directory/keep"), b"keep").expect("keep"); + let writer = context.clone().with_operation_id("reject-directory-write"); + assert!( + write + .invoke_json( + BuiltinToolContext::Environment(&writer), + json!({key: "directory", "content_ref": content}), + ) + .await + .is_err() + ); + assert_eq!( + std::fs::read(root.join("directory/keep")).expect("keep"), + b"keep" + ); + let reader = context + .clone() + .with_operation_id("reject-directory-capture"); + assert!( + capture + .invoke_json( + BuiltinToolContext::Environment(&reader), + json!({"path": "directory"}), + ) + .await + .is_err() + ); + assert!( + largest.load(Ordering::Relaxed) < 8 * 1024 * 1024, + "large binary transfers move in bounded chunks" + ); + } +} + +/// Skill discovery over envd's per-root scans: installer layouts under two +/// roots, per-root fingerprint caching, edits that do not change a skill's +/// metadata, and a malformed skill reported as a warning naming its root. +#[tokio::test(flavor = "current_thread")] +async fn skill_discovery_reads_installer_layouts_per_root() { + use tools::skills::environment::{ + apply_cached_fingerprints, environment_skill_catalog, environment_skill_scan_query, + }; + let daemon = Daemon::start().await; + let project_root = daemon.root.join("project/.agents/skills"); + let home_root = daemon.root.join("home/.codex/skills"); + let review = project_root.join("review"); + for directory in [&review, &home_root.join("copy")] { + std::fs::create_dir_all(directory).expect("skill directory"); + } + let doc = "---\nname: review\ndescription: >-\n Review code\n carefully.\n---\nbody one"; + std::fs::write(review.join("SKILL.md"), doc).expect("review"); + std::fs::write(home_root.join("copy/SKILL.md"), doc).expect("copy"); + let config = harness::EnvironmentSkillsConfig { + roots: Some(vec![ + project_root.to_string_lossy().into_owned(), + home_root.to_string_lossy().into_owned(), + ]), + }; + let cwd = daemon.root.join("project"); + let mut query = environment_skill_scan_query(&config, cwd.to_str(), None).expect("query"); + + let first = daemon.client.scan(&query).await.expect("scan"); + let catalog = environment_skill_catalog("machine-a", &first, None).expect("catalog"); + assert!(catalog.warnings.is_empty(), "{:?}", catalog.warnings); + assert_eq!(catalog.skills.len(), 2, "independent copies stay separate"); + assert!( + catalog + .skills + .iter() + .all(|skill| skill.description == "Review code carefully.") + ); + assert_ne!(catalog.skills[0].skill_id, catalog.skills[1].skill_id); + let other = environment_skill_catalog("machine-b", &first, None).expect("other catalog"); + assert_ne!(catalog.skills[0].skill_id, other.skills[0].skill_id); + assert_eq!(catalog.root_fingerprints.len(), 2); + + apply_cached_fingerprints(&mut query, &catalog); + let unchanged = daemon.client.scan(&query).await.expect("rescan"); + let cached = + environment_skill_catalog("machine-a", &unchanged, Some(&catalog)).expect("cached"); + assert_eq!(cached, catalog); + + std::fs::write(review.join("SKILL.md"), doc.replace("one", "two")).expect("body edit"); + let edited = daemon.client.scan(&query).await.expect("scan after edit"); + let edited = + environment_skill_catalog("machine-a", &edited, Some(&catalog)).expect("edited catalog"); + assert_eq!(edited.skills, catalog.skills); + + std::fs::write( + review.join("SKILL.md"), + "---\nname: broken\ndescription: [invalid]\n---", + ) + .expect("broken"); + apply_cached_fingerprints(&mut query, &edited); + let broken = daemon.client.scan(&query).await.expect("scan broken"); + let broken = + environment_skill_catalog("machine-a", &broken, Some(&edited)).expect("broken catalog"); + assert_eq!(broken.skills.len(), 1, "the other root's skill remains"); + assert_eq!(broken.warnings.len(), 1); + assert!( + broken.warnings[0].starts_with(project_root.to_str().expect("UTF-8")), + "{:?}", + broken.warnings + ); +} diff --git a/crates/environment-protocol/Cargo.toml b/crates/environment-protocol/Cargo.toml index b8eaf45db..13f9407a6 100644 --- a/crates/environment-protocol/Cargo.toml +++ b/crates/environment-protocol/Cargo.toml @@ -10,5 +10,17 @@ schema = ["dep:schemars"] base64 = "0.23" serde = { version = "1", features = ["derive"] } serde_json = "1" +serde_path_to_error = "0.1" +sha2 = "0.11" schemars = { version = "1", optional = true } thiserror = "2" + +[dev-dependencies] +# Integration tests check the committed contract, which needs the schema +# export. +environment-protocol = { path = ".", features = ["schema"] } + +[[bin]] +name = "export-contract" +path = "src/bin/export-contract.rs" +required-features = ["schema"] diff --git a/crates/environment-protocol/contract/controller.openrpc.json b/crates/environment-protocol/contract/controller.openrpc.json new file mode 100644 index 000000000..d3d08f019 --- /dev/null +++ b/crates/environment-protocol/contract/controller.openrpc.json @@ -0,0 +1,1726 @@ +{ + "components": { + "schemas": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "power", + "description": "The controller's `power` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "CloseTargetParams": { + "additionalProperties": false, + "description": "`target/close` params: destroy the owner's target and every resource it\nholds, public ingress included. Idempotent. Graceful within the\nprovider's own bound, then forced. Answers `closing` until the target is\ngone, then `closed`; `closed` also when the provider holds no target for\nthe owner.", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "ControllerCapabilities": { + "additionalProperties": false, + "description": "Optional controller methods. Calling an absent one is\n`capabilityUnavailable`.", + "properties": { + "ingress": { + "description": "`ingress/ensure` and `ingress/remove` are served.", + "type": "boolean" + }, + "power": { + "description": "`target/setPower` is served.", + "type": "boolean" + } + }, + "type": "object" + }, + "ControllerInitializeParams": { + "additionalProperties": false, + "description": "`initialize` params on the controller plane.", + "properties": { + "clientInfo": { + "$ref": "#/components/schemas/ImplementationInfo" + }, + "protocolVersion": { + "description": "The client's `CONTROLLER_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "clientInfo" + ], + "type": "object" + }, + "ControllerInitializeResponse": { + "additionalProperties": false, + "description": "`initialize` response on the controller plane.", + "properties": { + "capabilities": { + "$ref": "#/components/schemas/ControllerCapabilities" + }, + "plane": { + "allOf": [ + { + "$ref": "#/components/schemas/Plane" + } + ], + "description": "Always `controller`." + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "serverInfo": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + "required": [ + "protocolVersion", + "plane", + "serverInfo", + "capabilities" + ], + "type": "object" + }, + "CreateTargetParams": { + "additionalProperties": false, + "description": "`target/create` params. Idempotent per owner: when the owner already\nhas a target, the provider returns its observation and applies no\ncreation policy (template deprecation, image checks). Handing out a\ntarget from a warm pool counts as a create.", + "properties": { + "displayName": { + "anyOf": [ + { + "$ref": "#/components/schemas/DisplayName" + } + ], + "description": "Label for the target, where the provider shows one." + }, + "idle": { + "anyOf": [ + { + "$ref": "#/components/schemas/IdlePolicy" + } + ], + "description": "Overrides the template's `idle` for this target. A policy that does\nnot validate (`IdlePolicy::validate`) is `invalidParams`." + }, + "owner": { + "$ref": "#/components/schemas/TargetOwner" + }, + "templateId": { + "$ref": "#/components/schemas/TemplateId" + } + }, + "required": [ + "owner", + "templateId" + ], + "type": "object" + }, + "DataRouteRefusal": { + "description": "Body of the HTTP refusal a data route answers instead of upgrading, with\n`Content-Type: application/json`. A provider data route uses `waking`,\n`booting`, `gone` and `failed`; the gateway's worker route adds\n`provisioning`, `disconnected` and `away` and forwards a provider's\nrefusal unchanged. Every 503 carries `Retry-After` in whole seconds,\n1 to `RETRY_AFTER_MAX_SECS`.\nThe body is an HTTP refusal, not a JSON-RPC message, and is tagged by\n`state`.", + "oneOf": [ + { + "additionalProperties": false, + "description": "503: resuming from a powered-down state.", + "properties": { + "state": { + "const": "waking", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "503: running, envd not answering yet.", + "properties": { + "state": { + "const": "booting", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "503 (gateway): the environment is still provisioning.", + "properties": { + "state": { + "const": "provisioning", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "503 (gateway): the registered daemon is not connected.", + "properties": { + "state": { + "const": "disconnected", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "410 (gateway): the registered daemon has been away past its\nreconnect bound.", + "properties": { + "state": { + "const": "away", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "410: the target no longer exists.", + "properties": { + "state": { + "const": "gone", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "409: broken beyond the provider's recovery.", + "properties": { + "message": { + "$ref": "#/components/schemas/DescriptiveText" + }, + "state": { + "const": "failed", + "type": "string" + } + }, + "required": [ + "state", + "message" + ], + "type": "object" + } + ] + }, + "DescriptiveText": { + "description": "A short human-readable text: 1 to 1024 bytes without control characters\nother than line feed and tab. Never parsed.", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DisplayName": { + "description": "A display name: 1 to 128 bytes without control characters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "EnsureIngressParams": { + "additionalProperties": false, + "description": "`ingress/ensure` params: route the template's public ingress to the\ntarget. Idempotent. A target whose template has no `publicIngress` is\n`failedPrecondition` (`ingressUnsupported`).", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "EnvironmentId": { + "description": "`environmentId` of a Lightspeed environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "EnvironmentTemplate": { + "additionalProperties": false, + "description": "A template targets are created from.", + "properties": { + "deprecated": { + "description": "Creates no new targets; existing targets keep working.", + "type": "boolean" + }, + "description": { + "anyOf": [ + { + "$ref": "#/components/schemas/DescriptiveText" + } + ] + }, + "displayName": { + "$ref": "#/components/schemas/DisplayName" + }, + "idle": { + "anyOf": [ + { + "$ref": "#/components/schemas/IdlePolicy" + } + ], + "description": "Idle policy targets get when `target/create` names none; absent when\nthe provider applies none by default." + }, + "metadata": { + "allOf": [ + { + "$ref": "#/components/schemas/Metadata" + } + ], + "description": "Provider-specific display facts, such as an image fingerprint." + }, + "powerStates": { + "description": "Power states `target/setPower` can move this template's targets to.\nEmpty when the provider offers no power control beyond create and\nclose.", + "items": { + "$ref": "#/components/schemas/PowerState" + }, + "type": "array" + }, + "publicIngress": { + "description": "Targets of this template can serve public ingress.", + "type": "boolean" + }, + "resources": { + "anyOf": [ + { + "$ref": "#/components/schemas/TemplateResources" + } + ] + }, + "templateId": { + "$ref": "#/components/schemas/TemplateId" + } + }, + "required": [ + "templateId", + "displayName" + ], + "type": "object" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/components/schemas/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/components/schemas/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "GetTargetParams": { + "additionalProperties": false, + "description": "`target/get` params. A pure read: it never creates or changes anything.\nA target that no longer exists is `notFound`.", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "IdlePolicy": { + "additionalProperties": false, + "description": "Staged idle policy. Each set threshold is measured against envd\n`env/getIdle` `idleForMs`, and the set thresholds are non-decreasing in\nthe order pause → suspend → stop → close. Every stage is optional; a\npolicy sets at least one.", + "properties": { + "closeAfterMs": { + "description": "Close the target after this long idle; the provider then reports it\n`closed`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "pauseAfterMs": { + "description": "Pause the target after this long idle.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "stopAfterMs": { + "description": "Stop the target after this long idle.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "suspendAfterMs": { + "description": "Suspend the target after this long idle.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "IngressObservation": { + "additionalProperties": false, + "description": "Realized public ingress of a target.", + "properties": { + "endpoint": { + "allOf": [ + { + "$ref": "#/components/schemas/PublicUrl" + } + ], + "description": "Public URL that reaches the target's application." + } + }, + "required": [ + "endpoint" + ], + "type": "object" + }, + "ListTemplatesParams": { + "additionalProperties": false, + "description": "`template/list` params: the universe whose targets the templates would\ncreate. A provider may offer each universe a different set.", + "properties": { + "universeId": { + "$ref": "#/components/schemas/UniverseId" + } + }, + "required": [ + "universeId" + ], + "type": "object" + }, + "ListTemplatesResponse": { + "additionalProperties": false, + "description": "`template/list` response.", + "properties": { + "templates": { + "items": { + "$ref": "#/components/schemas/EnvironmentTemplate" + }, + "type": "array" + } + }, + "type": "object" + }, + "Metadata": { + "additionalProperties": { + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "maxProperties": 32, + "propertyNames": { + "maxLength": 64, + "minLength": 1, + "not": { + "pattern": "^lightspeed\\." + }, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "Plane": { + "description": "The plane a connection serves. Each `initialize` response names its\nplane, so a client that reached the wrong socket fails at the handshake.", + "oneOf": [ + { + "const": "data", + "description": "envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "controller", + "description": "A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "registration", + "description": "The registration channel between a daemon and the gateway, versioned\nby `ENVD_PROTOCOL_VERSION`.", + "type": "string" + } + ] + }, + "PowerState": { + "description": "Steady power state of an active target, in provider-neutral terms.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "paused", + "description": "Execution halted, memory retained; resuming is fast.", + "type": "string" + }, + { + "const": "suspended", + "description": "Execution state persisted, compute released; resuming restores it.", + "type": "string" + }, + { + "const": "stopped", + "description": "Powered off; disks retained.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/components/schemas/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "ProviderTargetId": { + "description": "`targetId`: the provider's own id for a target, opaque to Lightspeed.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "PublicUrl": { + "description": "A public `http://` or `https://` URL a provider reports, such as a\ntarget's ingress `endpoint`: lowercase scheme, a host and optional port,\noptional path and query, no user information or fragment, at most 2048\nbytes. Validated on decode.", + "maxLength": 2048, + "minLength": 1, + "pattern": "^https?://[^/?#@\\s]+([/?][^#\\s]*)?$", + "type": "string" + }, + "RemoveIngressParams": { + "additionalProperties": false, + "description": "`ingress/remove` params: stop routing public ingress to the target.\nIdempotent.", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "SetTargetPowerParams": { + "additionalProperties": false, + "description": "`target/setPower` params: move the target to `power`. Requires\n`capabilities.power`. A target already in, or moving toward, that state\nis observed without further action. Resuming may replace the underlying\nhost process. A state outside the template's `powerStates` is\n`invalidParams`; a target whose `phase` is not `active` is\n`failedPrecondition` (`targetNotActive`). The provider stays the owner\nof power afterwards: its own idle policy may move the target again.", + "properties": { + "power": { + "allOf": [ + { + "$ref": "#/components/schemas/PowerState" + } + ], + "description": "One of the template's `powerStates`." + }, + "target": { + "$ref": "#/components/schemas/TargetRef" + } + }, + "required": [ + "target", + "power" + ], + "type": "object" + }, + "TargetObservation": { + "additionalProperties": false, + "description": "What the provider observes about a target. Every target and ingress\nmethod answers with one; Lightspeed records it for display only.", + "properties": { + "ingress": { + "anyOf": [ + { + "$ref": "#/components/schemas/IngressObservation" + } + ], + "description": "Public ingress as realized; absent when disabled." + }, + "message": { + "anyOf": [ + { + "$ref": "#/components/schemas/DescriptiveText" + } + ], + "description": "Why the target is `failed` or `unknown`, or what a transition waits\nfor. Providers build it with `DescriptiveText::lossy` from\nwhatever their backend reports." + }, + "owner": { + "allOf": [ + { + "$ref": "#/components/schemas/TargetOwner" + } + ], + "description": "Echoed from the request." + }, + "phase": { + "$ref": "#/components/schemas/TargetPhase" + }, + "power": { + "anyOf": [ + { + "$ref": "#/components/schemas/PowerState" + } + ], + "description": "Present exactly when `phase` is `active`." + }, + "targetId": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProviderTargetId" + } + ], + "description": "The provider's id for the target. Absent in two cases only: while\n`phase` is `provisioning` and the provider has not assigned an id\nyet (an asynchronous create answers before it has one; the runtime\nrecords the id from the first observation that carries it), and in a\n`closed` observation of an owner the provider holds no target for." + } + }, + "required": [ + "owner", + "phase" + ], + "type": "object" + }, + "TargetOwner": { + "additionalProperties": false, + "description": "The Lightspeed environment a target belongs to. A provider owns at most\none target per owner.", + "properties": { + "environmentId": { + "$ref": "#/components/schemas/EnvironmentId" + }, + "universeId": { + "$ref": "#/components/schemas/UniverseId" + } + }, + "required": [ + "universeId", + "environmentId" + ], + "type": "object" + }, + "TargetPhase": { + "description": "Lifecycle phase of a target.", + "oneOf": [ + { + "const": "provisioning", + "description": "Being created; not yet usable.", + "type": "string" + }, + { + "const": "active", + "description": "Exists; see `power`.", + "type": "string" + }, + { + "const": "closing", + "description": "Being destroyed.", + "type": "string" + }, + { + "const": "closed", + "description": "Gone, and nothing of it remains.", + "type": "string" + }, + { + "const": "failed", + "description": "Broken beyond the provider's own recovery; see `message`.", + "type": "string" + }, + { + "const": "unknown", + "description": "The provider cannot tell; see `message`.", + "type": "string" + } + ] + }, + "TargetRef": { + "additionalProperties": false, + "description": "A reference to an existing target: its owner and, once known, its id.\nWithout `targetId` the provider resolves the owner's target itself.", + "properties": { + "owner": { + "$ref": "#/components/schemas/TargetOwner" + }, + "targetId": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProviderTargetId" + } + ] + } + }, + "required": [ + "owner" + ], + "type": "object" + }, + "TargetResponse": { + "additionalProperties": false, + "description": "Response of every target method and both ingress methods.", + "properties": { + "target": { + "$ref": "#/components/schemas/TargetObservation" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "TemplateId": { + "description": "`templateId`: the provider's own id for a template.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "TemplateResources": { + "additionalProperties": false, + "description": "Compute a template's targets get, in provider-neutral units. Each\nfield is absent when the provider does not fix it.", + "properties": { + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "UniverseId": { + "description": "`universeId` of a Lightspeed universe.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + } + } + }, + "info": { + "description": "JSON-RPC 2.0 over WebSocket text frames of at most 16777216 bytes. Exact-match protocolVersion 4. Errors carry code, message and data.kind; see x-lightspeed-error-kinds.", + "title": "Lightspeed environment controller plane", + "version": "4" + }, + "methods": [ + { + "name": "initialize", + "paramStructure": "by-name", + "params": [ + { + "name": "clientInfo", + "required": true, + "schema": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + { + "name": "protocolVersion", + "required": true, + "schema": { + "description": "The client's `CONTROLLER_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ControllerInitializeResponse" + } + }, + "summary": "Check the protocol version and read the controller's capabilities.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "handshake", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ControllerInitializeParams", + "x-lightspeed-result-type": "ControllerInitializeResponse" + }, + { + "name": "template/list", + "paramStructure": "by-name", + "params": [ + { + "name": "universeId", + "required": true, + "schema": { + "$ref": "#/components/schemas/UniverseId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ListTemplatesResponse" + } + }, + "summary": "List the templates a universe may create targets from.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "template", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ListTemplatesParams", + "x-lightspeed-result-type": "ListTemplatesResponse" + }, + { + "name": "target/create", + "paramStructure": "by-name", + "params": [ + { + "name": "displayName", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/DisplayName" + } + ], + "description": "Label for the target, where the provider shows one." + } + }, + { + "name": "idle", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/IdlePolicy" + } + ], + "description": "Overrides the template's `idle` for this target. A policy that does\nnot validate (`IdlePolicy::validate`) is `invalidParams`." + } + }, + { + "name": "owner", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetOwner" + } + }, + { + "name": "templateId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TemplateId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Create the owner's target, or observe the one it has.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "CreateTargetParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "target/get", + "paramStructure": "by-name", + "params": [ + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Observe a target without changing it.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "GetTargetParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "target/setPower", + "paramStructure": "by-name", + "params": [ + { + "name": "power", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PowerState" + } + ], + "description": "One of the template's `powerStates`." + } + }, + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Move a target to a power state.", + "x-lightspeed-capability": "power", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "SetTargetPowerParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "target/close", + "paramStructure": "by-name", + "params": [ + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Destroy the owner's target and everything it holds.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "target", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "CloseTargetParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "ingress/ensure", + "paramStructure": "by-name", + "params": [ + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Route public ingress to a target.", + "x-lightspeed-capability": "ingress", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "ingress", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "EnsureIngressParams", + "x-lightspeed-result-type": "TargetResponse" + }, + { + "name": "ingress/remove", + "paramStructure": "by-name", + "params": [ + { + "name": "target", + "required": true, + "schema": { + "$ref": "#/components/schemas/TargetRef" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TargetResponse" + } + }, + "summary": "Stop routing public ingress to a target that stays.", + "x-lightspeed-capability": "ingress", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "ingress", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "RemoveIngressParams", + "x-lightspeed-result-type": "TargetResponse" + } + ], + "openrpc": "1.3.2", + "x-lightspeed-data-route": { + "maxHoldMs": 5000, + "refusal": { + "$ref": "#/components/schemas/DataRouteRefusal" + }, + "retryAfterMaxSecs": 60 + }, + "x-lightspeed-error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "x-lightspeed-error-kinds": [ + { + "code": -32700, + "kind": "parseError", + "retryable": false + }, + { + "code": -32600, + "kind": "invalidRequest", + "retryable": false + }, + { + "code": -32601, + "kind": "methodNotFound", + "retryable": false + }, + { + "code": -32602, + "kind": "invalidParams", + "retryable": false + }, + { + "code": -32002, + "kind": "notInitialized", + "retryable": false + }, + { + "code": -32005, + "kind": "versionMismatch", + "retryable": false + }, + { + "code": -32004, + "kind": "notFound", + "retryable": false + }, + { + "code": -32009, + "kind": "alreadyExists", + "retryable": false + }, + { + "code": -32007, + "kind": "failedPrecondition", + "retryable": false + }, + { + "code": -32003, + "kind": "permissionDenied", + "retryable": false + }, + { + "code": -32006, + "kind": "capabilityUnavailable", + "retryable": false + }, + { + "code": -32008, + "kind": "busy", + "retryable": true + }, + { + "code": -32015, + "kind": "deadlineExceeded", + "retryable": false + }, + { + "code": -32016, + "kind": "expired", + "retryable": false + }, + { + "code": -32017, + "kind": "aborted", + "retryable": true + }, + { + "code": -32013, + "kind": "payloadTooLarge", + "retryable": false + }, + { + "code": -32018, + "kind": "processFailed", + "retryable": false + }, + { + "code": -32019, + "kind": "unavailable", + "retryable": true + }, + { + "code": -32603, + "kind": "internal", + "retryable": false + } + ], + "x-lightspeed-idle-policy": { + "$ref": "#/components/schemas/IdlePolicy" + }, + "x-lightspeed-plane": "controller" +} diff --git a/crates/environment-protocol/contract/controller.schema.json b/crates/environment-protocol/contract/controller.schema.json new file mode 100644 index 000000000..bc4b3af69 --- /dev/null +++ b/crates/environment-protocol/contract/controller.schema.json @@ -0,0 +1,1362 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "power", + "description": "The controller's `power` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "CloseTargetParams": { + "additionalProperties": false, + "description": "`target/close` params: destroy the owner's target and every resource it\nholds, public ingress included. Idempotent. Graceful within the\nprovider's own bound, then forced. Answers `closing` until the target is\ngone, then `closed`; `closed` also when the provider holds no target for\nthe owner.", + "properties": { + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "ControllerCapabilities": { + "additionalProperties": false, + "description": "Optional controller methods. Calling an absent one is\n`capabilityUnavailable`.", + "properties": { + "ingress": { + "description": "`ingress/ensure` and `ingress/remove` are served.", + "type": "boolean" + }, + "power": { + "description": "`target/setPower` is served.", + "type": "boolean" + } + }, + "type": "object" + }, + "ControllerInitializeParams": { + "additionalProperties": false, + "description": "`initialize` params on the controller plane.", + "properties": { + "clientInfo": { + "$ref": "#/definitions/ImplementationInfo" + }, + "protocolVersion": { + "description": "The client's `CONTROLLER_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "clientInfo" + ], + "type": "object" + }, + "ControllerInitializeResponse": { + "additionalProperties": false, + "description": "`initialize` response on the controller plane.", + "properties": { + "capabilities": { + "$ref": "#/definitions/ControllerCapabilities" + }, + "plane": { + "allOf": [ + { + "$ref": "#/definitions/Plane" + } + ], + "description": "Always `controller`." + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "serverInfo": { + "$ref": "#/definitions/ImplementationInfo" + } + }, + "required": [ + "protocolVersion", + "plane", + "serverInfo", + "capabilities" + ], + "type": "object" + }, + "CreateTargetParams": { + "additionalProperties": false, + "description": "`target/create` params. Idempotent per owner: when the owner already\nhas a target, the provider returns its observation and applies no\ncreation policy (template deprecation, image checks). Handing out a\ntarget from a warm pool counts as a create.", + "properties": { + "displayName": { + "anyOf": [ + { + "$ref": "#/definitions/DisplayName" + } + ], + "description": "Label for the target, where the provider shows one." + }, + "idle": { + "anyOf": [ + { + "$ref": "#/definitions/IdlePolicy" + } + ], + "description": "Overrides the template's `idle` for this target. A policy that does\nnot validate (`IdlePolicy::validate`) is `invalidParams`." + }, + "owner": { + "$ref": "#/definitions/TargetOwner" + }, + "templateId": { + "$ref": "#/definitions/TemplateId" + } + }, + "required": [ + "owner", + "templateId" + ], + "type": "object" + }, + "DataRouteRefusal": { + "description": "Body of the HTTP refusal a data route answers instead of upgrading, with\n`Content-Type: application/json`. A provider data route uses `waking`,\n`booting`, `gone` and `failed`; the gateway's worker route adds\n`provisioning`, `disconnected` and `away` and forwards a provider's\nrefusal unchanged. Every 503 carries `Retry-After` in whole seconds,\n1 to `RETRY_AFTER_MAX_SECS`.\nThe body is an HTTP refusal, not a JSON-RPC message, and is tagged by\n`state`.", + "oneOf": [ + { + "additionalProperties": false, + "description": "503: resuming from a powered-down state.", + "properties": { + "state": { + "const": "waking", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "503: running, envd not answering yet.", + "properties": { + "state": { + "const": "booting", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "503 (gateway): the environment is still provisioning.", + "properties": { + "state": { + "const": "provisioning", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "503 (gateway): the registered daemon is not connected.", + "properties": { + "state": { + "const": "disconnected", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "410 (gateway): the registered daemon has been away past its\nreconnect bound.", + "properties": { + "state": { + "const": "away", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "410: the target no longer exists.", + "properties": { + "state": { + "const": "gone", + "type": "string" + } + }, + "required": [ + "state" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "409: broken beyond the provider's recovery.", + "properties": { + "message": { + "$ref": "#/definitions/DescriptiveText" + }, + "state": { + "const": "failed", + "type": "string" + } + }, + "required": [ + "state", + "message" + ], + "type": "object" + } + ] + }, + "DescriptiveText": { + "description": "A short human-readable text: 1 to 1024 bytes without control characters\nother than line feed and tab. Never parsed.", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DisplayName": { + "description": "A display name: 1 to 128 bytes without control characters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "EnsureIngressParams": { + "additionalProperties": false, + "description": "`ingress/ensure` params: route the template's public ingress to the\ntarget. Idempotent. A target whose template has no `publicIngress` is\n`failedPrecondition` (`ingressUnsupported`).", + "properties": { + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "EnvironmentId": { + "description": "`environmentId` of a Lightspeed environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "EnvironmentTemplate": { + "additionalProperties": false, + "description": "A template targets are created from.", + "properties": { + "deprecated": { + "description": "Creates no new targets; existing targets keep working.", + "type": "boolean" + }, + "description": { + "anyOf": [ + { + "$ref": "#/definitions/DescriptiveText" + } + ] + }, + "displayName": { + "$ref": "#/definitions/DisplayName" + }, + "idle": { + "anyOf": [ + { + "$ref": "#/definitions/IdlePolicy" + } + ], + "description": "Idle policy targets get when `target/create` names none; absent when\nthe provider applies none by default." + }, + "metadata": { + "allOf": [ + { + "$ref": "#/definitions/Metadata" + } + ], + "description": "Provider-specific display facts, such as an image fingerprint." + }, + "powerStates": { + "description": "Power states `target/setPower` can move this template's targets to.\nEmpty when the provider offers no power control beyond create and\nclose.", + "items": { + "$ref": "#/definitions/PowerState" + }, + "type": "array" + }, + "publicIngress": { + "description": "Targets of this template can serve public ingress.", + "type": "boolean" + }, + "resources": { + "anyOf": [ + { + "$ref": "#/definitions/TemplateResources" + } + ] + }, + "templateId": { + "$ref": "#/definitions/TemplateId" + } + }, + "required": [ + "templateId", + "displayName" + ], + "type": "object" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/definitions/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/definitions/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "GetTargetParams": { + "additionalProperties": false, + "description": "`target/get` params. A pure read: it never creates or changes anything.\nA target that no longer exists is `notFound`.", + "properties": { + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "IdlePolicy": { + "additionalProperties": false, + "description": "Staged idle policy. Each set threshold is measured against envd\n`env/getIdle` `idleForMs`, and the set thresholds are non-decreasing in\nthe order pause → suspend → stop → close. Every stage is optional; a\npolicy sets at least one.", + "properties": { + "closeAfterMs": { + "description": "Close the target after this long idle; the provider then reports it\n`closed`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "pauseAfterMs": { + "description": "Pause the target after this long idle.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "stopAfterMs": { + "description": "Stop the target after this long idle.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "suspendAfterMs": { + "description": "Suspend the target after this long idle.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "IngressObservation": { + "additionalProperties": false, + "description": "Realized public ingress of a target.", + "properties": { + "endpoint": { + "allOf": [ + { + "$ref": "#/definitions/PublicUrl" + } + ], + "description": "Public URL that reaches the target's application." + } + }, + "required": [ + "endpoint" + ], + "type": "object" + }, + "ListTemplatesParams": { + "additionalProperties": false, + "description": "`template/list` params: the universe whose targets the templates would\ncreate. A provider may offer each universe a different set.", + "properties": { + "universeId": { + "$ref": "#/definitions/UniverseId" + } + }, + "required": [ + "universeId" + ], + "type": "object" + }, + "ListTemplatesResponse": { + "additionalProperties": false, + "description": "`template/list` response.", + "properties": { + "templates": { + "items": { + "$ref": "#/definitions/EnvironmentTemplate" + }, + "type": "array" + } + }, + "type": "object" + }, + "Metadata": { + "additionalProperties": { + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "maxProperties": 32, + "propertyNames": { + "maxLength": 64, + "minLength": 1, + "not": { + "pattern": "^lightspeed\\." + }, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "Plane": { + "description": "The plane a connection serves. Each `initialize` response names its\nplane, so a client that reached the wrong socket fails at the handshake.", + "oneOf": [ + { + "const": "data", + "description": "envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "controller", + "description": "A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "registration", + "description": "The registration channel between a daemon and the gateway, versioned\nby `ENVD_PROTOCOL_VERSION`.", + "type": "string" + } + ] + }, + "PowerState": { + "description": "Steady power state of an active target, in provider-neutral terms.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "paused", + "description": "Execution halted, memory retained; resuming is fast.", + "type": "string" + }, + { + "const": "suspended", + "description": "Execution state persisted, compute released; resuming restores it.", + "type": "string" + }, + { + "const": "stopped", + "description": "Powered off; disks retained.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "ProviderTargetId": { + "description": "`targetId`: the provider's own id for a target, opaque to Lightspeed.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "PublicUrl": { + "description": "A public `http://` or `https://` URL a provider reports, such as a\ntarget's ingress `endpoint`: lowercase scheme, a host and optional port,\noptional path and query, no user information or fragment, at most 2048\nbytes. Validated on decode.", + "maxLength": 2048, + "minLength": 1, + "pattern": "^https?://[^/?#@\\s]+([/?][^#\\s]*)?$", + "type": "string" + }, + "RemoveIngressParams": { + "additionalProperties": false, + "description": "`ingress/remove` params: stop routing public ingress to the target.\nIdempotent.", + "properties": { + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "SetTargetPowerParams": { + "additionalProperties": false, + "description": "`target/setPower` params: move the target to `power`. Requires\n`capabilities.power`. A target already in, or moving toward, that state\nis observed without further action. Resuming may replace the underlying\nhost process. A state outside the template's `powerStates` is\n`invalidParams`; a target whose `phase` is not `active` is\n`failedPrecondition` (`targetNotActive`). The provider stays the owner\nof power afterwards: its own idle policy may move the target again.", + "properties": { + "power": { + "allOf": [ + { + "$ref": "#/definitions/PowerState" + } + ], + "description": "One of the template's `powerStates`." + }, + "target": { + "$ref": "#/definitions/TargetRef" + } + }, + "required": [ + "target", + "power" + ], + "type": "object" + }, + "TargetObservation": { + "additionalProperties": false, + "description": "What the provider observes about a target. Every target and ingress\nmethod answers with one; Lightspeed records it for display only.", + "properties": { + "ingress": { + "anyOf": [ + { + "$ref": "#/definitions/IngressObservation" + } + ], + "description": "Public ingress as realized; absent when disabled." + }, + "message": { + "anyOf": [ + { + "$ref": "#/definitions/DescriptiveText" + } + ], + "description": "Why the target is `failed` or `unknown`, or what a transition waits\nfor. Providers build it with `DescriptiveText::lossy` from\nwhatever their backend reports." + }, + "owner": { + "allOf": [ + { + "$ref": "#/definitions/TargetOwner" + } + ], + "description": "Echoed from the request." + }, + "phase": { + "$ref": "#/definitions/TargetPhase" + }, + "power": { + "anyOf": [ + { + "$ref": "#/definitions/PowerState" + } + ], + "description": "Present exactly when `phase` is `active`." + }, + "targetId": { + "anyOf": [ + { + "$ref": "#/definitions/ProviderTargetId" + } + ], + "description": "The provider's id for the target. Absent in two cases only: while\n`phase` is `provisioning` and the provider has not assigned an id\nyet (an asynchronous create answers before it has one; the runtime\nrecords the id from the first observation that carries it), and in a\n`closed` observation of an owner the provider holds no target for." + } + }, + "required": [ + "owner", + "phase" + ], + "type": "object" + }, + "TargetOwner": { + "additionalProperties": false, + "description": "The Lightspeed environment a target belongs to. A provider owns at most\none target per owner.", + "properties": { + "environmentId": { + "$ref": "#/definitions/EnvironmentId" + }, + "universeId": { + "$ref": "#/definitions/UniverseId" + } + }, + "required": [ + "universeId", + "environmentId" + ], + "type": "object" + }, + "TargetPhase": { + "description": "Lifecycle phase of a target.", + "oneOf": [ + { + "const": "provisioning", + "description": "Being created; not yet usable.", + "type": "string" + }, + { + "const": "active", + "description": "Exists; see `power`.", + "type": "string" + }, + { + "const": "closing", + "description": "Being destroyed.", + "type": "string" + }, + { + "const": "closed", + "description": "Gone, and nothing of it remains.", + "type": "string" + }, + { + "const": "failed", + "description": "Broken beyond the provider's own recovery; see `message`.", + "type": "string" + }, + { + "const": "unknown", + "description": "The provider cannot tell; see `message`.", + "type": "string" + } + ] + }, + "TargetRef": { + "additionalProperties": false, + "description": "A reference to an existing target: its owner and, once known, its id.\nWithout `targetId` the provider resolves the owner's target itself.", + "properties": { + "owner": { + "$ref": "#/definitions/TargetOwner" + }, + "targetId": { + "anyOf": [ + { + "$ref": "#/definitions/ProviderTargetId" + } + ] + } + }, + "required": [ + "owner" + ], + "type": "object" + }, + "TargetResponse": { + "additionalProperties": false, + "description": "Response of every target method and both ingress methods.", + "properties": { + "target": { + "$ref": "#/definitions/TargetObservation" + } + }, + "required": [ + "target" + ], + "type": "object" + }, + "TemplateId": { + "description": "`templateId`: the provider's own id for a template.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "TemplateResources": { + "additionalProperties": false, + "description": "Compute a template's targets get, in provider-neutral units. Each\nfield is absent when the provider does not fix it.", + "properties": { + "diskBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "memoryBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "vcpus": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "UniverseId": { + "description": "`universeId` of a Lightspeed universe.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + } + }, + "description": "Every wire type of the controller plane, protocol version 4.", + "title": "Lightspeed environment controller plane", + "x-lightspeed-data-route": { + "maxHoldMs": 5000, + "refusal": { + "$ref": "#/definitions/DataRouteRefusal" + }, + "retryAfterMaxSecs": 60 + }, + "x-lightspeed-idle-policy": { + "$ref": "#/definitions/IdlePolicy" + } +} diff --git a/crates/environment-protocol/contract/data.openrpc.json b/crates/environment-protocol/contract/data.openrpc.json new file mode 100644 index 000000000..d92332897 --- /dev/null +++ b/crates/environment-protocol/contract/data.openrpc.json @@ -0,0 +1,5763 @@ +{ + "components": { + "schemas": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "AppendTransferInventoryParams": { + "additionalProperties": false, + "description": "`transfer/append` params: add incoming inventory entries in order.\nRetrying an identical page is safe.", + "properties": { + "entries": { + "description": "At most `limits.filesystem.transfer.maxInventoryPageEntries`.", + "items": { + "$ref": "#/components/schemas/InventoryEntry" + }, + "type": "array" + }, + "index": { + "description": "Index of the page's first entry: the number of entries appended\nbefore it.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "last": { + "description": "This page completes the inventory.", + "type": "boolean" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "index", + "entries" + ], + "type": "object" + }, + "BeginTransferParams": { + "additionalProperties": false, + "description": "`transfer/begin` params. Idempotent: a repeat with identical input\nreturns the operation's status; different input under the same id is\n`failedPrecondition` (`operationIdInUse`).", + "properties": { + "limits": { + "$ref": "#/components/schemas/TransferLimits" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + }, + "selection": { + "$ref": "#/components/schemas/TransferSelection" + } + }, + "required": [ + "operationId", + "selection", + "limits" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "ByteChunk": { + "contentEncoding": "base64", + "description": "Payload bytes, always named `data` on the wire, as padded standard\nbase64.", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$", + "type": "string" + }, + "CancelJobsParams": { + "additionalProperties": false, + "description": "`job/cancel` params.", + "properties": { + "force": { + "description": "Kill at once instead of `SIGTERM`, `limits.job.cancelGraceMs`, then\n`SIGKILL`.", + "type": "boolean" + }, + "jobIds": { + "description": "1 to `limits.job.maxRequestJobs` ids.", + "items": { + "$ref": "#/components/schemas/JobId" + }, + "type": "array" + }, + "scope": { + "allOf": [ + { + "$ref": "#/components/schemas/JobCancelScope" + } + ], + "description": "`job` by default." + } + }, + "required": [ + "jobIds" + ], + "type": "object" + }, + "CancelJobsResponse": { + "additionalProperties": false, + "description": "`job/cancel` response: one entry per affected job, named jobs first in\nrequest order, then dependents, at most `limits.job.maxRequestJobs`\nentries in total.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/components/schemas/JobSummaryEntry" + }, + "type": "array" + }, + "unlistedJobCount": { + "description": "Dependents cancelled as well but not listed, beyond the entry limit.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "CancelRequestParams": { + "additionalProperties": false, + "description": "`$/cancelRequest` params.", + "properties": { + "id": { + "allOf": [ + { + "$ref": "#/components/schemas/Id" + } + ], + "description": "Id of the request to abort." + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "power", + "description": "The controller's `power` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "CommandSpec": { + "additionalProperties": false, + "description": "What to run. Shared by `process/start` and `job/start`.", + "properties": { + "argv": { + "description": "Program and arguments; `argv[0]` is resolved through `PATH`. Not\nempty.", + "items": { + "type": "string" + }, + "type": "array" + }, + "cwd": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "Working directory; the daemon's `defaultCwd` when absent." + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "description": "Variables added to the daemon's scrubbed environment.", + "type": "object" + }, + "input": { + "allOf": [ + { + "$ref": "#/components/schemas/ExecInput" + } + ], + "description": "Standard input; none by default." + }, + "secretEnv": { + "additionalProperties": { + "$ref": "#/components/schemas/SecretString" + }, + "description": "Like `env`, but every value is redacted from the command's output.", + "type": "object" + }, + "timeoutMs": { + "description": "Stop deadline from start. On expiry the daemon terminates the process\ngroup (`SIGTERM`, the advertised grace, then `SIGKILL`) and records\n`stoppedBy: timeout`. Absent means no deadline.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "argv" + ], + "type": "object" + }, + "ContentDigest": { + "description": "`sha256:` followed by 64 lowercase hex digits: the digest of a file's raw\nbytes, independent of its path and mode.", + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + "CreateDirectoryParams": { + "additionalProperties": false, + "description": "`fs/createDirectory` params.", + "properties": { + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "recursive": { + "description": "Create missing parents, and succeed when the directory exists.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "CreateDirectoryResponse": { + "additionalProperties": false, + "description": "`fs/createDirectory` response.", + "type": "object" + }, + "DaemonInstanceId": { + "description": "`daemonInstanceId`: random per daemon start. A change between two\nconnections means the daemon restarted and every process handle from\nthe earlier instance is gone.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DataCapabilities": { + "additionalProperties": false, + "description": "Capability families. A present family means its base methods exist; a\nfamily's fields are facts that vary per instance. A request to an\nabsent family, or one needing an absent fact, is\n`capabilityUnavailable`.", + "properties": { + "filesystem": { + "anyOf": [ + { + "$ref": "#/components/schemas/FilesystemCapabilities" + } + ], + "description": "`fs/*` (search, glob and scan included) and `transfer/*`." + }, + "job": { + "anyOf": [ + { + "$ref": "#/components/schemas/JobCapabilities" + } + ], + "description": "`job/*`." + }, + "process": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProcessCapabilities" + } + ], + "description": "`process/*`." + } + }, + "type": "object" + }, + "DataLimits": { + "additionalProperties": false, + "description": "Operational ceilings this daemon enforces. A request above a ceiling is\n`invalidParams` naming the field and the ceiling; nothing is clamped.", + "properties": { + "filesystem": { + "$ref": "#/components/schemas/FilesystemLimits" + }, + "job": { + "$ref": "#/components/schemas/JobLimits" + }, + "maxConcurrentRequests": { + "description": "Requests in flight per connection before `busy`; at least\n`MIN_CONCURRENT_REQUESTS`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxWaitMs": { + "description": "Largest `waitMs` of `process/read` and `job/read`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "process": { + "$ref": "#/components/schemas/ProcessLimits" + } + }, + "required": [ + "maxConcurrentRequests", + "maxWaitMs", + "process", + "job", + "filesystem" + ], + "type": "object" + }, + "DirectoryCursor": { + "description": "`cursor` and `nextCursor` of `fs/readDirectory`: an opaque position\nthe daemon mints, echoed back unchanged.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DirectoryEntry": { + "additionalProperties": false, + "description": "One `fs/readDirectory` entry.", + "properties": { + "metadata": { + "allOf": [ + { + "$ref": "#/components/schemas/FileMetadata" + } + ], + "description": "The entry itself, not a symlink's target. Entries that vanish while\nthe page is built are left out." + }, + "name": { + "$ref": "#/components/schemas/EntryName" + } + }, + "required": [ + "name", + "metadata" + ], + "type": "object" + }, + "EntryName": { + "description": "One entry name: a single path segment, neither `.` nor `..`, without\n`/` or NUL, at most 4096 bytes. `fs/readDirectory` entries use it.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(?!\\.\\.?$)[^/\\u0000]+$", + "type": "string" + }, + "EnvironmentPath": { + "description": "An absolute, normalized POSIX path in the environment: `/`, or `/` and\none or more segments joined by `/`. Empty, `.` and `..` segments and a\ntrailing `/` are rejected, never rewritten. At most 4096 bytes.\n`defaultCwd`, `homeDirectory`, every `cwd` and every path parameter use\nthis type; see the module's path model and confinement rule.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(/|(/(?!\\.\\.?(/|$))[^/\\u0000]+)+)$", + "type": "string" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/components/schemas/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/components/schemas/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "ExecInput": { + "description": "Standard input of a command.", + "oneOf": [ + { + "additionalProperties": false, + "description": "`/dev/null`: the command reads end of file. Model-facing tools use\nthis.", + "properties": { + "kind": { + "const": "none", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "These bytes, then end of file. Written by a writer task after the\noutput readers, the exit watcher and the deadline are in place.", + "properties": { + "data": { + "$ref": "#/components/schemas/ByteChunk" + }, + "kind": { + "const": "bytes", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A persistent pipe fed by `process/write` until closed. Processes\nonly; `job/start` refuses it with `invalidParams`.", + "properties": { + "kind": { + "const": "pipe", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "ExecState": { + "description": "Lifecycle state of a process.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "exited", + "description": "The root exited and its output settled.", + "type": "string" + } + ] + }, + "ExitStatus": { + "description": "How the root process ended.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "kind": { + "const": "code", + "type": "string" + } + }, + "required": [ + "kind", + "code" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Killed by this signal.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "kind": { + "const": "signal", + "type": "string" + }, + "signal": { + "$ref": "#/components/schemas/SignalName" + } + }, + "required": [ + "kind", + "signal" + ], + "type": "object" + } + ] + }, + "FileKind": { + "description": "Kind of a filesystem entry.", + "oneOf": [ + { + "enum": [ + "file", + "directory", + "symlink" + ], + "type": "string" + }, + { + "const": "other", + "description": "Device, FIFO, socket or another special file.", + "type": "string" + } + ] + }, + "FileMetadata": { + "additionalProperties": false, + "description": "One file-entry shape for `fs/getMetadata` and `fs/readDirectory`.", + "properties": { + "createdAtMs": { + "description": "Absent when the filesystem records no birth time.", + "format": "int64", + "type": "integer" + }, + "executable": { + "description": "A regular file with any execute bit set; always false for\ndirectories, symlinks and special files.", + "type": "boolean" + }, + "kind": { + "$ref": "#/components/schemas/FileKind" + }, + "modifiedAtMs": { + "format": "int64", + "type": "integer" + }, + "sizeBytes": { + "description": "Size in bytes; for a symlink, the length of its target text.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "symlinkTarget": { + "description": "Target text of a symlink, when it is UTF-8.", + "type": "string" + }, + "version": { + "allOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Changes whenever the entry's content or identity changes; compare it\nwith `ifMatch`." + } + }, + "required": [ + "kind", + "sizeBytes", + "modifiedAtMs", + "version" + ], + "type": "object" + }, + "FileVersion": { + "description": "Opaque version token of a file's content and identity. Equal tokens for\none path mean the content is unchanged. 1 to 255 bytes without control\ncharacters.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "FilesystemCapabilities": { + "additionalProperties": false, + "description": "The `filesystem` family: `fs/*` and `transfer/*`.", + "properties": { + "write": { + "description": "Methods that change the filesystem are served; false on a daemon\nstarted read-only.", + "type": "boolean" + } + }, + "type": "object" + }, + "FilesystemLimits": { + "additionalProperties": false, + "description": "Ceilings of `fs/*` and `transfer/*`.", + "properties": { + "glob": { + "$ref": "#/components/schemas/GlobCeilings" + }, + "maxDirectoryEntries": { + "description": "Largest `limit` of `fs/readDirectory`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxReadFileBytes": { + "description": "Largest `maxBytes` of `fs/readFile`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxWriteFileBytes": { + "description": "Largest `data` of `fs/writeFile`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "scan": { + "$ref": "#/components/schemas/ScanCeilings" + }, + "search": { + "$ref": "#/components/schemas/SearchCeilings" + }, + "transfer": { + "$ref": "#/components/schemas/TransferCeilings" + } + }, + "required": [ + "maxReadFileBytes", + "maxWriteFileBytes", + "maxDirectoryEntries", + "search", + "glob", + "scan", + "transfer" + ], + "type": "object" + }, + "GapReason": { + "description": "Why a range of output cannot be returned.", + "oneOf": [ + { + "const": "overflow", + "description": "Never delivered: dropped from the middle because retained output\nexceeded the handle's cap. The head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Already delivered through the daemon cursor (or acknowledged) and\nreleased since.", + "type": "string" + } + ] + }, + "GetIdleParams": { + "additionalProperties": false, + "description": "`env/getIdle` params.", + "type": "object" + }, + "GetIdleResponse": { + "additionalProperties": false, + "description": "`env/getIdle` response.", + "properties": { + "idleForMs": { + "description": "Milliseconds since the last `use` request, or since running work\nlast ended, whichever is later. Zero while any process or job runs\nor any leftover process group has live members; the clock restarts\nwhen the last of them ends.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "leftoverProcessGroupCount": { + "description": "Process groups of exited processes that still have members alive:\nservices a command started and left running. They hold `idleForMs`\nat zero like running work.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "runningJobCount": { + "description": "Jobs that are `running`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "runningProcessCount": { + "description": "Processes started with `process/start` whose root has not exited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "idleForMs" + ], + "type": "object" + }, + "GetMetadataParams": { + "additionalProperties": false, + "description": "`fs/getMetadata` params.", + "properties": { + "followSymlinks": { + "description": "Describe the symlink's target instead of the link (subject to\nconfinement).", + "type": "boolean" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "GetMetadataResponse": { + "additionalProperties": false, + "description": "`fs/getMetadata` response.", + "properties": { + "metadata": { + "$ref": "#/components/schemas/FileMetadata" + } + }, + "required": [ + "metadata" + ], + "type": "object" + }, + "GlobCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/globFiles` quotas, field for field.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxDurationMs" + ], + "type": "object" + }, + "GlobFilesLimits": { + "additionalProperties": false, + "description": "Caller quotas of one enumeration; each at most its advertised ceiling.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "description": "Non-directory entries visited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxDurationMs" + ], + "type": "object" + }, + "GlobFilesParams": { + "additionalProperties": false, + "description": "`fs/globFiles` params: bounded recursive enumeration of non-directory\nentries matching `filter.include`.", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeFilter" + } + ], + "description": "`include` must not be empty." + }, + "limits": { + "$ref": "#/components/schemas/GlobFilesLimits" + }, + "root": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "root", + "filter", + "limits" + ], + "type": "object" + }, + "GlobFilesResponse": { + "additionalProperties": false, + "description": "`fs/globFiles` response.", + "properties": { + "elapsedMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "matches": { + "description": "In walk order (sorted by path).", + "items": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Entries the walk could not represent or list: names that are not\nUTF-8 and directories that could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "stopped": { + "anyOf": [ + { + "$ref": "#/components/schemas/GlobFilesStop" + } + ] + }, + "visitedFileCount": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "visitedFileCount", + "elapsedMs" + ], + "type": "object" + }, + "GlobFilesStop": { + "description": "Why an enumeration stopped before finishing its walk.", + "oneOf": [ + { + "const": "matchLimit", + "description": "`maxMatches` paths were found.", + "type": "string" + }, + { + "const": "fileLimit", + "description": "`maxFiles` non-directory entries were visited.", + "type": "string" + }, + { + "const": "timeLimit", + "description": "`maxDurationMs` ran out.", + "type": "string" + }, + { + "const": "responseLimit", + "description": "The next match would push the encoded matches past\n`MAX_RESULT_BYTES`.", + "type": "string" + } + ] + }, + "Id": { + "anyOf": [ + { + "format": "int64", + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "A request id: an integer or a string, echoed in the response." + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "InitializeParams": { + "additionalProperties": false, + "description": "`initialize` params on the data plane.", + "properties": { + "clientInfo": { + "$ref": "#/components/schemas/ImplementationInfo" + }, + "protocolVersion": { + "description": "The client's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "clientInfo" + ], + "type": "object" + }, + "InitializeResponse": { + "additionalProperties": false, + "description": "`initialize` response on the data plane.", + "properties": { + "capabilities": { + "$ref": "#/components/schemas/DataCapabilities" + }, + "daemonInstanceId": { + "allOf": [ + { + "$ref": "#/components/schemas/DaemonInstanceId" + } + ], + "description": "Random per daemon start; a different value than on an earlier\nconnection means the daemon restarted and the earlier instance's\nprocess handles are gone." + }, + "defaultCwd": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "The daemon's working directory: the default `cwd` of commands." + }, + "homeDirectory": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "Home directory of the user commands run as, when known." + }, + "limits": { + "$ref": "#/components/schemas/DataLimits" + }, + "plane": { + "allOf": [ + { + "$ref": "#/components/schemas/Plane" + } + ], + "description": "Always `data`." + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "serverInfo": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + "required": [ + "protocolVersion", + "plane", + "daemonInstanceId", + "serverInfo", + "capabilities", + "limits", + "defaultCwd" + ], + "type": "object" + }, + "InventoryEntry": { + "additionalProperties": false, + "description": "One inventory entry. Parents precede children.", + "properties": { + "content": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeContent" + } + ], + "description": "A file entry always carries its `digest`." + }, + "path": { + "allOf": [ + { + "$ref": "#/components/schemas/TreePath" + } + ], + "description": "Relative to the selected root; `.` is the root itself." + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + }, + "JobCancelScope": { + "description": "Which jobs a `job/cancel` affects.", + "oneOf": [ + { + "const": "job", + "description": "Only the named jobs.", + "type": "string" + }, + { + "const": "dependents", + "description": "The named jobs and every job that transitively depends on them.", + "type": "string" + } + ] + }, + "JobCapabilities": { + "additionalProperties": false, + "description": "The `job` family: `job/*`. It has no instance facts.", + "type": "object" + }, + "JobDependency": { + "description": "A dependency reference.", + "oneOf": [ + { + "additionalProperties": false, + "description": "An existing job on this daemon, or one in the same request.", + "properties": { + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "kind": { + "const": "jobId", + "type": "string" + } + }, + "required": [ + "kind", + "jobId" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The `name` of a job in the same `job/start`.", + "properties": { + "kind": { + "const": "name", + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/JobName" + } + }, + "required": [ + "kind", + "name" + ], + "type": "object" + } + ] + }, + "JobDependencyPolicy": { + "description": "What a job waits for from its `dependsOn` jobs.", + "oneOf": [ + { + "const": "allSucceeded", + "description": "Run once every dependency `succeeded`; any other terminal status\nmakes this job `dependencyFailed`.", + "type": "string" + }, + { + "const": "allTerminal", + "description": "Run once every dependency is terminal, whatever its status.", + "type": "string" + } + ] + }, + "JobId": { + "description": "`jobId`: caller-chosen name of one job run on one daemon.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "JobLimits": { + "additionalProperties": false, + "description": "Ceilings of `job/*`.", + "properties": { + "cancelGraceMs": { + "description": "Grace between `SIGTERM` and `SIGKILL` for a non-forced cancel and\ntimeouts.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxReadBytes": { + "description": "Largest `maxBytes` of one `job/read`, across its entries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxRequestJobs": { + "description": "Most jobs one `job/start`, `job/read` or `job/cancel` names, and most\nentries a `job/cancel` response lists.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxRetainedOutputBytes": { + "description": "Output a job retains: the head and the tail, each half of this; the\nmiddle is an `overflow` gap.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "recordRetentionMs": { + "description": "How long a terminal job's record and output are kept.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxReadBytes", + "maxRequestJobs", + "maxRetainedOutputBytes", + "recordRetentionMs", + "cancelGraceMs" + ], + "type": "object" + }, + "JobName": { + "description": "`name` of a job and `dependsOn {kind: name}`: a caller-chosen name,\nunique within one `job/start`, 1 to 128 bytes without control\ncharacters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "JobReadEntry": { + "description": "A `job/read` entry: the job's summary and output page, or why it could\nnot be read (`notFound` for an unknown or garbage-collected job).", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "job", + "type": "string" + }, + "output": { + "$ref": "#/components/schemas/OutputPage" + }, + "summary": { + "$ref": "#/components/schemas/JobSummary" + } + }, + "required": [ + "kind", + "summary", + "output" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "kind": { + "const": "error", + "type": "string" + } + }, + "required": [ + "kind", + "jobId", + "error" + ], + "type": "object" + } + ] + }, + "JobReadRequest": { + "additionalProperties": false, + "description": "One entry of a `job/read`. Job reads never consume: without `after` and\n`tailBytes` the page starts at offset 0. One job may be named by several\nentries, such as a head and a tail page.", + "properties": { + "after": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "maxBytes": { + "description": "Most `data` bytes for this entry; the request's `maxBytes` still\nbounds the total. Absent: whatever of the request's budget remains.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "tailBytes": { + "description": "Start at `end - tailBytes`; excludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobId" + ], + "type": "object" + }, + "JobRequestId": { + "description": "`requestId` of a `job/start` call. Part of each job's identity: a\nretried start with the same request id and spec replays the job; any\nother reuse of the job id is refused.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "JobSpec": { + "additionalProperties": false, + "description": "One job of a `job/start`.", + "properties": { + "command": { + "allOf": [ + { + "$ref": "#/components/schemas/CommandSpec" + } + ], + "description": "`input` must be `none` or `bytes`." + }, + "dependencyPolicy": { + "allOf": [ + { + "$ref": "#/components/schemas/JobDependencyPolicy" + } + ], + "description": "What \"finish\" means for `dependsOn`; `allSucceeded` by default." + }, + "dependsOn": { + "description": "Jobs that must finish first.", + "items": { + "$ref": "#/components/schemas/JobDependency" + }, + "type": "array" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "name": { + "anyOf": [ + { + "$ref": "#/components/schemas/JobName" + } + ], + "description": "Local name other jobs of the same `job/start` can depend on; unique\nwithin the request." + }, + "queueKey": { + "anyOf": [ + { + "$ref": "#/components/schemas/QueueKey" + } + ] + } + }, + "required": [ + "jobId", + "command" + ], + "type": "object" + }, + "JobStatus": { + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Accepted; waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its `timeoutMs` expired and the daemon stopped it.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of `job/cancel`.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "JobSummary": { + "additionalProperties": false, + "description": "What a job is and how it ended. Instants are the guest's wall clock.", + "properties": { + "cancelRequestedAtMs": { + "description": "When a `job/cancel` reached the job while it ran.", + "format": "int64", + "type": "integer" + }, + "createdAtMs": { + "format": "int64", + "type": "integer" + }, + "dependsOn": { + "description": "Resolved ids of the jobs this one depends on.", + "items": { + "$ref": "#/components/schemas/JobId" + }, + "type": "array" + }, + "durationMs": { + "description": "Monotonic run time from start to finish, immune to guest clock jumps.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProtocolError" + } + ], + "description": "Why a `failed` or `interrupted` job produced no exit, or an\ninfrastructure failure while it ran." + }, + "exit": { + "anyOf": [ + { + "$ref": "#/components/schemas/ExitStatus" + } + ], + "description": "How the root process ended, once it ran and exited." + }, + "finishedAtMs": { + "format": "int64", + "type": "integer" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "leftovers": { + "anyOf": [ + { + "$ref": "#/components/schemas/LeftoverReport" + } + ], + "description": "What the daemon swept from the job's group at completion\n(`disposition: swept`)." + }, + "name": { + "anyOf": [ + { + "$ref": "#/components/schemas/JobName" + } + ] + }, + "queueKey": { + "anyOf": [ + { + "$ref": "#/components/schemas/QueueKey" + } + ] + }, + "queuedAtMs": { + "description": "When the job became runnable and waited for its queue.", + "format": "int64", + "type": "integer" + }, + "requestId": { + "$ref": "#/components/schemas/JobRequestId" + }, + "startedAtMs": { + "format": "int64", + "type": "integer" + }, + "status": { + "$ref": "#/components/schemas/JobStatus" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/components/schemas/StoppedBy" + } + ], + "description": "Set when the daemon stopped the running job: `timeout` for\n`timedOut`, `terminate` or `kill` for a cancel without or with\n`force`." + } + }, + "required": [ + "jobId", + "requestId", + "status", + "createdAtMs" + ], + "type": "object" + }, + "JobSummaryEntry": { + "description": "A job summary, or why the named job could not be found.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "job", + "type": "string" + }, + "summary": { + "$ref": "#/components/schemas/JobSummary" + } + }, + "required": [ + "kind", + "summary" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "jobId": { + "$ref": "#/components/schemas/JobId" + }, + "kind": { + "const": "error", + "type": "string" + } + }, + "required": [ + "kind", + "jobId", + "error" + ], + "type": "object" + } + ] + }, + "LeftoverDisposition": { + "description": "What the daemon did with leftovers. The policies differ on purpose.", + "oneOf": [ + { + "const": "kept", + "description": "Left running (processes): they keep the environment's lifetime and\nare reported until they exit.", + "type": "string" + }, + { + "const": "swept", + "description": "Terminated when the job finished (jobs); the submitter should fix\nits script's teardown.", + "type": "string" + } + ] + }, + "LeftoverProcess": { + "additionalProperties": false, + "description": "One leftover process.", + "properties": { + "command": { + "description": "Command line when the platform exposes it, otherwise the executable\nname, cut to\n`MAX_LEFTOVER_COMMAND_BYTES`\non a character boundary; absent when neither could be read.", + "type": "string" + }, + "pid": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "LeftoverReport": { + "additionalProperties": false, + "description": "Processes of a command's group still alive after its root exited.\nPresent only when there is at least one.", + "properties": { + "disposition": { + "$ref": "#/components/schemas/LeftoverDisposition" + }, + "omittedProcessCount": { + "description": "Leftovers beyond the listed ones.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "processes": { + "description": "At most `MAX_LEFTOVER_PROCESSES`,\nlowest pid first; never empty.", + "items": { + "$ref": "#/components/schemas/LeftoverProcess" + }, + "type": "array" + } + }, + "required": [ + "disposition", + "processes" + ], + "type": "object" + }, + "ListTransferMissingParams": { + "additionalProperties": false, + "description": "`transfer/listMissing` params.", + "properties": { + "limit": { + "description": "Most digests to return, 1 to\n`limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "limit" + ], + "type": "object" + }, + "ListTransferMissingResponse": { + "additionalProperties": false, + "description": "`transfer/listMissing` response: digests whose content the daemon still\nneeds. The set shrinks as uploads complete; call again rather than\npaging.", + "properties": { + "digests": { + "items": { + "$ref": "#/components/schemas/ContentDigest" + }, + "type": "array" + }, + "remainingCount": { + "description": "Missing digests beyond this page.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "OutputChunk": { + "additionalProperties": false, + "description": "Contiguous output bytes from one stream.", + "properties": { + "data": { + "$ref": "#/components/schemas/ByteChunk" + }, + "offset": { + "description": "Byte offset of the chunk's first byte in the combined output.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "stream": { + "$ref": "#/components/schemas/OutputStream" + } + }, + "required": [ + "offset", + "stream", + "data" + ], + "type": "object" + }, + "OutputGap": { + "additionalProperties": false, + "description": "Output bytes in `[offset, offset + omittedBytes)` that a page cannot\nreturn.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/components/schemas/GapReason" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, + "OutputPage": { + "additionalProperties": false, + "description": "One page of a handle's output. Returned by `process/read` and in every\n`job/read` entry. A page holds at most\n`MAX_PAGE_CHUNKS` chunks and gaps\ntogether (for `job/read`, across all entries); one that reaches it ends\nearly, and `next` says where to continue.", + "properties": { + "chunks": { + "description": "In offset order, at most `maxBytes` of `data` in total.", + "items": { + "$ref": "#/components/schemas/OutputChunk" + }, + "type": "array" + }, + "end": { + "description": "Bytes produced so far; final once the handle has exited.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "gaps": { + "description": "Unreturnable ranges between the requested start and `next`.", + "items": { + "$ref": "#/components/schemas/OutputGap" + }, + "type": "array" + }, + "next": { + "description": "Offset to read from next: past the last returned byte or gap.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "next", + "end" + ], + "type": "object" + }, + "OutputStream": { + "description": "Which descriptor produced a chunk. Jobs never produce `pty`.", + "oneOf": [ + { + "enum": [ + "stdout", + "stderr" + ], + "type": "string" + }, + { + "const": "pty", + "description": "The PTY master, which merges stdout and stderr.", + "type": "string" + } + ] + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "Plane": { + "description": "The plane a connection serves. Each `initialize` response names its\nplane, so a client that reached the wrong socket fails at the handshake.", + "oneOf": [ + { + "const": "data", + "description": "envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "controller", + "description": "A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "registration", + "description": "The registration channel between a daemon and the gateway, versioned\nby `ENVD_PROTOCOL_VERSION`.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + } + ] + }, + "ProcessCapabilities": { + "additionalProperties": false, + "description": "The `process` family: `process/*`.", + "properties": { + "pty": { + "description": "`process/start` accepts `pty`.", + "type": "boolean" + } + }, + "type": "object" + }, + "ProcessId": { + "description": "`processId`: caller-chosen handle of a process on one daemon.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ProcessLimits": { + "additionalProperties": false, + "description": "Ceilings of `process/*`.", + "properties": { + "maxExitedRetainedBytes": { + "description": "Output retained across all exited processes; beyond it the oldest\nare evicted early.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxInputQueueBytes": { + "description": "Bytes queued for a process's input before `process/write` accepts\nfewer than it was sent.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxReadBytes": { + "description": "Largest `maxBytes` of one `process/read`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxRetainedOutputBytes": { + "description": "Output a process retains; beyond it the oldest delivered bytes are\nreleased (`expired` gaps), then the undelivered middle (`overflow`\ngaps).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "retentionAfterExitMs": { + "description": "How long an exited process stays readable after a read first\nreturned its settled exit with nothing left unread.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "terminateGraceMs": { + "description": "Grace between `SIGTERM` and `SIGKILL` for `terminate` and timeouts.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "unreadRetentionMs": { + "description": "How long an exited process whose output was never fully read stays\nreadable.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxReadBytes", + "maxRetainedOutputBytes", + "retentionAfterExitMs", + "unreadRetentionMs", + "maxExitedRetainedBytes", + "terminateGraceMs", + "maxInputQueueBytes" + ], + "type": "object" + }, + "ProcessSignal": { + "description": "Signal delivered by `process/terminate`.", + "oneOf": [ + { + "const": "interrupt", + "description": "`SIGINT` to the foreground process group (the terminal's under a\nPTY, otherwise the process group). Records nothing; the next read\nshows what the process did.", + "type": "string" + }, + { + "const": "terminate", + "description": "`SIGTERM` to the group, then `SIGKILL` after\n`limits.process.terminateGraceMs` if members remain. Records\n`stoppedBy: terminate`.", + "type": "string" + }, + { + "const": "kill", + "description": "`SIGKILL` to the group. Records `stoppedBy: kill`.", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/components/schemas/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "PtySize": { + "additionalProperties": false, + "description": "Terminal size of a PTY. Each dimension is 1 to 4096.", + "properties": { + "cols": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "rows": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "rows", + "cols" + ], + "type": "object" + }, + "QueueKey": { + "description": "`queueKey`: jobs sharing a key run one at a time, in start order.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ReadDirectoryParams": { + "additionalProperties": false, + "description": "`fs/readDirectory` params. Pages in name order.", + "properties": { + "cursor": { + "anyOf": [ + { + "$ref": "#/components/schemas/DirectoryCursor" + } + ], + "description": "`nextCursor` of the previous page; a cursor this daemon did not mint\nis `invalidParams`." + }, + "limit": { + "description": "Most entries per page, 1 to `limits.filesystem.maxDirectoryEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "path": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "A directory, or a symlink to one inside the root." + } + }, + "required": [ + "path", + "limit" + ], + "type": "object" + }, + "ReadDirectoryResponse": { + "additionalProperties": false, + "description": "`fs/readDirectory` response.", + "properties": { + "entries": { + "items": { + "$ref": "#/components/schemas/DirectoryEntry" + }, + "type": "array" + }, + "nextCursor": { + "anyOf": [ + { + "$ref": "#/components/schemas/DirectoryCursor" + } + ], + "description": "Opaque cursor of the next page; absent on the last page." + }, + "skippedEntryCount": { + "description": "Entries in this page's range skipped because their names are not\nUTF-8. Entries that vanish while the page is built are left out and\nnot counted.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "ReadFileParams": { + "additionalProperties": false, + "description": "`fs/readFile` params. Reads a regular file only; anything else is\n`failedPrecondition` (`notRegularFile`) and is never opened for a\nblocking read.", + "properties": { + "maxBytes": { + "description": "Most bytes to return, at most `limits.filesystem.maxReadFileBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "offset": { + "description": "First byte to return; 0 by default.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path", + "maxBytes" + ], + "type": "object" + }, + "ReadFileResponse": { + "additionalProperties": false, + "description": "`fs/readFile` response.", + "properties": { + "data": { + "$ref": "#/components/schemas/ByteChunk" + }, + "fileSize": { + "description": "Size of the whole file when it was read.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "truncated": { + "description": "Bytes exist after `offset` plus the length of `data`.", + "type": "boolean" + }, + "version": { + "$ref": "#/components/schemas/FileVersion" + } + }, + "required": [ + "data", + "fileSize", + "version" + ], + "type": "object" + }, + "ReadJobsParams": { + "additionalProperties": false, + "description": "`job/read` params.", + "properties": { + "jobs": { + "description": "1 to `limits.job.maxRequestJobs` entries, each with its own position.", + "items": { + "$ref": "#/components/schemas/JobReadRequest" + }, + "type": "array" + }, + "maxBytes": { + "description": "Most `data` bytes across all entries, at most\n`limits.job.maxReadBytes`, filled in request order: an entry reached\nafter the budget is spent returns its summary and an empty page at\nits start position. 0 reads only summaries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "returnOn": { + "allOf": [ + { + "$ref": "#/components/schemas/ReturnOn" + } + ], + "description": "`anyOutput`: return when any entry has output past its position or\nchanged status; `exit`: when every entry is terminal; `window`: wait\nthe whole window." + }, + "waitMs": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobs", + "maxBytes", + "waitMs", + "returnOn" + ], + "type": "object" + }, + "ReadJobsResponse": { + "additionalProperties": false, + "description": "`job/read` response: one entry per requested job, in request order.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/components/schemas/JobReadEntry" + }, + "type": "array" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "ReadProcessParams": { + "additionalProperties": false, + "description": "`process/read` params.", + "properties": { + "ack": { + "description": "Acknowledge every byte before this offset first: the cursor moves to\nit (never backwards) whether or not this read's response arrives. A\ncaller that tracks positions reads with `after` and acknowledges\nwith `ack` for exactly-once delivery.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "after": { + "description": "Start offset. Absent: the daemon cursor, which this read advances to\n`next` once its response is written. Present: a non-consuming read\nthat leaves the cursor alone.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxBytes": { + "description": "Most `data` bytes to return, at most `limits.process.maxReadBytes`.\n0 reads only the state.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "processId": { + "$ref": "#/components/schemas/ProcessId" + }, + "returnOn": { + "$ref": "#/components/schemas/ReturnOn" + }, + "tailBytes": { + "description": "Start at `end - tailBytes` instead (clamped at 0). Non-consuming;\nexcludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "waitMs": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "processId", + "maxBytes", + "waitMs", + "returnOn" + ], + "type": "object" + }, + "ReadProcessResponse": { + "additionalProperties": false, + "description": "`process/read` response.", + "properties": { + "error": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProtocolError" + } + ], + "description": "Infrastructure failure after start (output capture or wait failed)." + }, + "exit": { + "anyOf": [ + { + "$ref": "#/components/schemas/ExitStatus" + } + ], + "description": "How the root ended; present once `state` is `exited`, unless `error`\nexplains why it could not be observed." + }, + "leftovers": { + "anyOf": [ + { + "$ref": "#/components/schemas/LeftoverReport" + } + ], + "description": "Group members alive after the root exited (`disposition: kept`),\nsampled when this response was built. Absent while running, when\nnone remain, and on platforms without process enumeration." + }, + "output": { + "$ref": "#/components/schemas/OutputPage" + }, + "pid": { + "description": "OS pid of the root process.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "state": { + "$ref": "#/components/schemas/ExecState" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/components/schemas/StoppedBy" + } + ], + "description": "Set when the daemon stopped the process; absent on a natural exit." + } + }, + "required": [ + "output", + "state", + "pid" + ], + "type": "object" + }, + "ReadTransferContentParams": { + "additionalProperties": false, + "description": "`transfer/read` params: read a captured file's content by digest.", + "properties": { + "digest": { + "$ref": "#/components/schemas/ContentDigest" + }, + "maxBytes": { + "description": "1 to `limits.filesystem.transfer.maxContentChunkBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "digest", + "offset", + "maxBytes" + ], + "type": "object" + }, + "ReadTransferContentResponse": { + "additionalProperties": false, + "description": "`transfer/read` response.", + "properties": { + "data": { + "$ref": "#/components/schemas/ByteChunk" + }, + "truncated": { + "description": "Bytes exist after `offset` plus the length of `data`, as for\n`fs/readFile`.", + "type": "boolean" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "ReadTransferInventoryParams": { + "additionalProperties": false, + "description": "`transfer/readInventory` params: page a sealed inventory by index.", + "properties": { + "index": { + "description": "Index of the first entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "limit": { + "description": "1 to `limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "index", + "limit" + ], + "type": "object" + }, + "ReadTransferInventoryResponse": { + "additionalProperties": false, + "description": "`transfer/readInventory` response.", + "properties": { + "entries": { + "items": { + "$ref": "#/components/schemas/InventoryEntry" + }, + "type": "array" + }, + "nextIndex": { + "description": "Index of the next page; absent after the last entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "RemoveParams": { + "additionalProperties": false, + "description": "`fs/remove` params.", + "properties": { + "force": { + "description": "Succeed when the path does not exist.", + "type": "boolean" + }, + "path": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "A symlink is removed, never followed." + }, + "recursive": { + "description": "Remove a directory and its contents.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "RemoveResponse": { + "additionalProperties": false, + "description": "`fs/remove` response.", + "type": "object" + }, + "RenameParams": { + "additionalProperties": false, + "description": "`fs/rename` params. Atomic within one filesystem; across filesystems it\nis `failedPrecondition` (`crossDevice`). The entry keeps its mode.", + "properties": { + "destination": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Rename only when the source's version equals this token." + }, + "overwrite": { + "description": "Replace an existing destination file; otherwise an existing\ndestination is `alreadyExists`. A directory destination is never\nreplaced.", + "type": "boolean" + }, + "source": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "source", + "destination" + ], + "type": "object" + }, + "RenameResponse": { + "additionalProperties": false, + "description": "`fs/rename` response.", + "type": "object" + }, + "ReturnOn": { + "description": "When a waiting read returns. Every mode also returns when `waitMs`\nelapses, when the handle reaches a settled exit, or at once when there\nis nothing left to wait for.", + "oneOf": [ + { + "const": "window", + "description": "Collect for the whole window; return early only when unread output\nreaches `maxBytes`.", + "type": "string" + }, + { + "const": "anyOutput", + "description": "Return as soon as any output past the start position exists.", + "type": "string" + }, + { + "const": "exit", + "description": "Return only at the settled exit (for `job/read`: once every entry is\nterminal), even if more than `maxBytes` of output accumulates.", + "type": "string" + } + ] + }, + "ScanCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/scan`: root and pattern counts, then its quotas.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxPatterns": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxRoots": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxRoots", + "maxPatterns", + "maxEntries", + "maxFileBytes", + "maxTotalBytes", + "maxDurationMs" + ], + "type": "object" + }, + "ScanEntry": { + "additionalProperties": false, + "description": "One selected entry below a root.", + "properties": { + "content": { + "$ref": "#/components/schemas/TreeContent" + }, + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "The file's bytes when `readContent` was set." + }, + "path": { + "allOf": [ + { + "$ref": "#/components/schemas/TreePath" + } + ], + "description": "Relative to the root as requested." + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + }, + "ScanFingerprint": { + "description": "Opaque fingerprint of one root's selection, prefixed `scan:v1:`, at most\n255 bytes without control characters.", + "maxLength": 255, + "minLength": 9, + "pattern": "^scan:v1:[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "ScanLimits": { + "additionalProperties": false, + "description": "Caller quotas of one scan.", + "properties": { + "maxDurationMs": { + "description": "Walk time per root; exhausting it fails that root\n(`deadlineExceeded`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "description": "Selected entries per root; more fail that root (`payloadTooLarge`\nwith `limitEntries`).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "description": "Larger selected files are skipped (`fileTooLarge`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "description": "Content bytes returned across all roots, in root order: a root whose\ncontent would exceed the rest fails (`payloadTooLarge` with\n`limitBytes`) and later roots are still scanned. Independently, a\nroot whose encoded result (entries, skips and base64 content) would\npush the scan's results past\n`MAX_RESULT_BYTES` fails the same\nway.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxEntries", + "maxFileBytes", + "maxTotalBytes", + "maxDurationMs" + ], + "type": "object" + }, + "ScanParams": { + "additionalProperties": false, + "description": "`fs/scan` params.", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeFilter" + } + ], + "description": "Applied below every root; at most `limits.filesystem.scan.maxPatterns`\npatterns in total." + }, + "followSymlinks": { + "description": "Follow symlinks whose targets stay inside the filesystem root; others\nare skipped (`symlinkOutsideRoot`). Without it symlinks are skipped\n(`symlink`).", + "type": "boolean" + }, + "includeDigests": { + "description": "Return each selected file's `digest`.", + "type": "boolean" + }, + "limits": { + "allOf": [ + { + "$ref": "#/components/schemas/ScanLimits" + } + ], + "description": "Caller quotas; each at most its ceiling." + }, + "readContent": { + "description": "Return the bytes of every selected file in `data`.", + "type": "boolean" + }, + "roots": { + "description": "1 to `limits.filesystem.scan.maxRoots` roots.", + "items": { + "$ref": "#/components/schemas/ScanRoot" + }, + "type": "array" + } + }, + "required": [ + "roots", + "limits" + ], + "type": "object" + }, + "ScanResponse": { + "additionalProperties": false, + "description": "`fs/scan` response: one result per requested root, in request order.", + "properties": { + "roots": { + "description": "Never empty.", + "items": { + "$ref": "#/components/schemas/ScanRootResult" + }, + "type": "array" + } + }, + "required": [ + "roots" + ], + "type": "object" + }, + "ScanRoot": { + "additionalProperties": false, + "description": "One root of a scan.", + "properties": { + "ifNoneMatch": { + "anyOf": [ + { + "$ref": "#/components/schemas/ScanFingerprint" + } + ], + "description": "The root's previous `fingerprint`: when the selection is unchanged\nthe result is `unchanged` and carries no entries." + }, + "maxDepth": { + "description": "Overrides `filter.maxDepth` for this root. `0` examines only the\nroot itself: a `complete` result without entries proves it is an\nexisting directory, which is how discovery checks a working\ndirectory in the same call.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "ScanRootResult": { + "description": "The outcome of one root.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Every selected entry, plus what was skipped and why.", + "properties": { + "canonicalPath": { + "anyOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "Resolved absolute path of the root when it differs from `path`\n(the root is a symlink)." + }, + "entries": { + "items": { + "$ref": "#/components/schemas/ScanEntry" + }, + "type": "array" + }, + "fingerprint": { + "$ref": "#/components/schemas/ScanFingerprint" + }, + "kind": { + "const": "complete", + "type": "string" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "skipped": { + "description": "The first\n`MAX_LISTED_SKIPS` skipped\nentries, in walk order.", + "items": { + "$ref": "#/components/schemas/ScanSkip" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Total number of skipped entries, listed or not; present whenever\nit is not zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "path", + "fingerprint" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The selection matches `ifNoneMatch`.", + "properties": { + "fingerprint": { + "$ref": "#/components/schemas/ScanFingerprint" + }, + "kind": { + "const": "unchanged", + "type": "string" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "kind", + "path", + "fingerprint" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The root does not exist.", + "properties": { + "kind": { + "const": "missing", + "type": "string" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "kind", + "path" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The root could not be scanned; no entries are returned for it.", + "properties": { + "error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "kind": { + "const": "failed", + "type": "string" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "kind", + "path", + "error" + ], + "type": "object" + } + ] + }, + "ScanSkip": { + "additionalProperties": false, + "description": "An entry a scan left out.", + "properties": { + "path": { + "anyOf": [ + { + "$ref": "#/components/schemas/TreePath" + } + ], + "description": "Absent when the entry's name is not UTF-8." + }, + "reason": { + "$ref": "#/components/schemas/ScanSkipReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "ScanSkipReason": { + "description": "Why a scan left an entry out.", + "oneOf": [ + { + "const": "fileTooLarge", + "description": "Larger than `maxFileBytes`.", + "type": "string" + }, + { + "const": "symlink", + "description": "A symlink, and `followSymlinks` is not set.", + "type": "string" + }, + { + "const": "symlinkOutsideRoot", + "description": "A symlink whose target leaves the filesystem root, or a link cycle.", + "type": "string" + }, + { + "const": "specialFile", + "description": "A device, FIFO, socket or another special file.", + "type": "string" + }, + { + "const": "nonUtf8Name", + "description": "The name is not UTF-8.", + "type": "string" + }, + { + "const": "unreadable", + "description": "The entry vanished or could not be read during the walk.", + "type": "string" + } + ] + }, + "SearchCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/searchText` quotas, field for field.", + "properties": { + "maxBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxLineBytes": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxBytes", + "maxDurationMs", + "maxLineBytes" + ], + "type": "object" + }, + "SearchTextLimits": { + "additionalProperties": false, + "description": "Caller quotas of one search; each at most its advertised ceiling.", + "properties": { + "maxBytes": { + "description": "Bytes actually read across files.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "description": "Files searched, not merely visited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxLineBytes": { + "description": "Longer matching lines are cut and marked `lineTruncated`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxBytes", + "maxDurationMs", + "maxLineBytes" + ], + "type": "object" + }, + "SearchTextMatch": { + "additionalProperties": false, + "description": "One matching line.", + "properties": { + "line": { + "description": "The matching line without its terminator, cut at `maxLineBytes` on a\ncharacter boundary.", + "type": "string" + }, + "lineNumber": { + "description": "1-based.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "lineTruncated": { + "type": "boolean" + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path", + "lineNumber", + "line" + ], + "type": "object" + }, + "SearchTextParams": { + "additionalProperties": false, + "description": "`fs/searchText` params: a bounded recursive regex search run by the\ndaemon. A file larger than the remaining byte budget is skipped and\ncounted; it never ends the walk.", + "properties": { + "filter": { + "$ref": "#/components/schemas/TreeFilter" + }, + "limits": { + "$ref": "#/components/schemas/SearchTextLimits" + }, + "pattern": { + "description": "Regular expression in the Rust `regex` crate's syntax, matched per\nline.", + "type": "string" + }, + "patternIgnoreCase": { + "description": "Match `pattern` case-insensitively. `filter.ignoreCase` governs the\npath patterns instead.", + "type": "boolean" + }, + "root": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "root", + "pattern", + "limits" + ], + "type": "object" + }, + "SearchTextResponse": { + "additionalProperties": false, + "description": "`fs/searchText` response.", + "properties": { + "elapsedMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "matches": { + "items": { + "$ref": "#/components/schemas/SearchTextMatch" + }, + "type": "array" + }, + "searchedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "searchedFileCount": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "skippedEntryCount": { + "description": "Entries the walk could not represent or list: names that are not\nUTF-8 and directories that could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "skippedFileCount": { + "description": "Files skipped because they exceeded the remaining byte budget, are\nbinary or could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "stopped": { + "anyOf": [ + { + "$ref": "#/components/schemas/SearchTextStop" + } + ], + "description": "Absent when the walk finished within its limits." + } + }, + "required": [ + "searchedFileCount", + "searchedBytes", + "elapsedMs" + ], + "type": "object" + }, + "SearchTextStop": { + "description": "Why a search stopped before finishing its walk.", + "oneOf": [ + { + "const": "matchLimit", + "description": "`maxMatches` matches were found.", + "type": "string" + }, + { + "const": "fileLimit", + "description": "`maxFiles` files were searched.", + "type": "string" + }, + { + "const": "byteLimit", + "description": "`maxBytes` bytes were read.", + "type": "string" + }, + { + "const": "timeLimit", + "description": "`maxDurationMs` ran out.", + "type": "string" + }, + { + "const": "responseLimit", + "description": "The next match would push the encoded matches past\n`MAX_RESULT_BYTES`.", + "type": "string" + } + ] + }, + "SecretString": { + "description": "A secret value carried on the wire, such as a `secretEnv` value, a\nregistration key or a stream token. It serializes as a plain string;\n`Debug` is redacted so structured logging never prints it.", + "type": "string" + }, + "SignalName": { + "description": "A POSIX signal name such as `SIGSEGV` or `SIGRTMIN+3`: `SIG` followed by\n1 to 16 uppercase letters, digits or `+`. Names, not numbers, because\nnumbers differ between platforms.", + "pattern": "^SIG[A-Z0-9+]{1,16}$", + "type": "string" + }, + "SpecialFilePolicy": { + "description": "Devices, FIFOs and sockets met during a capture.", + "oneOf": [ + { + "const": "reject", + "description": "Fail the operation (`notRegularFile`).", + "type": "string" + }, + { + "const": "skip", + "description": "Leave them out and report them in `skipped`.", + "type": "string" + } + ] + }, + "StartJobsParams": { + "additionalProperties": false, + "description": "`job/start` params. All-or-nothing: either every job is accepted (or\nreplayed) or none is.", + "properties": { + "jobs": { + "description": "1 to `limits.job.maxRequestJobs` jobs.", + "items": { + "$ref": "#/components/schemas/JobSpec" + }, + "type": "array" + }, + "requestId": { + "allOf": [ + { + "$ref": "#/components/schemas/JobRequestId" + } + ], + "description": "Idempotency key of this start. Retrying with the same request id and\nidentical jobs replays the existing jobs; reusing a job id with\nanother request id or spec is `failedPrecondition` (`jobIdInUse`)." + } + }, + "required": [ + "requestId", + "jobs" + ], + "type": "object" + }, + "StartJobsResponse": { + "additionalProperties": false, + "description": "`job/start` response: one summary per requested job, in request order.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/components/schemas/JobSummary" + }, + "type": "array" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "StartProcessParams": { + "additionalProperties": false, + "description": "`process/start` params.", + "properties": { + "command": { + "$ref": "#/components/schemas/CommandSpec" + }, + "processId": { + "allOf": [ + { + "$ref": "#/components/schemas/ProcessId" + } + ], + "description": "Caller-chosen handle, reserved before any side effect: a start whose\nid is already in use (running, retained or remembered by a tombstone)\nis `alreadyExists` and runs nothing." + }, + "pty": { + "anyOf": [ + { + "$ref": "#/components/schemas/PtySize" + } + ], + "description": "Run under a PTY of this size. The PTY master is the input: `input`\nmust then be `none` or `bytes`, and `process/write` is always\naccepted while the process runs. Requires\n`capabilities.process.pty`." + } + }, + "required": [ + "processId", + "command" + ], + "type": "object" + }, + "StartProcessResponse": { + "additionalProperties": false, + "description": "`process/start` response, sent once the process runs.", + "properties": { + "pid": { + "description": "OS pid of the root process, which is also its process group id.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "StoppedBy": { + "description": "Why the daemon stopped a process or job that did not exit on its own.\nSet only when the daemon signalled a root that was still running.", + "oneOf": [ + { + "const": "timeout", + "description": "`timeoutMs` expired: `SIGTERM`, the grace, then `SIGKILL`.", + "type": "string" + }, + { + "const": "terminate", + "description": "`process/terminate` with `terminate`, or `job/cancel` without\n`force`: `SIGTERM`, the grace, then `SIGKILL`.", + "type": "string" + }, + { + "const": "kill", + "description": "`process/terminate` with `kill`, or `job/cancel` with `force`:\n`SIGKILL` at once.", + "type": "string" + } + ] + }, + "SymlinkPolicy": { + "description": "Symlinks met during a capture.", + "oneOf": [ + { + "const": "expandInside", + "description": "Capture a link whose target stays inside the selection as the entry\nit points to; a link leaving the selection or a cycle fails the\noperation (`symlinkRejected`). A dangling link inside the selection\nis left out and reported in `skipped` as `symlink`.", + "type": "string" + }, + { + "const": "skip", + "description": "Leave every link out and report it in `skipped`.", + "type": "string" + }, + { + "const": "reject", + "description": "Fail on the first link.", + "type": "string" + } + ] + }, + "TerminateProcessParams": { + "additionalProperties": false, + "description": "`process/terminate` params.", + "properties": { + "processId": { + "$ref": "#/components/schemas/ProcessId" + }, + "signal": { + "$ref": "#/components/schemas/ProcessSignal" + } + }, + "required": [ + "processId", + "signal" + ], + "type": "object" + }, + "TerminateProcessResponse": { + "additionalProperties": false, + "description": "`process/terminate` response, sent once the signal was delivered (any\nescalation continues in the background).", + "properties": { + "running": { + "description": "True when the root process was still running when the signal was\nsent.", + "type": "boolean" + } + }, + "type": "object" + }, + "TransferCeilings": { + "additionalProperties": false, + "description": "Ceilings of `transfer/*`: capacity, page and chunk sizes, receipt\nretention, then the per-operation quotas.", + "properties": { + "maxActiveOperations": { + "description": "Operations not yet complete, aborted or failed, across connections;\na begin beyond it is `busy` (`transferCapacity`).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxContentChunkBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDepth": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxInventoryPageEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxManifestBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "receiptRetentionMs": { + "description": "How long a finished operation's receipt is answered.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxActiveOperations", + "maxContentChunkBytes", + "maxInventoryPageEntries", + "receiptRetentionMs", + "maxEntries", + "maxDepth", + "maxFileBytes", + "maxTotalBytes", + "maxManifestBytes", + "maxDurationMs" + ], + "type": "object" + }, + "TransferLimits": { + "additionalProperties": false, + "description": "Caller quotas of one operation; each at most its advertised ceiling.", + "properties": { + "maxDepth": { + "description": "The selected root has depth 0.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "description": "From begin to commit.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "description": "Entries, the selected root included.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxManifestBytes": { + "description": "Encoded inventory size.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxEntries", + "maxDepth", + "maxFileBytes", + "maxTotalBytes", + "maxManifestBytes", + "maxDurationMs" + ], + "type": "object" + }, + "TransferOnExisting": { + "description": "What a materialize does with an existing destination.", + "oneOf": [ + { + "const": "error", + "description": "Refuse at begin (`alreadyExists`).", + "type": "string" + }, + { + "const": "replace", + "description": "Replace it atomically at commit.", + "type": "string" + } + ] + }, + "TransferOperationId": { + "description": "`operationId` of one transfer operation.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "TransferOperationParams": { + "additionalProperties": false, + "description": "Params of `transfer/advance`, `transfer/getStatus`, `transfer/commit`\nand `transfer/abort`: each names only the operation.", + "properties": { + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId" + ], + "type": "object" + }, + "TransferPhase": { + "description": "Where an operation stands; see the module docs for the order per\nselection kind.", + "oneOf": [ + { + "enum": [ + "complete", + "aborted" + ], + "type": "string" + }, + { + "const": "scanning", + "description": "The daemon walks a tree: the capture source, or the destination of a\nmaterialize. Advance until the phase changes.", + "type": "string" + }, + { + "const": "inventory", + "description": "Capture: the inventory is sealed and pageable. Materialize: waiting\nfor appended entries.", + "type": "string" + }, + { + "const": "content", + "description": "Waiting for the content of `transfer/listMissing` digests.", + "type": "string" + }, + { + "const": "staging", + "description": "The daemon assembles the staged tree. Advance until `ready`.", + "type": "string" + }, + { + "const": "ready", + "description": "Ready for `transfer/commit`.", + "type": "string" + }, + { + "const": "failed", + "description": "Stopped by an error; see `error`.", + "type": "string" + } + ] + }, + "TransferSelection": { + "description": "What one operation transfers.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read a file or tree out of the environment.", + "properties": { + "filter": { + "$ref": "#/components/schemas/TreeFilter" + }, + "kind": { + "const": "capture", + "type": "string" + }, + "source": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "specialFiles": { + "$ref": "#/components/schemas/SpecialFilePolicy" + }, + "symlinks": { + "$ref": "#/components/schemas/SymlinkPolicy" + } + }, + "required": [ + "kind", + "source" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Write a tree into the environment, replacing `destination` atomically\nat commit.", + "properties": { + "destination": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "kind": { + "const": "materialize", + "type": "string" + }, + "onExisting": { + "$ref": "#/components/schemas/TransferOnExisting" + } + }, + "required": [ + "kind", + "destination" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Write exactly one file with `fs/writeFile` semantics: staged,\natomic, an existing target keeps its permissions, an in-root symlink\nat `destination` is followed.", + "properties": { + "create": { + "$ref": "#/components/schemas/WriteCondition" + }, + "destination": { + "$ref": "#/components/schemas/EnvironmentPath" + }, + "executable": { + "type": "boolean" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ] + }, + "kind": { + "const": "writeFile", + "type": "string" + } + }, + "required": [ + "kind", + "destination" + ], + "type": "object" + } + ] + }, + "TransferSkip": { + "additionalProperties": false, + "description": "An entry a capture left out.", + "properties": { + "path": { + "anyOf": [ + { + "$ref": "#/components/schemas/TreePath" + } + ], + "description": "Absent when the entry's name is not UTF-8." + }, + "reason": { + "$ref": "#/components/schemas/TransferSkipReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "TransferSkipReason": { + "description": "Why a capture left an entry out.", + "oneOf": [ + { + "const": "symlink", + "description": "`symlinks: skip`, or a dangling link under `symlinks: expandInside`.", + "type": "string" + }, + { + "const": "specialFile", + "description": "`specialFiles: skip`.", + "type": "string" + }, + { + "const": "nonUtf8Name", + "description": "The name is not UTF-8; such entries are always skipped.", + "type": "string" + } + ] + }, + "TransferStatus": { + "additionalProperties": false, + "description": "The operation's status: the response of `transfer/begin`,\n`transfer/advance`, `transfer/getStatus`, `transfer/append`,\n`transfer/commit` and `transfer/abort`.", + "properties": { + "deadlineAtMs": { + "description": "When `maxDurationMs` runs out for an active operation.", + "format": "int64", + "type": "integer" + }, + "entryCount": { + "description": "Entries in the inventory so far.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/components/schemas/ProtocolError" + } + ], + "description": "Why the operation failed; present exactly when `phase` is `failed`." + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + }, + "phase": { + "$ref": "#/components/schemas/TransferPhase" + }, + "receiptExpiresAtMs": { + "description": "When the receipt of a finished (complete, aborted or failed)\noperation stops being answered; afterwards calls are `notFound`\n(`expired`).", + "format": "int64", + "type": "integer" + }, + "reusedBytes": { + "description": "Content bytes satisfied from files already at the destination.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "skipped": { + "description": "The first\n`MAX_LISTED_SKIPS` entries a\ncapture left out under its policies.", + "items": { + "$ref": "#/components/schemas/TransferSkip" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Total number of entries left out, listed or not; present whenever it\nis not zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "totalBytes": { + "description": "File bytes in the inventory so far.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "transferredBytes": { + "description": "Content bytes moved over the wire.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "operationId", + "phase", + "entryCount", + "totalBytes", + "transferredBytes", + "reusedBytes" + ], + "type": "object" + }, + "TreeContent": { + "description": "What a tree entry is: a scan entry or a transfer inventory entry.\nDigests identify raw file bytes, not paths or modes.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "directory", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "digest": { + "anyOf": [ + { + "$ref": "#/components/schemas/ContentDigest" + } + ], + "description": "Scans: present when `includeDigests` was set. Transfer\ninventories: always present; an appended file entry without it\nis `invalidParams`." + }, + "executable": { + "type": "boolean" + }, + "kind": { + "const": "file", + "type": "string" + }, + "sizeBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "sizeBytes" + ], + "type": "object" + } + ] + }, + "TreeFilter": { + "additionalProperties": false, + "description": "Which entries a walk visits. The default visits everything except\nhidden and ignored entries.", + "properties": { + "exclude": { + "description": "Drop entries matching any pattern; a matching directory is pruned.", + "items": { + "type": "string" + }, + "type": "array" + }, + "ignoreCase": { + "description": "Match patterns case-insensitively.", + "type": "boolean" + }, + "include": { + "description": "Select entries matching any pattern; empty selects every entry.", + "items": { + "type": "string" + }, + "type": "array" + }, + "includeHidden": { + "description": "Visit entries whose name starts with `.`.", + "type": "boolean" + }, + "includeIgnored": { + "description": "Visit entries that ignore files exclude.", + "type": "boolean" + }, + "maxDepth": { + "description": "Deepest entry depth visited: the root's direct children are depth\n1, so `1` selects direct children only. Unbounded when absent.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "TreePath": { + "description": "A relative, normalized path below a selected root: segments joined by\n`/`, without empty, `.` or `..` segments. The single segment `.` names\nthe root itself. At most 4096 bytes. Used for scan, inventory and\ntransfer entries.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(\\.|(?!\\.\\.?(/|$))[^/\\u0000]+(/(?!\\.\\.?(/|$))[^/\\u0000]+)*)$", + "type": "string" + }, + "WriteCondition": { + "description": "When `fs/writeFile` may write.", + "oneOf": [ + { + "const": "always", + "description": "Create or replace.", + "type": "string" + }, + { + "const": "ifAbsent", + "description": "Only create; an existing entry is `failedPrecondition`\n(`createCondition`).", + "type": "string" + }, + { + "const": "ifPresent", + "description": "Only replace; a missing file is `failedPrecondition`\n(`createCondition`).", + "type": "string" + } + ] + }, + "WriteFileParams": { + "additionalProperties": false, + "description": "`fs/writeFile` params. The write is staged in the target's directory and\npublished by an atomic rename: readers see the old or the new content,\nnever a mix. An existing target keeps its permissions. A symlink at\n`path` whose target stays inside the root is followed like any path: the\nfinal target is replaced and the link itself is kept (a dangling link\ncreates its target, subject to `create`); a link that leaves the root is\n`permissionDenied` (`outsideRoot`). A special file is\n`failedPrecondition` (`notRegularFile`), a directory\n`failedPrecondition` (`isDirectory`). The parent of the final target\nmust exist.", + "properties": { + "create": { + "$ref": "#/components/schemas/WriteCondition" + }, + "data": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "At most `limits.filesystem.maxWriteFileBytes`; larger files go\nthrough `transfer/*` with a `writeFile` selection." + }, + "executable": { + "description": "Set the execute bits (the read bits' pattern). Without it a new file\ngets the default mode and an existing file keeps its mode.", + "type": "boolean" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Write only when the current file's version equals this token;\notherwise `failedPrecondition` (`ifMatchFailed`)." + }, + "path": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + "required": [ + "path", + "data" + ], + "type": "object" + }, + "WriteFileResponse": { + "additionalProperties": false, + "description": "`fs/writeFile` response.", + "properties": { + "version": { + "allOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Version of the written file." + } + }, + "required": [ + "version" + ], + "type": "object" + }, + "WriteProcessParams": { + "additionalProperties": false, + "description": "`process/write` params.", + "properties": { + "closeInput": { + "description": "Close the input after `data`: end of file on a pipe, `VEOF` on a\nPTY. Applied only when every byte of `data` was accepted.", + "type": "boolean" + }, + "data": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "Bytes for the process's input pipe or PTY." + }, + "processId": { + "$ref": "#/components/schemas/ProcessId" + } + }, + "required": [ + "processId" + ], + "type": "object" + }, + "WriteProcessResponse": { + "additionalProperties": false, + "description": "`process/write` response.", + "properties": { + "acceptedBytes": { + "description": "Leading bytes of `data` queued to the process's writer. Fewer than\nsent (possibly 0) means the input queue is full: resend the rest\nlater.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "acceptedBytes" + ], + "type": "object" + }, + "WriteTransferContentParams": { + "additionalProperties": false, + "description": "`transfer/write` params: upload content for a missing digest. Chunks for\none digest arrive in order; an identical chunk may be retried.\nDifferent digests may be uploaded concurrently.", + "properties": { + "data": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "At most `limits.filesystem.transfer.maxContentChunkBytes`." + }, + "digest": { + "$ref": "#/components/schemas/ContentDigest" + }, + "offset": { + "description": "Must equal the digest's next expected offset.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + "required": [ + "operationId", + "digest", + "offset", + "data" + ], + "type": "object" + }, + "WriteTransferContentResponse": { + "additionalProperties": false, + "description": "`transfer/write` response.", + "properties": { + "nextOffset": { + "description": "The digest's next expected offset; equal to its size once complete\nand verified.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "nextOffset" + ], + "type": "object" + } + } + }, + "info": { + "description": "JSON-RPC 2.0 over WebSocket text frames of at most 16777216 bytes. Exact-match protocolVersion 4. Errors carry code, message and data.kind; see x-lightspeed-error-kinds.", + "title": "Lightspeed environment data plane", + "version": "4" + }, + "methods": [ + { + "name": "initialize", + "paramStructure": "by-name", + "params": [ + { + "name": "clientInfo", + "required": true, + "schema": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + { + "name": "protocolVersion", + "required": true, + "schema": { + "description": "The client's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/InitializeResponse" + } + }, + "summary": "Check the protocol version and read the daemon's instance id, capabilities and ceilings.", + "x-lightspeed-activity": "observation", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "handshake", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "InitializeParams", + "x-lightspeed-result-type": "InitializeResponse" + }, + { + "name": "env/getIdle", + "paramStructure": "by-name", + "params": [], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/GetIdleResponse" + } + }, + "summary": "Report how long the environment has been idle.", + "x-lightspeed-activity": "observation", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "env", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "GetIdleParams", + "x-lightspeed-result-type": "GetIdleResponse" + }, + { + "name": "fs/readFile", + "paramStructure": "by-name", + "params": [ + { + "name": "maxBytes", + "required": true, + "schema": { + "description": "Most bytes to return, at most `limits.filesystem.maxReadFileBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "offset", + "required": false, + "schema": { + "description": "First byte to return; 0 by default.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadFileResponse" + } + }, + "summary": "Read a byte range of a regular file.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadFileParams", + "x-lightspeed-result-type": "ReadFileResponse" + }, + { + "name": "fs/writeFile", + "paramStructure": "by-name", + "params": [ + { + "name": "create", + "required": false, + "schema": { + "$ref": "#/components/schemas/WriteCondition" + } + }, + { + "name": "data", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "At most `limits.filesystem.maxWriteFileBytes`; larger files go\nthrough `transfer/*` with a `writeFile` selection." + } + }, + { + "name": "executable", + "required": false, + "schema": { + "description": "Set the execute bits (the read bits' pattern). Without it a new file\ngets the default mode and an existing file keeps its mode.", + "type": "boolean" + } + }, + { + "name": "ifMatch", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Write only when the current file's version equals this token;\notherwise `failedPrecondition` (`ifMatchFailed`)." + } + }, + { + "name": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/WriteFileResponse" + } + }, + "summary": "Write a file atomically, optionally conditioned on its version.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystemWrite", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "WriteFileParams", + "x-lightspeed-result-type": "WriteFileResponse" + }, + { + "name": "fs/createDirectory", + "paramStructure": "by-name", + "params": [ + { + "name": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + { + "name": "recursive", + "required": false, + "schema": { + "description": "Create missing parents, and succeed when the directory exists.", + "type": "boolean" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/CreateDirectoryResponse" + } + }, + "summary": "Create a directory.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystemWrite", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "CreateDirectoryParams", + "x-lightspeed-result-type": "CreateDirectoryResponse" + }, + { + "name": "fs/getMetadata", + "paramStructure": "by-name", + "params": [ + { + "name": "followSymlinks", + "required": false, + "schema": { + "description": "Describe the symlink's target instead of the link (subject to\nconfinement).", + "type": "boolean" + } + }, + { + "name": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/GetMetadataResponse" + } + }, + "summary": "Describe one entry.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "GetMetadataParams", + "x-lightspeed-result-type": "GetMetadataResponse" + }, + { + "name": "fs/readDirectory", + "paramStructure": "by-name", + "params": [ + { + "name": "cursor", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/DirectoryCursor" + } + ], + "description": "`nextCursor` of the previous page; a cursor this daemon did not mint\nis `invalidParams`." + } + }, + { + "name": "limit", + "required": true, + "schema": { + "description": "Most entries per page, 1 to `limits.filesystem.maxDirectoryEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "path", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "A directory, or a symlink to one inside the root." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadDirectoryResponse" + } + }, + "summary": "List one page of a directory's entries.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadDirectoryParams", + "x-lightspeed-result-type": "ReadDirectoryResponse" + }, + { + "name": "fs/remove", + "paramStructure": "by-name", + "params": [ + { + "name": "force", + "required": false, + "schema": { + "description": "Succeed when the path does not exist.", + "type": "boolean" + } + }, + { + "name": "path", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/EnvironmentPath" + } + ], + "description": "A symlink is removed, never followed." + } + }, + { + "name": "recursive", + "required": false, + "schema": { + "description": "Remove a directory and its contents.", + "type": "boolean" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/RemoveResponse" + } + }, + "summary": "Remove a file, symlink or directory.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystemWrite", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "RemoveParams", + "x-lightspeed-result-type": "RemoveResponse" + }, + { + "name": "fs/rename", + "paramStructure": "by-name", + "params": [ + { + "name": "destination", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + }, + { + "name": "ifMatch", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/FileVersion" + } + ], + "description": "Rename only when the source's version equals this token." + } + }, + { + "name": "overwrite", + "required": false, + "schema": { + "description": "Replace an existing destination file; otherwise an existing\ndestination is `alreadyExists`. A directory destination is never\nreplaced.", + "type": "boolean" + } + }, + { + "name": "source", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/RenameResponse" + } + }, + "summary": "Rename an entry atomically.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystemWrite", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "RenameParams", + "x-lightspeed-result-type": "RenameResponse" + }, + { + "name": "fs/searchText", + "paramStructure": "by-name", + "params": [ + { + "name": "filter", + "required": false, + "schema": { + "$ref": "#/components/schemas/TreeFilter" + } + }, + { + "name": "limits", + "required": true, + "schema": { + "$ref": "#/components/schemas/SearchTextLimits" + } + }, + { + "name": "pattern", + "required": true, + "schema": { + "description": "Regular expression in the Rust `regex` crate's syntax, matched per\nline.", + "type": "string" + } + }, + { + "name": "patternIgnoreCase", + "required": false, + "schema": { + "description": "Match `pattern` case-insensitively. `filter.ignoreCase` governs the\npath patterns instead.", + "type": "boolean" + } + }, + { + "name": "root", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/SearchTextResponse" + } + }, + "summary": "Search file contents below a root with a regular expression.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "SearchTextParams", + "x-lightspeed-result-type": "SearchTextResponse" + }, + { + "name": "fs/globFiles", + "paramStructure": "by-name", + "params": [ + { + "name": "filter", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeFilter" + } + ], + "description": "`include` must not be empty." + } + }, + { + "name": "limits", + "required": true, + "schema": { + "$ref": "#/components/schemas/GlobFilesLimits" + } + }, + { + "name": "root", + "required": true, + "schema": { + "$ref": "#/components/schemas/EnvironmentPath" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/GlobFilesResponse" + } + }, + "summary": "List files below a root that match patterns.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "GlobFilesParams", + "x-lightspeed-result-type": "GlobFilesResponse" + }, + { + "name": "fs/scan", + "paramStructure": "by-name", + "params": [ + { + "name": "filter", + "required": false, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/TreeFilter" + } + ], + "description": "Applied below every root; at most `limits.filesystem.scan.maxPatterns`\npatterns in total." + } + }, + { + "name": "followSymlinks", + "required": false, + "schema": { + "description": "Follow symlinks whose targets stay inside the filesystem root; others\nare skipped (`symlinkOutsideRoot`). Without it symlinks are skipped\n(`symlink`).", + "type": "boolean" + } + }, + { + "name": "includeDigests", + "required": false, + "schema": { + "description": "Return each selected file's `digest`.", + "type": "boolean" + } + }, + { + "name": "limits", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ScanLimits" + } + ], + "description": "Caller quotas; each at most its ceiling." + } + }, + { + "name": "readContent", + "required": false, + "schema": { + "description": "Return the bytes of every selected file in `data`.", + "type": "boolean" + } + }, + { + "name": "roots", + "required": true, + "schema": { + "description": "1 to `limits.filesystem.scan.maxRoots` roots.", + "items": { + "$ref": "#/components/schemas/ScanRoot" + }, + "type": "array" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ScanResponse" + } + }, + "summary": "Observe small trees below several roots, each reported on its own.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "observation", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "filesystem", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ScanParams", + "x-lightspeed-result-type": "ScanResponse" + }, + { + "name": "transfer/begin", + "paramStructure": "by-name", + "params": [ + { + "name": "limits", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferLimits" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + }, + { + "name": "selection", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferSelection" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Start or resume a transfer operation.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "BeginTransferParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/advance", + "paramStructure": "by-name", + "params": [ + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Do a bounded step of scanning or staging work.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TransferOperationParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/getStatus", + "paramStructure": "by-name", + "params": [ + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Read an operation's status or receipt.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TransferOperationParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/readInventory", + "paramStructure": "by-name", + "params": [ + { + "name": "index", + "required": true, + "schema": { + "description": "Index of the first entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "limit", + "required": true, + "schema": { + "description": "1 to `limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadTransferInventoryResponse" + } + }, + "summary": "Read one page of a sealed inventory.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadTransferInventoryParams", + "x-lightspeed-result-type": "ReadTransferInventoryResponse" + }, + { + "name": "transfer/append", + "paramStructure": "by-name", + "params": [ + { + "name": "entries", + "required": true, + "schema": { + "description": "At most `limits.filesystem.transfer.maxInventoryPageEntries`.", + "items": { + "$ref": "#/components/schemas/InventoryEntry" + }, + "type": "array" + } + }, + { + "name": "index", + "required": true, + "schema": { + "description": "Index of the page's first entry: the number of entries appended\nbefore it.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "last", + "required": false, + "schema": { + "description": "This page completes the inventory.", + "type": "boolean" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Append incoming inventory entries.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "AppendTransferInventoryParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/listMissing", + "paramStructure": "by-name", + "params": [ + { + "name": "limit", + "required": true, + "schema": { + "description": "Most digests to return, 1 to\n`limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ListTransferMissingResponse" + } + }, + "summary": "List digests whose content the daemon still needs.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ListTransferMissingParams", + "x-lightspeed-result-type": "ListTransferMissingResponse" + }, + { + "name": "transfer/read", + "paramStructure": "by-name", + "params": [ + { + "name": "digest", + "required": true, + "schema": { + "$ref": "#/components/schemas/ContentDigest" + } + }, + { + "name": "maxBytes", + "required": true, + "schema": { + "description": "1 to `limits.filesystem.transfer.maxContentChunkBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "offset", + "required": true, + "schema": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadTransferContentResponse" + } + }, + "summary": "Read captured content by digest.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadTransferContentParams", + "x-lightspeed-result-type": "ReadTransferContentResponse" + }, + { + "name": "transfer/write", + "paramStructure": "by-name", + "params": [ + { + "name": "data", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "At most `limits.filesystem.transfer.maxContentChunkBytes`." + } + }, + { + "name": "digest", + "required": true, + "schema": { + "$ref": "#/components/schemas/ContentDigest" + } + }, + { + "name": "offset", + "required": true, + "schema": { + "description": "Must equal the digest's next expected offset.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/WriteTransferContentResponse" + } + }, + "summary": "Upload content for a missing digest.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "WriteTransferContentParams", + "x-lightspeed-result-type": "WriteTransferContentResponse" + }, + { + "name": "transfer/commit", + "paramStructure": "by-name", + "params": [ + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Finish a ready operation: publish a staged tree, or release a capture.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TransferOperationParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "transfer/abort", + "paramStructure": "by-name", + "params": [ + { + "name": "operationId", + "required": true, + "schema": { + "$ref": "#/components/schemas/TransferOperationId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TransferStatus" + } + }, + "summary": "Abandon an operation and discard its staging.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "filesystem", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "transfer", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TransferOperationParams", + "x-lightspeed-result-type": "TransferStatus" + }, + { + "name": "process/start", + "paramStructure": "by-name", + "params": [ + { + "name": "command", + "required": true, + "schema": { + "$ref": "#/components/schemas/CommandSpec" + } + }, + { + "name": "processId", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ProcessId" + } + ], + "description": "Caller-chosen handle, reserved before any side effect: a start whose\nid is already in use (running, retained or remembered by a tombstone)\nis `alreadyExists` and runs nothing." + } + }, + { + "name": "pty", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/PtySize" + } + ], + "description": "Run under a PTY of this size. The PTY master is the input: `input`\nmust then be `none` or `bytes`, and `process/write` is always\naccepted while the process runs. Requires\n`capabilities.process.pty`." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/StartProcessResponse" + } + }, + "summary": "Start a process under a caller-chosen handle.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "process", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "process", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "StartProcessParams", + "x-lightspeed-result-type": "StartProcessResponse" + }, + { + "name": "process/read", + "paramStructure": "by-name", + "params": [ + { + "name": "ack", + "required": false, + "schema": { + "description": "Acknowledge every byte before this offset first: the cursor moves to\nit (never backwards) whether or not this read's response arrives. A\ncaller that tracks positions reads with `after` and acknowledges\nwith `ack` for exactly-once delivery.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "after", + "required": false, + "schema": { + "description": "Start offset. Absent: the daemon cursor, which this read advances to\n`next` once its response is written. Present: a non-consuming read\nthat leaves the cursor alone.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "maxBytes", + "required": true, + "schema": { + "description": "Most `data` bytes to return, at most `limits.process.maxReadBytes`.\n0 reads only the state.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "processId", + "required": true, + "schema": { + "$ref": "#/components/schemas/ProcessId" + } + }, + { + "name": "returnOn", + "required": true, + "schema": { + "$ref": "#/components/schemas/ReturnOn" + } + }, + { + "name": "tailBytes", + "required": false, + "schema": { + "description": "Start at `end - tailBytes` instead (clamped at 0). Non-consuming;\nexcludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "waitMs", + "required": true, + "schema": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadProcessResponse" + } + }, + "summary": "Wait for and read a page of a process's output and its state.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "process", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "process", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadProcessParams", + "x-lightspeed-result-type": "ReadProcessResponse" + }, + { + "name": "process/write", + "paramStructure": "by-name", + "params": [ + { + "name": "closeInput", + "required": false, + "schema": { + "description": "Close the input after `data`: end of file on a pipe, `VEOF` on a\nPTY. Applied only when every byte of `data` was accepted.", + "type": "boolean" + } + }, + { + "name": "data", + "required": false, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ByteChunk" + } + ], + "description": "Bytes for the process's input pipe or PTY." + } + }, + { + "name": "processId", + "required": true, + "schema": { + "$ref": "#/components/schemas/ProcessId" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/WriteProcessResponse" + } + }, + "summary": "Queue bytes for a process's input pipe or PTY.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "process", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "process", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "WriteProcessParams", + "x-lightspeed-result-type": "WriteProcessResponse" + }, + { + "name": "process/terminate", + "paramStructure": "by-name", + "params": [ + { + "name": "processId", + "required": true, + "schema": { + "$ref": "#/components/schemas/ProcessId" + } + }, + { + "name": "signal", + "required": true, + "schema": { + "$ref": "#/components/schemas/ProcessSignal" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/TerminateProcessResponse" + } + }, + "summary": "Interrupt, terminate or kill a process's group.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "process", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "process", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "TerminateProcessParams", + "x-lightspeed-result-type": "TerminateProcessResponse" + }, + { + "name": "job/start", + "paramStructure": "by-name", + "params": [ + { + "name": "jobs", + "required": true, + "schema": { + "description": "1 to `limits.job.maxRequestJobs` jobs.", + "items": { + "$ref": "#/components/schemas/JobSpec" + }, + "type": "array" + } + }, + { + "name": "requestId", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/JobRequestId" + } + ], + "description": "Idempotency key of this start. Retrying with the same request id and\nidentical jobs replays the existing jobs; reusing a job id with\nanother request id or spec is `failedPrecondition` (`jobIdInUse`)." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/StartJobsResponse" + } + }, + "summary": "Start jobs, or replay an identical earlier start.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "job", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "job", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "StartJobsParams", + "x-lightspeed-result-type": "StartJobsResponse" + }, + { + "name": "job/read", + "paramStructure": "by-name", + "params": [ + { + "name": "jobs", + "required": true, + "schema": { + "description": "1 to `limits.job.maxRequestJobs` entries, each with its own position.", + "items": { + "$ref": "#/components/schemas/JobReadRequest" + }, + "type": "array" + } + }, + { + "name": "maxBytes", + "required": true, + "schema": { + "description": "Most `data` bytes across all entries, at most\n`limits.job.maxReadBytes`, filled in request order: an entry reached\nafter the budget is spent returns its summary and an empty page at\nits start position. 0 reads only summaries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "returnOn", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/ReturnOn" + } + ], + "description": "`anyOutput`: return when any entry has output past its position or\nchanged status; `exit`: when every entry is terminal; `window`: wait\nthe whole window." + } + }, + { + "name": "waitMs", + "required": true, + "schema": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/ReadJobsResponse" + } + }, + "summary": "Wait for and read jobs' summaries and output pages.", + "x-lightspeed-abortable": true, + "x-lightspeed-activity": "observation", + "x-lightspeed-capability": "job", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "job", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "ReadJobsParams", + "x-lightspeed-result-type": "ReadJobsResponse" + }, + { + "name": "job/cancel", + "paramStructure": "by-name", + "params": [ + { + "name": "force", + "required": false, + "schema": { + "description": "Kill at once instead of `SIGTERM`, `limits.job.cancelGraceMs`, then\n`SIGKILL`.", + "type": "boolean" + } + }, + { + "name": "jobIds", + "required": true, + "schema": { + "description": "1 to `limits.job.maxRequestJobs` ids.", + "items": { + "$ref": "#/components/schemas/JobId" + }, + "type": "array" + } + }, + { + "name": "scope", + "required": false, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/JobCancelScope" + } + ], + "description": "`job` by default." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/CancelJobsResponse" + } + }, + "summary": "Cancel jobs and optionally their dependents.", + "x-lightspeed-activity": "use", + "x-lightspeed-capability": "job", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "job", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "CancelJobsParams", + "x-lightspeed-result-type": "CancelJobsResponse" + }, + { + "name": "$/cancelRequest", + "paramStructure": "by-name", + "params": [ + { + "name": "id", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/Id" + } + ], + "description": "Id of the request to abort." + } + } + ], + "summary": "Abort a pending abortable request.", + "x-lightspeed-activity": "observation", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "envelope", + "x-lightspeed-kind": "notification", + "x-lightspeed-params-type": "CancelRequestParams" + } + ], + "openrpc": "1.3.2", + "x-lightspeed-error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "x-lightspeed-error-kinds": [ + { + "code": -32700, + "kind": "parseError", + "retryable": false + }, + { + "code": -32600, + "kind": "invalidRequest", + "retryable": false + }, + { + "code": -32601, + "kind": "methodNotFound", + "retryable": false + }, + { + "code": -32602, + "kind": "invalidParams", + "retryable": false + }, + { + "code": -32002, + "kind": "notInitialized", + "retryable": false + }, + { + "code": -32005, + "kind": "versionMismatch", + "retryable": false + }, + { + "code": -32004, + "kind": "notFound", + "retryable": false + }, + { + "code": -32009, + "kind": "alreadyExists", + "retryable": false + }, + { + "code": -32007, + "kind": "failedPrecondition", + "retryable": false + }, + { + "code": -32003, + "kind": "permissionDenied", + "retryable": false + }, + { + "code": -32006, + "kind": "capabilityUnavailable", + "retryable": false + }, + { + "code": -32008, + "kind": "busy", + "retryable": true + }, + { + "code": -32015, + "kind": "deadlineExceeded", + "retryable": false + }, + { + "code": -32016, + "kind": "expired", + "retryable": false + }, + { + "code": -32017, + "kind": "aborted", + "retryable": true + }, + { + "code": -32013, + "kind": "payloadTooLarge", + "retryable": false + }, + { + "code": -32018, + "kind": "processFailed", + "retryable": false + }, + { + "code": -32019, + "kind": "unavailable", + "retryable": true + }, + { + "code": -32603, + "kind": "internal", + "retryable": false + } + ], + "x-lightspeed-plane": "data" +} diff --git a/crates/environment-protocol/contract/data.schema.json b/crates/environment-protocol/contract/data.schema.json new file mode 100644 index 000000000..e6475ab20 --- /dev/null +++ b/crates/environment-protocol/contract/data.schema.json @@ -0,0 +1,4280 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "AppendTransferInventoryParams": { + "additionalProperties": false, + "description": "`transfer/append` params: add incoming inventory entries in order.\nRetrying an identical page is safe.", + "properties": { + "entries": { + "description": "At most `limits.filesystem.transfer.maxInventoryPageEntries`.", + "items": { + "$ref": "#/definitions/InventoryEntry" + }, + "type": "array" + }, + "index": { + "description": "Index of the page's first entry: the number of entries appended\nbefore it.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "last": { + "description": "This page completes the inventory.", + "type": "boolean" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "index", + "entries" + ], + "type": "object" + }, + "BeginTransferParams": { + "additionalProperties": false, + "description": "`transfer/begin` params. Idempotent: a repeat with identical input\nreturns the operation's status; different input under the same id is\n`failedPrecondition` (`operationIdInUse`).", + "properties": { + "limits": { + "$ref": "#/definitions/TransferLimits" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + }, + "selection": { + "$ref": "#/definitions/TransferSelection" + } + }, + "required": [ + "operationId", + "selection", + "limits" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "ByteChunk": { + "contentEncoding": "base64", + "description": "Payload bytes, always named `data` on the wire, as padded standard\nbase64.", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$", + "type": "string" + }, + "CancelJobsParams": { + "additionalProperties": false, + "description": "`job/cancel` params.", + "properties": { + "force": { + "description": "Kill at once instead of `SIGTERM`, `limits.job.cancelGraceMs`, then\n`SIGKILL`.", + "type": "boolean" + }, + "jobIds": { + "description": "1 to `limits.job.maxRequestJobs` ids.", + "items": { + "$ref": "#/definitions/JobId" + }, + "type": "array" + }, + "scope": { + "allOf": [ + { + "$ref": "#/definitions/JobCancelScope" + } + ], + "description": "`job` by default." + } + }, + "required": [ + "jobIds" + ], + "type": "object" + }, + "CancelJobsResponse": { + "additionalProperties": false, + "description": "`job/cancel` response: one entry per affected job, named jobs first in\nrequest order, then dependents, at most `limits.job.maxRequestJobs`\nentries in total.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/definitions/JobSummaryEntry" + }, + "type": "array" + }, + "unlistedJobCount": { + "description": "Dependents cancelled as well but not listed, beyond the entry limit.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "CancelRequestParams": { + "additionalProperties": false, + "description": "`$/cancelRequest` params.", + "properties": { + "id": { + "allOf": [ + { + "$ref": "#/definitions/Id" + } + ], + "description": "Id of the request to abort." + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "power", + "description": "The controller's `power` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "CommandSpec": { + "additionalProperties": false, + "description": "What to run. Shared by `process/start` and `job/start`.", + "properties": { + "argv": { + "description": "Program and arguments; `argv[0]` is resolved through `PATH`. Not\nempty.", + "items": { + "type": "string" + }, + "type": "array" + }, + "cwd": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "Working directory; the daemon's `defaultCwd` when absent." + }, + "env": { + "additionalProperties": { + "type": "string" + }, + "description": "Variables added to the daemon's scrubbed environment.", + "type": "object" + }, + "input": { + "allOf": [ + { + "$ref": "#/definitions/ExecInput" + } + ], + "description": "Standard input; none by default." + }, + "secretEnv": { + "additionalProperties": { + "$ref": "#/definitions/SecretString" + }, + "description": "Like `env`, but every value is redacted from the command's output.", + "type": "object" + }, + "timeoutMs": { + "description": "Stop deadline from start. On expiry the daemon terminates the process\ngroup (`SIGTERM`, the advertised grace, then `SIGKILL`) and records\n`stoppedBy: timeout`. Absent means no deadline.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "argv" + ], + "type": "object" + }, + "ContentDigest": { + "description": "`sha256:` followed by 64 lowercase hex digits: the digest of a file's raw\nbytes, independent of its path and mode.", + "pattern": "^sha256:[0-9a-f]{64}$", + "type": "string" + }, + "CreateDirectoryParams": { + "additionalProperties": false, + "description": "`fs/createDirectory` params.", + "properties": { + "path": { + "$ref": "#/definitions/EnvironmentPath" + }, + "recursive": { + "description": "Create missing parents, and succeed when the directory exists.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "CreateDirectoryResponse": { + "additionalProperties": false, + "description": "`fs/createDirectory` response.", + "type": "object" + }, + "DaemonInstanceId": { + "description": "`daemonInstanceId`: random per daemon start. A change between two\nconnections means the daemon restarted and every process handle from\nthe earlier instance is gone.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DataCapabilities": { + "additionalProperties": false, + "description": "Capability families. A present family means its base methods exist; a\nfamily's fields are facts that vary per instance. A request to an\nabsent family, or one needing an absent fact, is\n`capabilityUnavailable`.", + "properties": { + "filesystem": { + "anyOf": [ + { + "$ref": "#/definitions/FilesystemCapabilities" + } + ], + "description": "`fs/*` (search, glob and scan included) and `transfer/*`." + }, + "job": { + "anyOf": [ + { + "$ref": "#/definitions/JobCapabilities" + } + ], + "description": "`job/*`." + }, + "process": { + "anyOf": [ + { + "$ref": "#/definitions/ProcessCapabilities" + } + ], + "description": "`process/*`." + } + }, + "type": "object" + }, + "DataLimits": { + "additionalProperties": false, + "description": "Operational ceilings this daemon enforces. A request above a ceiling is\n`invalidParams` naming the field and the ceiling; nothing is clamped.", + "properties": { + "filesystem": { + "$ref": "#/definitions/FilesystemLimits" + }, + "job": { + "$ref": "#/definitions/JobLimits" + }, + "maxConcurrentRequests": { + "description": "Requests in flight per connection before `busy`; at least\n`MIN_CONCURRENT_REQUESTS`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxWaitMs": { + "description": "Largest `waitMs` of `process/read` and `job/read`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "process": { + "$ref": "#/definitions/ProcessLimits" + } + }, + "required": [ + "maxConcurrentRequests", + "maxWaitMs", + "process", + "job", + "filesystem" + ], + "type": "object" + }, + "DirectoryCursor": { + "description": "`cursor` and `nextCursor` of `fs/readDirectory`: an opaque position\nthe daemon mints, echoed back unchanged.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DirectoryEntry": { + "additionalProperties": false, + "description": "One `fs/readDirectory` entry.", + "properties": { + "metadata": { + "allOf": [ + { + "$ref": "#/definitions/FileMetadata" + } + ], + "description": "The entry itself, not a symlink's target. Entries that vanish while\nthe page is built are left out." + }, + "name": { + "$ref": "#/definitions/EntryName" + } + }, + "required": [ + "name", + "metadata" + ], + "type": "object" + }, + "EntryName": { + "description": "One entry name: a single path segment, neither `.` nor `..`, without\n`/` or NUL, at most 4096 bytes. `fs/readDirectory` entries use it.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(?!\\.\\.?$)[^/\\u0000]+$", + "type": "string" + }, + "EnvironmentPath": { + "description": "An absolute, normalized POSIX path in the environment: `/`, or `/` and\none or more segments joined by `/`. Empty, `.` and `..` segments and a\ntrailing `/` are rejected, never rewritten. At most 4096 bytes.\n`defaultCwd`, `homeDirectory`, every `cwd` and every path parameter use\nthis type; see the module's path model and confinement rule.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(/|(/(?!\\.\\.?(/|$))[^/\\u0000]+)+)$", + "type": "string" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/definitions/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/definitions/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "ExecInput": { + "description": "Standard input of a command.", + "oneOf": [ + { + "additionalProperties": false, + "description": "`/dev/null`: the command reads end of file. Model-facing tools use\nthis.", + "properties": { + "kind": { + "const": "none", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "These bytes, then end of file. Written by a writer task after the\noutput readers, the exit watcher and the deadline are in place.", + "properties": { + "data": { + "$ref": "#/definitions/ByteChunk" + }, + "kind": { + "const": "bytes", + "type": "string" + } + }, + "required": [ + "kind", + "data" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A persistent pipe fed by `process/write` until closed. Processes\nonly; `job/start` refuses it with `invalidParams`.", + "properties": { + "kind": { + "const": "pipe", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "ExecState": { + "description": "Lifecycle state of a process.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "exited", + "description": "The root exited and its output settled.", + "type": "string" + } + ] + }, + "ExitStatus": { + "description": "How the root process ended.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Normal exit with this status code.", + "properties": { + "code": { + "format": "int32", + "type": "integer" + }, + "kind": { + "const": "code", + "type": "string" + } + }, + "required": [ + "kind", + "code" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Killed by this signal.", + "properties": { + "coreDumped": { + "type": "boolean" + }, + "kind": { + "const": "signal", + "type": "string" + }, + "signal": { + "$ref": "#/definitions/SignalName" + } + }, + "required": [ + "kind", + "signal" + ], + "type": "object" + } + ] + }, + "FileKind": { + "description": "Kind of a filesystem entry.", + "oneOf": [ + { + "enum": [ + "file", + "directory", + "symlink" + ], + "type": "string" + }, + { + "const": "other", + "description": "Device, FIFO, socket or another special file.", + "type": "string" + } + ] + }, + "FileMetadata": { + "additionalProperties": false, + "description": "One file-entry shape for `fs/getMetadata` and `fs/readDirectory`.", + "properties": { + "createdAtMs": { + "description": "Absent when the filesystem records no birth time.", + "format": "int64", + "type": "integer" + }, + "executable": { + "description": "A regular file with any execute bit set; always false for\ndirectories, symlinks and special files.", + "type": "boolean" + }, + "kind": { + "$ref": "#/definitions/FileKind" + }, + "modifiedAtMs": { + "format": "int64", + "type": "integer" + }, + "sizeBytes": { + "description": "Size in bytes; for a symlink, the length of its target text.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "symlinkTarget": { + "description": "Target text of a symlink, when it is UTF-8.", + "type": "string" + }, + "version": { + "allOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ], + "description": "Changes whenever the entry's content or identity changes; compare it\nwith `ifMatch`." + } + }, + "required": [ + "kind", + "sizeBytes", + "modifiedAtMs", + "version" + ], + "type": "object" + }, + "FileVersion": { + "description": "Opaque version token of a file's content and identity. Equal tokens for\none path mean the content is unchanged. 1 to 255 bytes without control\ncharacters.", + "maxLength": 255, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "FilesystemCapabilities": { + "additionalProperties": false, + "description": "The `filesystem` family: `fs/*` and `transfer/*`.", + "properties": { + "write": { + "description": "Methods that change the filesystem are served; false on a daemon\nstarted read-only.", + "type": "boolean" + } + }, + "type": "object" + }, + "FilesystemLimits": { + "additionalProperties": false, + "description": "Ceilings of `fs/*` and `transfer/*`.", + "properties": { + "glob": { + "$ref": "#/definitions/GlobCeilings" + }, + "maxDirectoryEntries": { + "description": "Largest `limit` of `fs/readDirectory`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxReadFileBytes": { + "description": "Largest `maxBytes` of `fs/readFile`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxWriteFileBytes": { + "description": "Largest `data` of `fs/writeFile`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "scan": { + "$ref": "#/definitions/ScanCeilings" + }, + "search": { + "$ref": "#/definitions/SearchCeilings" + }, + "transfer": { + "$ref": "#/definitions/TransferCeilings" + } + }, + "required": [ + "maxReadFileBytes", + "maxWriteFileBytes", + "maxDirectoryEntries", + "search", + "glob", + "scan", + "transfer" + ], + "type": "object" + }, + "GapReason": { + "description": "Why a range of output cannot be returned.", + "oneOf": [ + { + "const": "overflow", + "description": "Never delivered: dropped from the middle because retained output\nexceeded the handle's cap. The head and the newest bytes are kept.", + "type": "string" + }, + { + "const": "expired", + "description": "Already delivered through the daemon cursor (or acknowledged) and\nreleased since.", + "type": "string" + } + ] + }, + "GetIdleParams": { + "additionalProperties": false, + "description": "`env/getIdle` params.", + "type": "object" + }, + "GetIdleResponse": { + "additionalProperties": false, + "description": "`env/getIdle` response.", + "properties": { + "idleForMs": { + "description": "Milliseconds since the last `use` request, or since running work\nlast ended, whichever is later. Zero while any process or job runs\nor any leftover process group has live members; the clock restarts\nwhen the last of them ends.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "leftoverProcessGroupCount": { + "description": "Process groups of exited processes that still have members alive:\nservices a command started and left running. They hold `idleForMs`\nat zero like running work.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "runningJobCount": { + "description": "Jobs that are `running`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "runningProcessCount": { + "description": "Processes started with `process/start` whose root has not exited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "idleForMs" + ], + "type": "object" + }, + "GetMetadataParams": { + "additionalProperties": false, + "description": "`fs/getMetadata` params.", + "properties": { + "followSymlinks": { + "description": "Describe the symlink's target instead of the link (subject to\nconfinement).", + "type": "boolean" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "GetMetadataResponse": { + "additionalProperties": false, + "description": "`fs/getMetadata` response.", + "properties": { + "metadata": { + "$ref": "#/definitions/FileMetadata" + } + }, + "required": [ + "metadata" + ], + "type": "object" + }, + "GlobCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/globFiles` quotas, field for field.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxDurationMs" + ], + "type": "object" + }, + "GlobFilesLimits": { + "additionalProperties": false, + "description": "Caller quotas of one enumeration; each at most its advertised ceiling.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "description": "Non-directory entries visited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxDurationMs" + ], + "type": "object" + }, + "GlobFilesParams": { + "additionalProperties": false, + "description": "`fs/globFiles` params: bounded recursive enumeration of non-directory\nentries matching `filter.include`.", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/definitions/TreeFilter" + } + ], + "description": "`include` must not be empty." + }, + "limits": { + "$ref": "#/definitions/GlobFilesLimits" + }, + "root": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "root", + "filter", + "limits" + ], + "type": "object" + }, + "GlobFilesResponse": { + "additionalProperties": false, + "description": "`fs/globFiles` response.", + "properties": { + "elapsedMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "matches": { + "description": "In walk order (sorted by path).", + "items": { + "$ref": "#/definitions/EnvironmentPath" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Entries the walk could not represent or list: names that are not\nUTF-8 and directories that could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "stopped": { + "anyOf": [ + { + "$ref": "#/definitions/GlobFilesStop" + } + ] + }, + "visitedFileCount": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "visitedFileCount", + "elapsedMs" + ], + "type": "object" + }, + "GlobFilesStop": { + "description": "Why an enumeration stopped before finishing its walk.", + "oneOf": [ + { + "const": "matchLimit", + "description": "`maxMatches` paths were found.", + "type": "string" + }, + { + "const": "fileLimit", + "description": "`maxFiles` non-directory entries were visited.", + "type": "string" + }, + { + "const": "timeLimit", + "description": "`maxDurationMs` ran out.", + "type": "string" + }, + { + "const": "responseLimit", + "description": "The next match would push the encoded matches past\n`MAX_RESULT_BYTES`.", + "type": "string" + } + ] + }, + "Id": { + "anyOf": [ + { + "format": "int64", + "type": "integer" + }, + { + "type": "string" + } + ], + "description": "A request id: an integer or a string, echoed in the response." + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "InitializeParams": { + "additionalProperties": false, + "description": "`initialize` params on the data plane.", + "properties": { + "clientInfo": { + "$ref": "#/definitions/ImplementationInfo" + }, + "protocolVersion": { + "description": "The client's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "clientInfo" + ], + "type": "object" + }, + "InitializeResponse": { + "additionalProperties": false, + "description": "`initialize` response on the data plane.", + "properties": { + "capabilities": { + "$ref": "#/definitions/DataCapabilities" + }, + "daemonInstanceId": { + "allOf": [ + { + "$ref": "#/definitions/DaemonInstanceId" + } + ], + "description": "Random per daemon start; a different value than on an earlier\nconnection means the daemon restarted and the earlier instance's\nprocess handles are gone." + }, + "defaultCwd": { + "allOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "The daemon's working directory: the default `cwd` of commands." + }, + "homeDirectory": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "Home directory of the user commands run as, when known." + }, + "limits": { + "$ref": "#/definitions/DataLimits" + }, + "plane": { + "allOf": [ + { + "$ref": "#/definitions/Plane" + } + ], + "description": "Always `data`." + }, + "protocolVersion": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "serverInfo": { + "$ref": "#/definitions/ImplementationInfo" + } + }, + "required": [ + "protocolVersion", + "plane", + "daemonInstanceId", + "serverInfo", + "capabilities", + "limits", + "defaultCwd" + ], + "type": "object" + }, + "InventoryEntry": { + "additionalProperties": false, + "description": "One inventory entry. Parents precede children.", + "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/TreeContent" + } + ], + "description": "A file entry always carries its `digest`." + }, + "path": { + "allOf": [ + { + "$ref": "#/definitions/TreePath" + } + ], + "description": "Relative to the selected root; `.` is the root itself." + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + }, + "JobCancelScope": { + "description": "Which jobs a `job/cancel` affects.", + "oneOf": [ + { + "const": "job", + "description": "Only the named jobs.", + "type": "string" + }, + { + "const": "dependents", + "description": "The named jobs and every job that transitively depends on them.", + "type": "string" + } + ] + }, + "JobCapabilities": { + "additionalProperties": false, + "description": "The `job` family: `job/*`. It has no instance facts.", + "type": "object" + }, + "JobDependency": { + "description": "A dependency reference.", + "oneOf": [ + { + "additionalProperties": false, + "description": "An existing job on this daemon, or one in the same request.", + "properties": { + "jobId": { + "$ref": "#/definitions/JobId" + }, + "kind": { + "const": "jobId", + "type": "string" + } + }, + "required": [ + "kind", + "jobId" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The `name` of a job in the same `job/start`.", + "properties": { + "kind": { + "const": "name", + "type": "string" + }, + "name": { + "$ref": "#/definitions/JobName" + } + }, + "required": [ + "kind", + "name" + ], + "type": "object" + } + ] + }, + "JobDependencyPolicy": { + "description": "What a job waits for from its `dependsOn` jobs.", + "oneOf": [ + { + "const": "allSucceeded", + "description": "Run once every dependency `succeeded`; any other terminal status\nmakes this job `dependencyFailed`.", + "type": "string" + }, + { + "const": "allTerminal", + "description": "Run once every dependency is terminal, whatever its status.", + "type": "string" + } + ] + }, + "JobId": { + "description": "`jobId`: caller-chosen name of one job run on one daemon.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "JobLimits": { + "additionalProperties": false, + "description": "Ceilings of `job/*`.", + "properties": { + "cancelGraceMs": { + "description": "Grace between `SIGTERM` and `SIGKILL` for a non-forced cancel and\ntimeouts.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxReadBytes": { + "description": "Largest `maxBytes` of one `job/read`, across its entries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxRequestJobs": { + "description": "Most jobs one `job/start`, `job/read` or `job/cancel` names, and most\nentries a `job/cancel` response lists.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxRetainedOutputBytes": { + "description": "Output a job retains: the head and the tail, each half of this; the\nmiddle is an `overflow` gap.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "recordRetentionMs": { + "description": "How long a terminal job's record and output are kept.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxReadBytes", + "maxRequestJobs", + "maxRetainedOutputBytes", + "recordRetentionMs", + "cancelGraceMs" + ], + "type": "object" + }, + "JobName": { + "description": "`name` of a job and `dependsOn {kind: name}`: a caller-chosen name,\nunique within one `job/start`, 1 to 128 bytes without control\ncharacters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "JobReadEntry": { + "description": "A `job/read` entry: the job's summary and output page, or why it could\nnot be read (`notFound` for an unknown or garbage-collected job).", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "job", + "type": "string" + }, + "output": { + "$ref": "#/definitions/OutputPage" + }, + "summary": { + "$ref": "#/definitions/JobSummary" + } + }, + "required": [ + "kind", + "summary", + "output" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "$ref": "#/definitions/ProtocolError" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "kind": { + "const": "error", + "type": "string" + } + }, + "required": [ + "kind", + "jobId", + "error" + ], + "type": "object" + } + ] + }, + "JobReadRequest": { + "additionalProperties": false, + "description": "One entry of a `job/read`. Job reads never consume: without `after` and\n`tailBytes` the page starts at offset 0. One job may be named by several\nentries, such as a head and a tail page.", + "properties": { + "after": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "maxBytes": { + "description": "Most `data` bytes for this entry; the request's `maxBytes` still\nbounds the total. Absent: whatever of the request's budget remains.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "tailBytes": { + "description": "Start at `end - tailBytes`; excludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobId" + ], + "type": "object" + }, + "JobRequestId": { + "description": "`requestId` of a `job/start` call. Part of each job's identity: a\nretried start with the same request id and spec replays the job; any\nother reuse of the job id is refused.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "JobSpec": { + "additionalProperties": false, + "description": "One job of a `job/start`.", + "properties": { + "command": { + "allOf": [ + { + "$ref": "#/definitions/CommandSpec" + } + ], + "description": "`input` must be `none` or `bytes`." + }, + "dependencyPolicy": { + "allOf": [ + { + "$ref": "#/definitions/JobDependencyPolicy" + } + ], + "description": "What \"finish\" means for `dependsOn`; `allSucceeded` by default." + }, + "dependsOn": { + "description": "Jobs that must finish first.", + "items": { + "$ref": "#/definitions/JobDependency" + }, + "type": "array" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "name": { + "anyOf": [ + { + "$ref": "#/definitions/JobName" + } + ], + "description": "Local name other jobs of the same `job/start` can depend on; unique\nwithin the request." + }, + "queueKey": { + "anyOf": [ + { + "$ref": "#/definitions/QueueKey" + } + ] + } + }, + "required": [ + "jobId", + "command" + ], + "type": "object" + }, + "JobStatus": { + "description": "Job lifecycle. `queued` and `running` are live; the rest are terminal.", + "oneOf": [ + { + "enum": [ + "running" + ], + "type": "string" + }, + { + "const": "queued", + "description": "Accepted; waiting for dependencies or its queue.", + "type": "string" + }, + { + "const": "succeeded", + "description": "Exited with code 0.", + "type": "string" + }, + { + "const": "failed", + "description": "Exited otherwise, or could not start.", + "type": "string" + }, + { + "const": "timedOut", + "description": "Its `timeoutMs` expired and the daemon stopped it.", + "type": "string" + }, + { + "const": "cancelled", + "description": "Stopped or never started because of `job/cancel`.", + "type": "string" + }, + { + "const": "dependencyFailed", + "description": "A dependency did not finish as the dependency policy requires.", + "type": "string" + }, + { + "const": "interrupted", + "description": "The daemon stopped while the job was queued or running.", + "type": "string" + } + ] + }, + "JobSummary": { + "additionalProperties": false, + "description": "What a job is and how it ended. Instants are the guest's wall clock.", + "properties": { + "cancelRequestedAtMs": { + "description": "When a `job/cancel` reached the job while it ran.", + "format": "int64", + "type": "integer" + }, + "createdAtMs": { + "format": "int64", + "type": "integer" + }, + "dependsOn": { + "description": "Resolved ids of the jobs this one depends on.", + "items": { + "$ref": "#/definitions/JobId" + }, + "type": "array" + }, + "durationMs": { + "description": "Monotonic run time from start to finish, immune to guest clock jumps.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/definitions/ProtocolError" + } + ], + "description": "Why a `failed` or `interrupted` job produced no exit, or an\ninfrastructure failure while it ran." + }, + "exit": { + "anyOf": [ + { + "$ref": "#/definitions/ExitStatus" + } + ], + "description": "How the root process ended, once it ran and exited." + }, + "finishedAtMs": { + "format": "int64", + "type": "integer" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "leftovers": { + "anyOf": [ + { + "$ref": "#/definitions/LeftoverReport" + } + ], + "description": "What the daemon swept from the job's group at completion\n(`disposition: swept`)." + }, + "name": { + "anyOf": [ + { + "$ref": "#/definitions/JobName" + } + ] + }, + "queueKey": { + "anyOf": [ + { + "$ref": "#/definitions/QueueKey" + } + ] + }, + "queuedAtMs": { + "description": "When the job became runnable and waited for its queue.", + "format": "int64", + "type": "integer" + }, + "requestId": { + "$ref": "#/definitions/JobRequestId" + }, + "startedAtMs": { + "format": "int64", + "type": "integer" + }, + "status": { + "$ref": "#/definitions/JobStatus" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/definitions/StoppedBy" + } + ], + "description": "Set when the daemon stopped the running job: `timeout` for\n`timedOut`, `terminate` or `kill` for a cancel without or with\n`force`." + } + }, + "required": [ + "jobId", + "requestId", + "status", + "createdAtMs" + ], + "type": "object" + }, + "JobSummaryEntry": { + "description": "A job summary, or why the named job could not be found.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "job", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/JobSummary" + } + }, + "required": [ + "kind", + "summary" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "$ref": "#/definitions/ProtocolError" + }, + "jobId": { + "$ref": "#/definitions/JobId" + }, + "kind": { + "const": "error", + "type": "string" + } + }, + "required": [ + "kind", + "jobId", + "error" + ], + "type": "object" + } + ] + }, + "LeftoverDisposition": { + "description": "What the daemon did with leftovers. The policies differ on purpose.", + "oneOf": [ + { + "const": "kept", + "description": "Left running (processes): they keep the environment's lifetime and\nare reported until they exit.", + "type": "string" + }, + { + "const": "swept", + "description": "Terminated when the job finished (jobs); the submitter should fix\nits script's teardown.", + "type": "string" + } + ] + }, + "LeftoverProcess": { + "additionalProperties": false, + "description": "One leftover process.", + "properties": { + "command": { + "description": "Command line when the platform exposes it, otherwise the executable\nname, cut to\n`MAX_LEFTOVER_COMMAND_BYTES`\non a character boundary; absent when neither could be read.", + "type": "string" + }, + "pid": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "LeftoverReport": { + "additionalProperties": false, + "description": "Processes of a command's group still alive after its root exited.\nPresent only when there is at least one.", + "properties": { + "disposition": { + "$ref": "#/definitions/LeftoverDisposition" + }, + "omittedProcessCount": { + "description": "Leftovers beyond the listed ones.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "processes": { + "description": "At most `MAX_LEFTOVER_PROCESSES`,\nlowest pid first; never empty.", + "items": { + "$ref": "#/definitions/LeftoverProcess" + }, + "type": "array" + } + }, + "required": [ + "disposition", + "processes" + ], + "type": "object" + }, + "ListTransferMissingParams": { + "additionalProperties": false, + "description": "`transfer/listMissing` params.", + "properties": { + "limit": { + "description": "Most digests to return, 1 to\n`limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "limit" + ], + "type": "object" + }, + "ListTransferMissingResponse": { + "additionalProperties": false, + "description": "`transfer/listMissing` response: digests whose content the daemon still\nneeds. The set shrinks as uploads complete; call again rather than\npaging.", + "properties": { + "digests": { + "items": { + "$ref": "#/definitions/ContentDigest" + }, + "type": "array" + }, + "remainingCount": { + "description": "Missing digests beyond this page.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "OutputChunk": { + "additionalProperties": false, + "description": "Contiguous output bytes from one stream.", + "properties": { + "data": { + "$ref": "#/definitions/ByteChunk" + }, + "offset": { + "description": "Byte offset of the chunk's first byte in the combined output.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "stream": { + "$ref": "#/definitions/OutputStream" + } + }, + "required": [ + "offset", + "stream", + "data" + ], + "type": "object" + }, + "OutputGap": { + "additionalProperties": false, + "description": "Output bytes in `[offset, offset + omittedBytes)` that a page cannot\nreturn.", + "properties": { + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "omittedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "reason": { + "$ref": "#/definitions/GapReason" + } + }, + "required": [ + "offset", + "omittedBytes", + "reason" + ], + "type": "object" + }, + "OutputPage": { + "additionalProperties": false, + "description": "One page of a handle's output. Returned by `process/read` and in every\n`job/read` entry. A page holds at most\n`MAX_PAGE_CHUNKS` chunks and gaps\ntogether (for `job/read`, across all entries); one that reaches it ends\nearly, and `next` says where to continue.", + "properties": { + "chunks": { + "description": "In offset order, at most `maxBytes` of `data` in total.", + "items": { + "$ref": "#/definitions/OutputChunk" + }, + "type": "array" + }, + "end": { + "description": "Bytes produced so far; final once the handle has exited.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "gaps": { + "description": "Unreturnable ranges between the requested start and `next`.", + "items": { + "$ref": "#/definitions/OutputGap" + }, + "type": "array" + }, + "next": { + "description": "Offset to read from next: past the last returned byte or gap.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "next", + "end" + ], + "type": "object" + }, + "OutputStream": { + "description": "Which descriptor produced a chunk. Jobs never produce `pty`.", + "oneOf": [ + { + "enum": [ + "stdout", + "stderr" + ], + "type": "string" + }, + { + "const": "pty", + "description": "The PTY master, which merges stdout and stderr.", + "type": "string" + } + ] + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "Plane": { + "description": "The plane a connection serves. Each `initialize` response names its\nplane, so a client that reached the wrong socket fails at the handshake.", + "oneOf": [ + { + "const": "data", + "description": "envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "controller", + "description": "A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`.", + "type": "string" + }, + { + "const": "registration", + "description": "The registration channel between a daemon and the gateway, versioned\nby `ENVD_PROTOCOL_VERSION`.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + } + ] + }, + "ProcessCapabilities": { + "additionalProperties": false, + "description": "The `process` family: `process/*`.", + "properties": { + "pty": { + "description": "`process/start` accepts `pty`.", + "type": "boolean" + } + }, + "type": "object" + }, + "ProcessId": { + "description": "`processId`: caller-chosen handle of a process on one daemon.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ProcessLimits": { + "additionalProperties": false, + "description": "Ceilings of `process/*`.", + "properties": { + "maxExitedRetainedBytes": { + "description": "Output retained across all exited processes; beyond it the oldest\nare evicted early.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxInputQueueBytes": { + "description": "Bytes queued for a process's input before `process/write` accepts\nfewer than it was sent.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxReadBytes": { + "description": "Largest `maxBytes` of one `process/read`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxRetainedOutputBytes": { + "description": "Output a process retains; beyond it the oldest delivered bytes are\nreleased (`expired` gaps), then the undelivered middle (`overflow`\ngaps).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "retentionAfterExitMs": { + "description": "How long an exited process stays readable after a read first\nreturned its settled exit with nothing left unread.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "terminateGraceMs": { + "description": "Grace between `SIGTERM` and `SIGKILL` for `terminate` and timeouts.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "unreadRetentionMs": { + "description": "How long an exited process whose output was never fully read stays\nreadable.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxReadBytes", + "maxRetainedOutputBytes", + "retentionAfterExitMs", + "unreadRetentionMs", + "maxExitedRetainedBytes", + "terminateGraceMs", + "maxInputQueueBytes" + ], + "type": "object" + }, + "ProcessSignal": { + "description": "Signal delivered by `process/terminate`.", + "oneOf": [ + { + "const": "interrupt", + "description": "`SIGINT` to the foreground process group (the terminal's under a\nPTY, otherwise the process group). Records nothing; the next read\nshows what the process did.", + "type": "string" + }, + { + "const": "terminate", + "description": "`SIGTERM` to the group, then `SIGKILL` after\n`limits.process.terminateGraceMs` if members remain. Records\n`stoppedBy: terminate`.", + "type": "string" + }, + { + "const": "kill", + "description": "`SIGKILL` to the group. Records `stoppedBy: kill`.", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "PtySize": { + "additionalProperties": false, + "description": "Terminal size of a PTY. Each dimension is 1 to 4096.", + "properties": { + "cols": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "rows": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "rows", + "cols" + ], + "type": "object" + }, + "QueueKey": { + "description": "`queueKey`: jobs sharing a key run one at a time, in start order.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ReadDirectoryParams": { + "additionalProperties": false, + "description": "`fs/readDirectory` params. Pages in name order.", + "properties": { + "cursor": { + "anyOf": [ + { + "$ref": "#/definitions/DirectoryCursor" + } + ], + "description": "`nextCursor` of the previous page; a cursor this daemon did not mint\nis `invalidParams`." + }, + "limit": { + "description": "Most entries per page, 1 to `limits.filesystem.maxDirectoryEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "path": { + "allOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "A directory, or a symlink to one inside the root." + } + }, + "required": [ + "path", + "limit" + ], + "type": "object" + }, + "ReadDirectoryResponse": { + "additionalProperties": false, + "description": "`fs/readDirectory` response.", + "properties": { + "entries": { + "items": { + "$ref": "#/definitions/DirectoryEntry" + }, + "type": "array" + }, + "nextCursor": { + "anyOf": [ + { + "$ref": "#/definitions/DirectoryCursor" + } + ], + "description": "Opaque cursor of the next page; absent on the last page." + }, + "skippedEntryCount": { + "description": "Entries in this page's range skipped because their names are not\nUTF-8. Entries that vanish while the page is built are left out and\nnot counted.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "ReadFileParams": { + "additionalProperties": false, + "description": "`fs/readFile` params. Reads a regular file only; anything else is\n`failedPrecondition` (`notRegularFile`) and is never opened for a\nblocking read.", + "properties": { + "maxBytes": { + "description": "Most bytes to return, at most `limits.filesystem.maxReadFileBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "offset": { + "description": "First byte to return; 0 by default.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path", + "maxBytes" + ], + "type": "object" + }, + "ReadFileResponse": { + "additionalProperties": false, + "description": "`fs/readFile` response.", + "properties": { + "data": { + "$ref": "#/definitions/ByteChunk" + }, + "fileSize": { + "description": "Size of the whole file when it was read.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "truncated": { + "description": "Bytes exist after `offset` plus the length of `data`.", + "type": "boolean" + }, + "version": { + "$ref": "#/definitions/FileVersion" + } + }, + "required": [ + "data", + "fileSize", + "version" + ], + "type": "object" + }, + "ReadJobsParams": { + "additionalProperties": false, + "description": "`job/read` params.", + "properties": { + "jobs": { + "description": "1 to `limits.job.maxRequestJobs` entries, each with its own position.", + "items": { + "$ref": "#/definitions/JobReadRequest" + }, + "type": "array" + }, + "maxBytes": { + "description": "Most `data` bytes across all entries, at most\n`limits.job.maxReadBytes`, filled in request order: an entry reached\nafter the budget is spent returns its summary and an empty page at\nits start position. 0 reads only summaries.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "returnOn": { + "allOf": [ + { + "$ref": "#/definitions/ReturnOn" + } + ], + "description": "`anyOutput`: return when any entry has output past its position or\nchanged status; `exit`: when every entry is terminal; `window`: wait\nthe whole window." + }, + "waitMs": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "jobs", + "maxBytes", + "waitMs", + "returnOn" + ], + "type": "object" + }, + "ReadJobsResponse": { + "additionalProperties": false, + "description": "`job/read` response: one entry per requested job, in request order.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/definitions/JobReadEntry" + }, + "type": "array" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "ReadProcessParams": { + "additionalProperties": false, + "description": "`process/read` params.", + "properties": { + "ack": { + "description": "Acknowledge every byte before this offset first: the cursor moves to\nit (never backwards) whether or not this read's response arrives. A\ncaller that tracks positions reads with `after` and acknowledges\nwith `ack` for exactly-once delivery.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "after": { + "description": "Start offset. Absent: the daemon cursor, which this read advances to\n`next` once its response is written. Present: a non-consuming read\nthat leaves the cursor alone.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxBytes": { + "description": "Most `data` bytes to return, at most `limits.process.maxReadBytes`.\n0 reads only the state.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "processId": { + "$ref": "#/definitions/ProcessId" + }, + "returnOn": { + "$ref": "#/definitions/ReturnOn" + }, + "tailBytes": { + "description": "Start at `end - tailBytes` instead (clamped at 0). Non-consuming;\nexcludes `after`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "waitMs": { + "description": "How long to wait, at most `limits.maxWaitMs`. 0 polls.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "processId", + "maxBytes", + "waitMs", + "returnOn" + ], + "type": "object" + }, + "ReadProcessResponse": { + "additionalProperties": false, + "description": "`process/read` response.", + "properties": { + "error": { + "anyOf": [ + { + "$ref": "#/definitions/ProtocolError" + } + ], + "description": "Infrastructure failure after start (output capture or wait failed)." + }, + "exit": { + "anyOf": [ + { + "$ref": "#/definitions/ExitStatus" + } + ], + "description": "How the root ended; present once `state` is `exited`, unless `error`\nexplains why it could not be observed." + }, + "leftovers": { + "anyOf": [ + { + "$ref": "#/definitions/LeftoverReport" + } + ], + "description": "Group members alive after the root exited (`disposition: kept`),\nsampled when this response was built. Absent while running, when\nnone remain, and on platforms without process enumeration." + }, + "output": { + "$ref": "#/definitions/OutputPage" + }, + "pid": { + "description": "OS pid of the root process.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "state": { + "$ref": "#/definitions/ExecState" + }, + "stoppedBy": { + "anyOf": [ + { + "$ref": "#/definitions/StoppedBy" + } + ], + "description": "Set when the daemon stopped the process; absent on a natural exit." + } + }, + "required": [ + "output", + "state", + "pid" + ], + "type": "object" + }, + "ReadTransferContentParams": { + "additionalProperties": false, + "description": "`transfer/read` params: read a captured file's content by digest.", + "properties": { + "digest": { + "$ref": "#/definitions/ContentDigest" + }, + "maxBytes": { + "description": "1 to `limits.filesystem.transfer.maxContentChunkBytes`.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "offset": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "digest", + "offset", + "maxBytes" + ], + "type": "object" + }, + "ReadTransferContentResponse": { + "additionalProperties": false, + "description": "`transfer/read` response.", + "properties": { + "data": { + "$ref": "#/definitions/ByteChunk" + }, + "truncated": { + "description": "Bytes exist after `offset` plus the length of `data`, as for\n`fs/readFile`.", + "type": "boolean" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "ReadTransferInventoryParams": { + "additionalProperties": false, + "description": "`transfer/readInventory` params: page a sealed inventory by index.", + "properties": { + "index": { + "description": "Index of the first entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "limit": { + "description": "1 to `limits.filesystem.transfer.maxInventoryPageEntries`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "index", + "limit" + ], + "type": "object" + }, + "ReadTransferInventoryResponse": { + "additionalProperties": false, + "description": "`transfer/readInventory` response.", + "properties": { + "entries": { + "items": { + "$ref": "#/definitions/InventoryEntry" + }, + "type": "array" + }, + "nextIndex": { + "description": "Index of the next page; absent after the last entry.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "RemoveParams": { + "additionalProperties": false, + "description": "`fs/remove` params.", + "properties": { + "force": { + "description": "Succeed when the path does not exist.", + "type": "boolean" + }, + "path": { + "allOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "A symlink is removed, never followed." + }, + "recursive": { + "description": "Remove a directory and its contents.", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "RemoveResponse": { + "additionalProperties": false, + "description": "`fs/remove` response.", + "type": "object" + }, + "RenameParams": { + "additionalProperties": false, + "description": "`fs/rename` params. Atomic within one filesystem; across filesystems it\nis `failedPrecondition` (`crossDevice`). The entry keeps its mode.", + "properties": { + "destination": { + "$ref": "#/definitions/EnvironmentPath" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ], + "description": "Rename only when the source's version equals this token." + }, + "overwrite": { + "description": "Replace an existing destination file; otherwise an existing\ndestination is `alreadyExists`. A directory destination is never\nreplaced.", + "type": "boolean" + }, + "source": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "source", + "destination" + ], + "type": "object" + }, + "RenameResponse": { + "additionalProperties": false, + "description": "`fs/rename` response.", + "type": "object" + }, + "ReturnOn": { + "description": "When a waiting read returns. Every mode also returns when `waitMs`\nelapses, when the handle reaches a settled exit, or at once when there\nis nothing left to wait for.", + "oneOf": [ + { + "const": "window", + "description": "Collect for the whole window; return early only when unread output\nreaches `maxBytes`.", + "type": "string" + }, + { + "const": "anyOutput", + "description": "Return as soon as any output past the start position exists.", + "type": "string" + }, + { + "const": "exit", + "description": "Return only at the settled exit (for `job/read`: once every entry is\nterminal), even if more than `maxBytes` of output accumulates.", + "type": "string" + } + ] + }, + "ScanCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/scan`: root and pattern counts, then its quotas.", + "properties": { + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxPatterns": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxRoots": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxRoots", + "maxPatterns", + "maxEntries", + "maxFileBytes", + "maxTotalBytes", + "maxDurationMs" + ], + "type": "object" + }, + "ScanEntry": { + "additionalProperties": false, + "description": "One selected entry below a root.", + "properties": { + "content": { + "$ref": "#/definitions/TreeContent" + }, + "data": { + "anyOf": [ + { + "$ref": "#/definitions/ByteChunk" + } + ], + "description": "The file's bytes when `readContent` was set." + }, + "path": { + "allOf": [ + { + "$ref": "#/definitions/TreePath" + } + ], + "description": "Relative to the root as requested." + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + }, + "ScanFingerprint": { + "description": "Opaque fingerprint of one root's selection, prefixed `scan:v1:`, at most\n255 bytes without control characters.", + "maxLength": 255, + "minLength": 9, + "pattern": "^scan:v1:[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "ScanLimits": { + "additionalProperties": false, + "description": "Caller quotas of one scan.", + "properties": { + "maxDurationMs": { + "description": "Walk time per root; exhausting it fails that root\n(`deadlineExceeded`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "description": "Selected entries per root; more fail that root (`payloadTooLarge`\nwith `limitEntries`).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "description": "Larger selected files are skipped (`fileTooLarge`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "description": "Content bytes returned across all roots, in root order: a root whose\ncontent would exceed the rest fails (`payloadTooLarge` with\n`limitBytes`) and later roots are still scanned. Independently, a\nroot whose encoded result (entries, skips and base64 content) would\npush the scan's results past\n`MAX_RESULT_BYTES` fails the same\nway.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxEntries", + "maxFileBytes", + "maxTotalBytes", + "maxDurationMs" + ], + "type": "object" + }, + "ScanParams": { + "additionalProperties": false, + "description": "`fs/scan` params.", + "properties": { + "filter": { + "allOf": [ + { + "$ref": "#/definitions/TreeFilter" + } + ], + "description": "Applied below every root; at most `limits.filesystem.scan.maxPatterns`\npatterns in total." + }, + "followSymlinks": { + "description": "Follow symlinks whose targets stay inside the filesystem root; others\nare skipped (`symlinkOutsideRoot`). Without it symlinks are skipped\n(`symlink`).", + "type": "boolean" + }, + "includeDigests": { + "description": "Return each selected file's `digest`.", + "type": "boolean" + }, + "limits": { + "allOf": [ + { + "$ref": "#/definitions/ScanLimits" + } + ], + "description": "Caller quotas; each at most its ceiling." + }, + "readContent": { + "description": "Return the bytes of every selected file in `data`.", + "type": "boolean" + }, + "roots": { + "description": "1 to `limits.filesystem.scan.maxRoots` roots.", + "items": { + "$ref": "#/definitions/ScanRoot" + }, + "type": "array" + } + }, + "required": [ + "roots", + "limits" + ], + "type": "object" + }, + "ScanResponse": { + "additionalProperties": false, + "description": "`fs/scan` response: one result per requested root, in request order.", + "properties": { + "roots": { + "description": "Never empty.", + "items": { + "$ref": "#/definitions/ScanRootResult" + }, + "type": "array" + } + }, + "required": [ + "roots" + ], + "type": "object" + }, + "ScanRoot": { + "additionalProperties": false, + "description": "One root of a scan.", + "properties": { + "ifNoneMatch": { + "anyOf": [ + { + "$ref": "#/definitions/ScanFingerprint" + } + ], + "description": "The root's previous `fingerprint`: when the selection is unchanged\nthe result is `unchanged` and carries no entries." + }, + "maxDepth": { + "description": "Overrides `filter.maxDepth` for this root. `0` examines only the\nroot itself: a `complete` result without entries proves it is an\nexisting directory, which is how discovery checks a working\ndirectory in the same call.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "ScanRootResult": { + "description": "The outcome of one root.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Every selected entry, plus what was skipped and why.", + "properties": { + "canonicalPath": { + "anyOf": [ + { + "$ref": "#/definitions/EnvironmentPath" + } + ], + "description": "Resolved absolute path of the root when it differs from `path`\n(the root is a symlink)." + }, + "entries": { + "items": { + "$ref": "#/definitions/ScanEntry" + }, + "type": "array" + }, + "fingerprint": { + "$ref": "#/definitions/ScanFingerprint" + }, + "kind": { + "const": "complete", + "type": "string" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + }, + "skipped": { + "description": "The first\n`MAX_LISTED_SKIPS` skipped\nentries, in walk order.", + "items": { + "$ref": "#/definitions/ScanSkip" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Total number of skipped entries, listed or not; present whenever\nit is not zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "path", + "fingerprint" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The selection matches `ifNoneMatch`.", + "properties": { + "fingerprint": { + "$ref": "#/definitions/ScanFingerprint" + }, + "kind": { + "const": "unchanged", + "type": "string" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "kind", + "path", + "fingerprint" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The root does not exist.", + "properties": { + "kind": { + "const": "missing", + "type": "string" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "kind", + "path" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The root could not be scanned; no entries are returned for it.", + "properties": { + "error": { + "$ref": "#/definitions/ProtocolError" + }, + "kind": { + "const": "failed", + "type": "string" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "kind", + "path", + "error" + ], + "type": "object" + } + ] + }, + "ScanSkip": { + "additionalProperties": false, + "description": "An entry a scan left out.", + "properties": { + "path": { + "anyOf": [ + { + "$ref": "#/definitions/TreePath" + } + ], + "description": "Absent when the entry's name is not UTF-8." + }, + "reason": { + "$ref": "#/definitions/ScanSkipReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "ScanSkipReason": { + "description": "Why a scan left an entry out.", + "oneOf": [ + { + "const": "fileTooLarge", + "description": "Larger than `maxFileBytes`.", + "type": "string" + }, + { + "const": "symlink", + "description": "A symlink, and `followSymlinks` is not set.", + "type": "string" + }, + { + "const": "symlinkOutsideRoot", + "description": "A symlink whose target leaves the filesystem root, or a link cycle.", + "type": "string" + }, + { + "const": "specialFile", + "description": "A device, FIFO, socket or another special file.", + "type": "string" + }, + { + "const": "nonUtf8Name", + "description": "The name is not UTF-8.", + "type": "string" + }, + { + "const": "unreadable", + "description": "The entry vanished or could not be read during the walk.", + "type": "string" + } + ] + }, + "SearchCeilings": { + "additionalProperties": false, + "description": "Ceilings of `fs/searchText` quotas, field for field.", + "properties": { + "maxBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxLineBytes": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxBytes", + "maxDurationMs", + "maxLineBytes" + ], + "type": "object" + }, + "SearchTextLimits": { + "additionalProperties": false, + "description": "Caller quotas of one search; each at most its advertised ceiling.", + "properties": { + "maxBytes": { + "description": "Bytes actually read across files.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxFiles": { + "description": "Files searched, not merely visited.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxLineBytes": { + "description": "Longer matching lines are cut and marked `lineTruncated`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxMatches": { + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxMatches", + "maxFiles", + "maxBytes", + "maxDurationMs", + "maxLineBytes" + ], + "type": "object" + }, + "SearchTextMatch": { + "additionalProperties": false, + "description": "One matching line.", + "properties": { + "line": { + "description": "The matching line without its terminator, cut at `maxLineBytes` on a\ncharacter boundary.", + "type": "string" + }, + "lineNumber": { + "description": "1-based.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "lineTruncated": { + "type": "boolean" + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path", + "lineNumber", + "line" + ], + "type": "object" + }, + "SearchTextParams": { + "additionalProperties": false, + "description": "`fs/searchText` params: a bounded recursive regex search run by the\ndaemon. A file larger than the remaining byte budget is skipped and\ncounted; it never ends the walk.", + "properties": { + "filter": { + "$ref": "#/definitions/TreeFilter" + }, + "limits": { + "$ref": "#/definitions/SearchTextLimits" + }, + "pattern": { + "description": "Regular expression in the Rust `regex` crate's syntax, matched per\nline.", + "type": "string" + }, + "patternIgnoreCase": { + "description": "Match `pattern` case-insensitively. `filter.ignoreCase` governs the\npath patterns instead.", + "type": "boolean" + }, + "root": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "root", + "pattern", + "limits" + ], + "type": "object" + }, + "SearchTextResponse": { + "additionalProperties": false, + "description": "`fs/searchText` response.", + "properties": { + "elapsedMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "matches": { + "items": { + "$ref": "#/definitions/SearchTextMatch" + }, + "type": "array" + }, + "searchedBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "searchedFileCount": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "skippedEntryCount": { + "description": "Entries the walk could not represent or list: names that are not\nUTF-8 and directories that could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "skippedFileCount": { + "description": "Files skipped because they exceeded the remaining byte budget, are\nbinary or could not be read.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "stopped": { + "anyOf": [ + { + "$ref": "#/definitions/SearchTextStop" + } + ], + "description": "Absent when the walk finished within its limits." + } + }, + "required": [ + "searchedFileCount", + "searchedBytes", + "elapsedMs" + ], + "type": "object" + }, + "SearchTextStop": { + "description": "Why a search stopped before finishing its walk.", + "oneOf": [ + { + "const": "matchLimit", + "description": "`maxMatches` matches were found.", + "type": "string" + }, + { + "const": "fileLimit", + "description": "`maxFiles` files were searched.", + "type": "string" + }, + { + "const": "byteLimit", + "description": "`maxBytes` bytes were read.", + "type": "string" + }, + { + "const": "timeLimit", + "description": "`maxDurationMs` ran out.", + "type": "string" + }, + { + "const": "responseLimit", + "description": "The next match would push the encoded matches past\n`MAX_RESULT_BYTES`.", + "type": "string" + } + ] + }, + "SecretString": { + "description": "A secret value carried on the wire, such as a `secretEnv` value, a\nregistration key or a stream token. It serializes as a plain string;\n`Debug` is redacted so structured logging never prints it.", + "type": "string" + }, + "SignalName": { + "description": "A POSIX signal name such as `SIGSEGV` or `SIGRTMIN+3`: `SIG` followed by\n1 to 16 uppercase letters, digits or `+`. Names, not numbers, because\nnumbers differ between platforms.", + "pattern": "^SIG[A-Z0-9+]{1,16}$", + "type": "string" + }, + "SpecialFilePolicy": { + "description": "Devices, FIFOs and sockets met during a capture.", + "oneOf": [ + { + "const": "reject", + "description": "Fail the operation (`notRegularFile`).", + "type": "string" + }, + { + "const": "skip", + "description": "Leave them out and report them in `skipped`.", + "type": "string" + } + ] + }, + "StartJobsParams": { + "additionalProperties": false, + "description": "`job/start` params. All-or-nothing: either every job is accepted (or\nreplayed) or none is.", + "properties": { + "jobs": { + "description": "1 to `limits.job.maxRequestJobs` jobs.", + "items": { + "$ref": "#/definitions/JobSpec" + }, + "type": "array" + }, + "requestId": { + "allOf": [ + { + "$ref": "#/definitions/JobRequestId" + } + ], + "description": "Idempotency key of this start. Retrying with the same request id and\nidentical jobs replays the existing jobs; reusing a job id with\nanother request id or spec is `failedPrecondition` (`jobIdInUse`)." + } + }, + "required": [ + "requestId", + "jobs" + ], + "type": "object" + }, + "StartJobsResponse": { + "additionalProperties": false, + "description": "`job/start` response: one summary per requested job, in request order.", + "properties": { + "jobs": { + "description": "Never empty.", + "items": { + "$ref": "#/definitions/JobSummary" + }, + "type": "array" + } + }, + "required": [ + "jobs" + ], + "type": "object" + }, + "StartProcessParams": { + "additionalProperties": false, + "description": "`process/start` params.", + "properties": { + "command": { + "$ref": "#/definitions/CommandSpec" + }, + "processId": { + "allOf": [ + { + "$ref": "#/definitions/ProcessId" + } + ], + "description": "Caller-chosen handle, reserved before any side effect: a start whose\nid is already in use (running, retained or remembered by a tombstone)\nis `alreadyExists` and runs nothing." + }, + "pty": { + "anyOf": [ + { + "$ref": "#/definitions/PtySize" + } + ], + "description": "Run under a PTY of this size. The PTY master is the input: `input`\nmust then be `none` or `bytes`, and `process/write` is always\naccepted while the process runs. Requires\n`capabilities.process.pty`." + } + }, + "required": [ + "processId", + "command" + ], + "type": "object" + }, + "StartProcessResponse": { + "additionalProperties": false, + "description": "`process/start` response, sent once the process runs.", + "properties": { + "pid": { + "description": "OS pid of the root process, which is also its process group id.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "pid" + ], + "type": "object" + }, + "StoppedBy": { + "description": "Why the daemon stopped a process or job that did not exit on its own.\nSet only when the daemon signalled a root that was still running.", + "oneOf": [ + { + "const": "timeout", + "description": "`timeoutMs` expired: `SIGTERM`, the grace, then `SIGKILL`.", + "type": "string" + }, + { + "const": "terminate", + "description": "`process/terminate` with `terminate`, or `job/cancel` without\n`force`: `SIGTERM`, the grace, then `SIGKILL`.", + "type": "string" + }, + { + "const": "kill", + "description": "`process/terminate` with `kill`, or `job/cancel` with `force`:\n`SIGKILL` at once.", + "type": "string" + } + ] + }, + "SymlinkPolicy": { + "description": "Symlinks met during a capture.", + "oneOf": [ + { + "const": "expandInside", + "description": "Capture a link whose target stays inside the selection as the entry\nit points to; a link leaving the selection or a cycle fails the\noperation (`symlinkRejected`). A dangling link inside the selection\nis left out and reported in `skipped` as `symlink`.", + "type": "string" + }, + { + "const": "skip", + "description": "Leave every link out and report it in `skipped`.", + "type": "string" + }, + { + "const": "reject", + "description": "Fail on the first link.", + "type": "string" + } + ] + }, + "TerminateProcessParams": { + "additionalProperties": false, + "description": "`process/terminate` params.", + "properties": { + "processId": { + "$ref": "#/definitions/ProcessId" + }, + "signal": { + "$ref": "#/definitions/ProcessSignal" + } + }, + "required": [ + "processId", + "signal" + ], + "type": "object" + }, + "TerminateProcessResponse": { + "additionalProperties": false, + "description": "`process/terminate` response, sent once the signal was delivered (any\nescalation continues in the background).", + "properties": { + "running": { + "description": "True when the root process was still running when the signal was\nsent.", + "type": "boolean" + } + }, + "type": "object" + }, + "TransferCeilings": { + "additionalProperties": false, + "description": "Ceilings of `transfer/*`: capacity, page and chunk sizes, receipt\nretention, then the per-operation quotas.", + "properties": { + "maxActiveOperations": { + "description": "Operations not yet complete, aborted or failed, across connections;\na begin beyond it is `busy` (`transferCapacity`).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxContentChunkBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxDepth": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxInventoryPageEntries": { + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxManifestBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "receiptRetentionMs": { + "description": "How long a finished operation's receipt is answered.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxActiveOperations", + "maxContentChunkBytes", + "maxInventoryPageEntries", + "receiptRetentionMs", + "maxEntries", + "maxDepth", + "maxFileBytes", + "maxTotalBytes", + "maxManifestBytes", + "maxDurationMs" + ], + "type": "object" + }, + "TransferLimits": { + "additionalProperties": false, + "description": "Caller quotas of one operation; each at most its advertised ceiling.", + "properties": { + "maxDepth": { + "description": "The selected root has depth 0.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxDurationMs": { + "description": "From begin to commit.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxEntries": { + "description": "Entries, the selected root included.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "maxFileBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxManifestBytes": { + "description": "Encoded inventory size.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "maxTotalBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "maxEntries", + "maxDepth", + "maxFileBytes", + "maxTotalBytes", + "maxManifestBytes", + "maxDurationMs" + ], + "type": "object" + }, + "TransferOnExisting": { + "description": "What a materialize does with an existing destination.", + "oneOf": [ + { + "const": "error", + "description": "Refuse at begin (`alreadyExists`).", + "type": "string" + }, + { + "const": "replace", + "description": "Replace it atomically at commit.", + "type": "string" + } + ] + }, + "TransferOperationId": { + "description": "`operationId` of one transfer operation.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "TransferOperationParams": { + "additionalProperties": false, + "description": "Params of `transfer/advance`, `transfer/getStatus`, `transfer/commit`\nand `transfer/abort`: each names only the operation.", + "properties": { + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId" + ], + "type": "object" + }, + "TransferPhase": { + "description": "Where an operation stands; see the module docs for the order per\nselection kind.", + "oneOf": [ + { + "enum": [ + "complete", + "aborted" + ], + "type": "string" + }, + { + "const": "scanning", + "description": "The daemon walks a tree: the capture source, or the destination of a\nmaterialize. Advance until the phase changes.", + "type": "string" + }, + { + "const": "inventory", + "description": "Capture: the inventory is sealed and pageable. Materialize: waiting\nfor appended entries.", + "type": "string" + }, + { + "const": "content", + "description": "Waiting for the content of `transfer/listMissing` digests.", + "type": "string" + }, + { + "const": "staging", + "description": "The daemon assembles the staged tree. Advance until `ready`.", + "type": "string" + }, + { + "const": "ready", + "description": "Ready for `transfer/commit`.", + "type": "string" + }, + { + "const": "failed", + "description": "Stopped by an error; see `error`.", + "type": "string" + } + ] + }, + "TransferSelection": { + "description": "What one operation transfers.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read a file or tree out of the environment.", + "properties": { + "filter": { + "$ref": "#/definitions/TreeFilter" + }, + "kind": { + "const": "capture", + "type": "string" + }, + "source": { + "$ref": "#/definitions/EnvironmentPath" + }, + "specialFiles": { + "$ref": "#/definitions/SpecialFilePolicy" + }, + "symlinks": { + "$ref": "#/definitions/SymlinkPolicy" + } + }, + "required": [ + "kind", + "source" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Write a tree into the environment, replacing `destination` atomically\nat commit.", + "properties": { + "destination": { + "$ref": "#/definitions/EnvironmentPath" + }, + "kind": { + "const": "materialize", + "type": "string" + }, + "onExisting": { + "$ref": "#/definitions/TransferOnExisting" + } + }, + "required": [ + "kind", + "destination" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Write exactly one file with `fs/writeFile` semantics: staged,\natomic, an existing target keeps its permissions, an in-root symlink\nat `destination` is followed.", + "properties": { + "create": { + "$ref": "#/definitions/WriteCondition" + }, + "destination": { + "$ref": "#/definitions/EnvironmentPath" + }, + "executable": { + "type": "boolean" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ] + }, + "kind": { + "const": "writeFile", + "type": "string" + } + }, + "required": [ + "kind", + "destination" + ], + "type": "object" + } + ] + }, + "TransferSkip": { + "additionalProperties": false, + "description": "An entry a capture left out.", + "properties": { + "path": { + "anyOf": [ + { + "$ref": "#/definitions/TreePath" + } + ], + "description": "Absent when the entry's name is not UTF-8." + }, + "reason": { + "$ref": "#/definitions/TransferSkipReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "TransferSkipReason": { + "description": "Why a capture left an entry out.", + "oneOf": [ + { + "const": "symlink", + "description": "`symlinks: skip`, or a dangling link under `symlinks: expandInside`.", + "type": "string" + }, + { + "const": "specialFile", + "description": "`specialFiles: skip`.", + "type": "string" + }, + { + "const": "nonUtf8Name", + "description": "The name is not UTF-8; such entries are always skipped.", + "type": "string" + } + ] + }, + "TransferStatus": { + "additionalProperties": false, + "description": "The operation's status: the response of `transfer/begin`,\n`transfer/advance`, `transfer/getStatus`, `transfer/append`,\n`transfer/commit` and `transfer/abort`.", + "properties": { + "deadlineAtMs": { + "description": "When `maxDurationMs` runs out for an active operation.", + "format": "int64", + "type": "integer" + }, + "entryCount": { + "description": "Entries in the inventory so far.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "error": { + "anyOf": [ + { + "$ref": "#/definitions/ProtocolError" + } + ], + "description": "Why the operation failed; present exactly when `phase` is `failed`." + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + }, + "phase": { + "$ref": "#/definitions/TransferPhase" + }, + "receiptExpiresAtMs": { + "description": "When the receipt of a finished (complete, aborted or failed)\noperation stops being answered; afterwards calls are `notFound`\n(`expired`).", + "format": "int64", + "type": "integer" + }, + "reusedBytes": { + "description": "Content bytes satisfied from files already at the destination.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "skipped": { + "description": "The first\n`MAX_LISTED_SKIPS` entries a\ncapture left out under its policies.", + "items": { + "$ref": "#/definitions/TransferSkip" + }, + "type": "array" + }, + "skippedEntryCount": { + "description": "Total number of entries left out, listed or not; present whenever it\nis not zero.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "totalBytes": { + "description": "File bytes in the inventory so far.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "transferredBytes": { + "description": "Content bytes moved over the wire.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "operationId", + "phase", + "entryCount", + "totalBytes", + "transferredBytes", + "reusedBytes" + ], + "type": "object" + }, + "TreeContent": { + "description": "What a tree entry is: a scan entry or a transfer inventory entry.\nDigests identify raw file bytes, not paths or modes.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "directory", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "digest": { + "anyOf": [ + { + "$ref": "#/definitions/ContentDigest" + } + ], + "description": "Scans: present when `includeDigests` was set. Transfer\ninventories: always present; an appended file entry without it\nis `invalidParams`." + }, + "executable": { + "type": "boolean" + }, + "kind": { + "const": "file", + "type": "string" + }, + "sizeBytes": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "sizeBytes" + ], + "type": "object" + } + ] + }, + "TreeFilter": { + "additionalProperties": false, + "description": "Which entries a walk visits. The default visits everything except\nhidden and ignored entries.", + "properties": { + "exclude": { + "description": "Drop entries matching any pattern; a matching directory is pruned.", + "items": { + "type": "string" + }, + "type": "array" + }, + "ignoreCase": { + "description": "Match patterns case-insensitively.", + "type": "boolean" + }, + "include": { + "description": "Select entries matching any pattern; empty selects every entry.", + "items": { + "type": "string" + }, + "type": "array" + }, + "includeHidden": { + "description": "Visit entries whose name starts with `.`.", + "type": "boolean" + }, + "includeIgnored": { + "description": "Visit entries that ignore files exclude.", + "type": "boolean" + }, + "maxDepth": { + "description": "Deepest entry depth visited: the root's direct children are depth\n1, so `1` selects direct children only. Unbounded when absent.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "TreePath": { + "description": "A relative, normalized path below a selected root: segments joined by\n`/`, without empty, `.` or `..` segments. The single segment `.` names\nthe root itself. At most 4096 bytes. Used for scan, inventory and\ntransfer entries.", + "maxLength": 4096, + "minLength": 1, + "pattern": "^(\\.|(?!\\.\\.?(/|$))[^/\\u0000]+(/(?!\\.\\.?(/|$))[^/\\u0000]+)*)$", + "type": "string" + }, + "WriteCondition": { + "description": "When `fs/writeFile` may write.", + "oneOf": [ + { + "const": "always", + "description": "Create or replace.", + "type": "string" + }, + { + "const": "ifAbsent", + "description": "Only create; an existing entry is `failedPrecondition`\n(`createCondition`).", + "type": "string" + }, + { + "const": "ifPresent", + "description": "Only replace; a missing file is `failedPrecondition`\n(`createCondition`).", + "type": "string" + } + ] + }, + "WriteFileParams": { + "additionalProperties": false, + "description": "`fs/writeFile` params. The write is staged in the target's directory and\npublished by an atomic rename: readers see the old or the new content,\nnever a mix. An existing target keeps its permissions. A symlink at\n`path` whose target stays inside the root is followed like any path: the\nfinal target is replaced and the link itself is kept (a dangling link\ncreates its target, subject to `create`); a link that leaves the root is\n`permissionDenied` (`outsideRoot`). A special file is\n`failedPrecondition` (`notRegularFile`), a directory\n`failedPrecondition` (`isDirectory`). The parent of the final target\nmust exist.", + "properties": { + "create": { + "$ref": "#/definitions/WriteCondition" + }, + "data": { + "allOf": [ + { + "$ref": "#/definitions/ByteChunk" + } + ], + "description": "At most `limits.filesystem.maxWriteFileBytes`; larger files go\nthrough `transfer/*` with a `writeFile` selection." + }, + "executable": { + "description": "Set the execute bits (the read bits' pattern). Without it a new file\ngets the default mode and an existing file keeps its mode.", + "type": "boolean" + }, + "ifMatch": { + "anyOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ], + "description": "Write only when the current file's version equals this token;\notherwise `failedPrecondition` (`ifMatchFailed`)." + }, + "path": { + "$ref": "#/definitions/EnvironmentPath" + } + }, + "required": [ + "path", + "data" + ], + "type": "object" + }, + "WriteFileResponse": { + "additionalProperties": false, + "description": "`fs/writeFile` response.", + "properties": { + "version": { + "allOf": [ + { + "$ref": "#/definitions/FileVersion" + } + ], + "description": "Version of the written file." + } + }, + "required": [ + "version" + ], + "type": "object" + }, + "WriteProcessParams": { + "additionalProperties": false, + "description": "`process/write` params.", + "properties": { + "closeInput": { + "description": "Close the input after `data`: end of file on a pipe, `VEOF` on a\nPTY. Applied only when every byte of `data` was accepted.", + "type": "boolean" + }, + "data": { + "allOf": [ + { + "$ref": "#/definitions/ByteChunk" + } + ], + "description": "Bytes for the process's input pipe or PTY." + }, + "processId": { + "$ref": "#/definitions/ProcessId" + } + }, + "required": [ + "processId" + ], + "type": "object" + }, + "WriteProcessResponse": { + "additionalProperties": false, + "description": "`process/write` response.", + "properties": { + "acceptedBytes": { + "description": "Leading bytes of `data` queued to the process's writer. Fewer than\nsent (possibly 0) means the input queue is full: resend the rest\nlater.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "acceptedBytes" + ], + "type": "object" + }, + "WriteTransferContentParams": { + "additionalProperties": false, + "description": "`transfer/write` params: upload content for a missing digest. Chunks for\none digest arrive in order; an identical chunk may be retried.\nDifferent digests may be uploaded concurrently.", + "properties": { + "data": { + "allOf": [ + { + "$ref": "#/definitions/ByteChunk" + } + ], + "description": "At most `limits.filesystem.transfer.maxContentChunkBytes`." + }, + "digest": { + "$ref": "#/definitions/ContentDigest" + }, + "offset": { + "description": "Must equal the digest's next expected offset.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "operationId": { + "$ref": "#/definitions/TransferOperationId" + } + }, + "required": [ + "operationId", + "digest", + "offset", + "data" + ], + "type": "object" + }, + "WriteTransferContentResponse": { + "additionalProperties": false, + "description": "`transfer/write` response.", + "properties": { + "nextOffset": { + "description": "The digest's next expected offset; equal to its size once complete\nand verified.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "nextOffset" + ], + "type": "object" + } + }, + "description": "Every wire type of the data plane, protocol version 4.", + "title": "Lightspeed environment data plane" +} diff --git a/crates/environment-protocol/contract/registration.openrpc.json b/crates/environment-protocol/contract/registration.openrpc.json new file mode 100644 index 000000000..b5f1f91d6 --- /dev/null +++ b/crates/environment-protocol/contract/registration.openrpc.json @@ -0,0 +1,1404 @@ +{ + "components": { + "schemas": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "Admission": { + "description": "Whether this admission created the environment or rejoined it.", + "oneOf": [ + { + "const": "created", + "description": "First admission of this daemon identity: a new environment.", + "type": "string" + }, + { + "const": "reconnected", + "description": "A known identity reconnected to its environment.", + "type": "string" + } + ] + }, + "ArtifactSha256": { + "description": "`sha256` of a published archive: 64 lowercase hex digits.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "BuildFacts": { + "description": "What `lightspeed-envd --print-build` prints, as one JSON object.", + "properties": { + "gitSha": { + "type": "string" + }, + "name": { + "description": "Always `lightspeed-envd`.", + "type": "string" + }, + "protocolVersion": { + "description": "The binary's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "target": { + "description": "Rust target triple.", + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "name", + "version", + "gitSha", + "target", + "protocolVersion" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "power", + "description": "The controller's `power` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "ChallengeNonce": { + "description": "`nonce`: the gateway's 32 random bytes for one control connection.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "ChallengeParams": { + "description": "`registration/challenge` params.\n\n**Frozen.** This is the first frame of every control connection and the\nonly frame a daemon of any version must parse to learn it is stale: it\nonly ever gains fields, is decoded tolerantly (unknown members are\nignored), and a receiver reads `protocolVersion` with\n`challenge_protocol_version` before decoding anything else. The whole\nframe is\n`{\"jsonrpc\":\"2.0\",\"method\":\"registration/challenge\",\"params\":{\"protocolVersion\":N,\"nonce\":\"<64 hex>\"}}`.", + "properties": { + "nonce": { + "$ref": "#/components/schemas/ChallengeNonce" + }, + "protocolVersion": { + "description": "The gateway's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "nonce" + ], + "type": "object" + }, + "ClosedParams": { + "additionalProperties": false, + "description": "`registration/closed` params: the gateway ends the session for good and\ncloses the socket after this frame. The daemon exits or parks, as\nconfigured, and does not reconnect.", + "properties": { + "message": { + "anyOf": [ + { + "$ref": "#/components/schemas/DescriptiveText" + } + ] + }, + "reason": { + "$ref": "#/components/schemas/ClosedReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "ClosedReason": { + "description": "Why the gateway ended the session for good.", + "oneOf": [ + { + "const": "superseded", + "description": "Another process with this daemon identity connected; the newest\nconnection wins and this daemon exits with its own status naming the\nshared identity.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The identity's environment was closed; the identity is spent.", + "type": "string" + } + ] + }, + "ControlConnectionId": { + "description": "`controlConnectionId` the gateway assigns to one control connection.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DaemonId": { + "description": "`daemonId` the gateway assigns to an admitted daemon identity.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DaemonPublicKey": { + "description": "`daemonPublicKey`: a raw Ed25519 public key.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "DescriptiveText": { + "description": "A short human-readable text: 1 to 1024 bytes without control characters\nother than line feed and tab. Never parsed.", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DiscoveryArtifact": { + "description": "One published envd archive.", + "properties": { + "file": { + "description": "Archive file name.", + "type": "string" + }, + "sha256": { + "allOf": [ + { + "$ref": "#/components/schemas/ArtifactSha256" + } + ], + "description": "Digest of the archive's bytes." + }, + "url": { + "description": "Absolute download URL; absent (or null) when the archive is served\nnext to the document.", + "type": "string" + } + }, + "required": [ + "file", + "sha256" + ], + "type": "object" + }, + "DiscoveryDocument": { + "description": "The discovery document.", + "properties": { + "artifacts": { + "additionalProperties": { + "$ref": "#/components/schemas/DiscoveryArtifact" + }, + "description": "Archives by Rust target triple.", + "type": "object" + }, + "builtAtMs": { + "format": "int64", + "type": "integer" + }, + "channel": { + "description": "Release channel, such as `stable` or `snapshot`.", + "type": "string" + }, + "gitSha": { + "type": "string" + }, + "protocolVersion": { + "description": "The published envd's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "version": { + "description": "Release version of the published envd.", + "type": "string" + } + }, + "required": [ + "version", + "gitSha", + "protocolVersion", + "artifacts" + ], + "type": "object" + }, + "DisplayName": { + "description": "A display name: 1 to 128 bytes without control characters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "Ed25519Signature": { + "description": "`signature`: an Ed25519 signature over\n`signed_registration_message`.", + "pattern": "^[0-9a-f]{128}$", + "type": "string" + }, + "EnvironmentId": { + "description": "`environmentId` of a Lightspeed environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/components/schemas/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/components/schemas/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/components/schemas/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "GoodbyeParams": { + "additionalProperties": false, + "description": "`registration/goodbye` params: the daemon is stopping deliberately.", + "properties": { + "reason": { + "$ref": "#/components/schemas/GoodbyeReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "GoodbyeReason": { + "description": "Why the daemon is leaving.", + "oneOf": [ + { + "const": "shutdown", + "description": "`SIGTERM` or `SIGINT`.", + "type": "string" + } + ] + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "Metadata": { + "additionalProperties": { + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "maxProperties": 32, + "propertyNames": { + "maxLength": 64, + "minLength": 1, + "not": { + "pattern": "^lightspeed\\." + }, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "OpenStreamParams": { + "additionalProperties": false, + "description": "`registration/openStream` params: the gateway asks the daemon to dial\none stream.", + "properties": { + "stream": { + "$ref": "#/components/schemas/StreamSpec" + }, + "token": { + "allOf": [ + { + "$ref": "#/components/schemas/SecretString" + } + ], + "description": "One-time bearer token for the stream dial; short-lived." + } + }, + "required": [ + "token", + "stream" + ], + "type": "object" + }, + "OpenStreamResponse": { + "additionalProperties": false, + "description": "`registration/openStream` response, sent once the stream socket is\nupgraded and served. A failed dial answers `unavailable`; the gateway\nfails the waiting route at once.", + "type": "object" + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/components/schemas/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "RegisterParams": { + "additionalProperties": false, + "description": "`registration/register` params. `registrationKey`, `displayName` and\n`metadata` are read only at a daemon identity's first admission; a\ndaemon sends the key until it is admitted once and never afterwards.\n\nThe gateway decodes them with `RegisterParams::decode`: build facts,\na display name or metadata out of bounds refuse admission with\n`invalidRequest`; anything else that does not decode is\n`invalidParams`.", + "properties": { + "daemonPublicKey": { + "$ref": "#/components/schemas/DaemonPublicKey" + }, + "displayName": { + "anyOf": [ + { + "$ref": "#/components/schemas/DisplayName" + } + ] + }, + "implementation": { + "$ref": "#/components/schemas/ImplementationInfo" + }, + "metadata": { + "$ref": "#/components/schemas/Metadata" + }, + "protocolVersion": { + "description": "The daemon's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "registrationKey": { + "anyOf": [ + { + "$ref": "#/components/schemas/SecretString" + } + ] + }, + "signature": { + "allOf": [ + { + "$ref": "#/components/schemas/Ed25519Signature" + } + ], + "description": "Over `signed_registration_message` of the challenge nonce and the\ncanonical origin of the URL the daemon dialed." + } + }, + "required": [ + "protocolVersion", + "daemonPublicKey", + "signature", + "implementation" + ], + "type": "object" + }, + "RegisterResponse": { + "additionalProperties": false, + "description": "`registration/register` response: the daemon is admitted.", + "properties": { + "heartbeatIntervalMs": { + "description": "How often the gateway pings; see the liveness rule.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "receipt": { + "$ref": "#/components/schemas/RegistrationReceipt" + } + }, + "required": [ + "receipt", + "heartbeatIntervalMs" + ], + "type": "object" + }, + "RegistrationIdentityMode": { + "description": "Registration-key policy for the daemon's environment.", + "oneOf": [ + { + "const": "persistent", + "description": "The environment outlives disconnects.", + "type": "string" + }, + { + "const": "ephemeral", + "description": "The environment closes after it stays disconnected past its grace.", + "type": "string" + } + ] + }, + "RegistrationReceipt": { + "additionalProperties": false, + "description": "What the daemon learns about the environment it serves. Written to the\nreceipt file as is; contains no secret.", + "properties": { + "admission": { + "$ref": "#/components/schemas/Admission" + }, + "controlConnectionId": { + "allOf": [ + { + "$ref": "#/components/schemas/ControlConnectionId" + } + ], + "description": "The gateway's id for this control connection." + }, + "daemonId": { + "$ref": "#/components/schemas/DaemonId" + }, + "environmentId": { + "$ref": "#/components/schemas/EnvironmentId" + }, + "identityMode": { + "$ref": "#/components/schemas/RegistrationIdentityMode" + } + }, + "required": [ + "environmentId", + "daemonId", + "controlConnectionId", + "identityMode", + "admission" + ], + "type": "object" + }, + "SecretString": { + "description": "A secret value carried on the wire, such as a `secretEnv` value, a\nregistration key or a stream token. It serializes as a plain string;\n`Debug` is redacted so structured logging never prints it.", + "type": "string" + }, + "StreamSpec": { + "description": "What the dialed stream carries.", + "oneOf": [ + { + "additionalProperties": false, + "description": "The data plane, served by the daemon as on any data connection.", + "properties": { + "kind": { + "const": "dataProtocol", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + } + } + }, + "info": { + "description": "JSON-RPC 2.0 over WebSocket text frames of at most 16777216 bytes. Exact-match protocolVersion 4. Errors carry code, message and data.kind; see x-lightspeed-error-kinds.", + "title": "Lightspeed environment registration channel", + "version": "4" + }, + "methods": [ + { + "name": "registration/register", + "paramStructure": "by-name", + "params": [ + { + "name": "daemonPublicKey", + "required": true, + "schema": { + "$ref": "#/components/schemas/DaemonPublicKey" + } + }, + { + "name": "displayName", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/DisplayName" + } + ] + } + }, + { + "name": "implementation", + "required": true, + "schema": { + "$ref": "#/components/schemas/ImplementationInfo" + } + }, + { + "name": "metadata", + "required": false, + "schema": { + "$ref": "#/components/schemas/Metadata" + } + }, + { + "name": "protocolVersion", + "required": true, + "schema": { + "description": "The daemon's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + { + "name": "registrationKey", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/SecretString" + } + ] + } + }, + { + "name": "signature", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/Ed25519Signature" + } + ], + "description": "Over `signed_registration_message` of the challenge nonce and the\ncanonical origin of the URL the daemon dialed." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/RegisterResponse" + } + }, + "summary": "Prove the daemon identity and be admitted to its environment.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "RegisterParams", + "x-lightspeed-result-type": "RegisterResponse" + }, + { + "name": "registration/openStream", + "paramStructure": "by-name", + "params": [ + { + "name": "stream", + "required": true, + "schema": { + "$ref": "#/components/schemas/StreamSpec" + } + }, + { + "name": "token", + "required": true, + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/SecretString" + } + ], + "description": "One-time bearer token for the stream dial; short-lived." + } + } + ], + "result": { + "name": "result", + "schema": { + "$ref": "#/components/schemas/OpenStreamResponse" + } + }, + "summary": "Ask the daemon to dial one reverse stream.", + "x-lightspeed-direction": "serverToClient", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "request", + "x-lightspeed-params-type": "OpenStreamParams", + "x-lightspeed-result-type": "OpenStreamResponse" + }, + { + "name": "registration/challenge", + "paramStructure": "by-name", + "params": [ + { + "name": "nonce", + "required": true, + "schema": { + "$ref": "#/components/schemas/ChallengeNonce" + } + }, + { + "name": "protocolVersion", + "required": true, + "schema": { + "description": "The gateway's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + } + ], + "summary": "The frozen first frame: the gateway's version and nonce.", + "x-lightspeed-direction": "serverToClient", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "notification", + "x-lightspeed-params-type": "ChallengeParams" + }, + { + "name": "registration/goodbye", + "paramStructure": "by-name", + "params": [ + { + "name": "reason", + "required": true, + "schema": { + "$ref": "#/components/schemas/GoodbyeReason" + } + } + ], + "summary": "The daemon is stopping deliberately.", + "x-lightspeed-direction": "clientToServer", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "notification", + "x-lightspeed-params-type": "GoodbyeParams" + }, + { + "name": "registration/closed", + "paramStructure": "by-name", + "params": [ + { + "name": "message", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/DescriptiveText" + } + ] + } + }, + { + "name": "reason", + "required": true, + "schema": { + "$ref": "#/components/schemas/ClosedReason" + } + } + ], + "summary": "The gateway ends the session for good.", + "x-lightspeed-direction": "serverToClient", + "x-lightspeed-family": "registration", + "x-lightspeed-kind": "notification", + "x-lightspeed-params-type": "ClosedParams" + } + ], + "openrpc": "1.3.2", + "x-lightspeed-bootstrap": { + "buildFacts": { + "$ref": "#/components/schemas/BuildFacts" + }, + "discoveryDocument": { + "$ref": "#/components/schemas/DiscoveryDocument" + } + }, + "x-lightspeed-error": { + "$ref": "#/components/schemas/ProtocolError" + }, + "x-lightspeed-error-kinds": [ + { + "code": -32700, + "kind": "parseError", + "retryable": false + }, + { + "code": -32600, + "kind": "invalidRequest", + "retryable": false + }, + { + "code": -32601, + "kind": "methodNotFound", + "retryable": false + }, + { + "code": -32602, + "kind": "invalidParams", + "retryable": false + }, + { + "code": -32002, + "kind": "notInitialized", + "retryable": false + }, + { + "code": -32005, + "kind": "versionMismatch", + "retryable": false + }, + { + "code": -32004, + "kind": "notFound", + "retryable": false + }, + { + "code": -32009, + "kind": "alreadyExists", + "retryable": false + }, + { + "code": -32007, + "kind": "failedPrecondition", + "retryable": false + }, + { + "code": -32003, + "kind": "permissionDenied", + "retryable": false + }, + { + "code": -32006, + "kind": "capabilityUnavailable", + "retryable": false + }, + { + "code": -32008, + "kind": "busy", + "retryable": true + }, + { + "code": -32015, + "kind": "deadlineExceeded", + "retryable": false + }, + { + "code": -32016, + "kind": "expired", + "retryable": false + }, + { + "code": -32017, + "kind": "aborted", + "retryable": true + }, + { + "code": -32013, + "kind": "payloadTooLarge", + "retryable": false + }, + { + "code": -32018, + "kind": "processFailed", + "retryable": false + }, + { + "code": -32019, + "kind": "unavailable", + "retryable": true + }, + { + "code": -32603, + "kind": "internal", + "retryable": false + } + ], + "x-lightspeed-plane": "registration" +} diff --git a/crates/environment-protocol/contract/registration.schema.json b/crates/environment-protocol/contract/registration.schema.json new file mode 100644 index 000000000..826980f46 --- /dev/null +++ b/crates/environment-protocol/contract/registration.schema.json @@ -0,0 +1,1101 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "AbortReason": { + "description": "Why an operation was `aborted`.", + "oneOf": [ + { + "const": "cancelled", + "description": "`$/cancelRequest` named the request.", + "type": "string" + }, + { + "const": "sourceChanged", + "description": "A file or directory changed while it was being read.", + "type": "string" + } + ] + }, + "Admission": { + "description": "Whether this admission created the environment or rejoined it.", + "oneOf": [ + { + "const": "created", + "description": "First admission of this daemon identity: a new environment.", + "type": "string" + }, + { + "const": "reconnected", + "description": "A known identity reconnected to its environment.", + "type": "string" + } + ] + }, + "ArtifactSha256": { + "description": "`sha256` of a published archive: 64 lowercase hex digits.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "BuildFacts": { + "description": "What `lightspeed-envd --print-build` prints, as one JSON object.", + "properties": { + "gitSha": { + "type": "string" + }, + "name": { + "description": "Always `lightspeed-envd`.", + "type": "string" + }, + "protocolVersion": { + "description": "The binary's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "target": { + "description": "Rust target triple.", + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "name", + "version", + "gitSha", + "target", + "protocolVersion" + ], + "type": "object" + }, + "BusyReason": { + "description": "What limit a `busy` request hit.", + "oneOf": [ + { + "const": "requestLimit", + "description": "The connection's concurrent request limit.", + "type": "string" + }, + { + "const": "transferCapacity", + "description": "The daemon's limit on active transfer operations.", + "type": "string" + }, + { + "const": "rateLimited", + "description": "A rate limit on the caller's identity.", + "type": "string" + }, + { + "const": "capacityExhausted", + "description": "A registration key's limit on active environments.", + "type": "string" + } + ] + }, + "Capability": { + "description": "A capability a request needed and the endpoint lacks.", + "oneOf": [ + { + "const": "filesystem", + "description": "The `filesystem` family.", + "type": "string" + }, + { + "const": "filesystemWrite", + "description": "`capabilities.filesystem.write`: the filesystem is writable.", + "type": "string" + }, + { + "const": "process", + "description": "The `process` family.", + "type": "string" + }, + { + "const": "processPty", + "description": "`capabilities.process.pty`.", + "type": "string" + }, + { + "const": "job", + "description": "The `job` family.", + "type": "string" + }, + { + "const": "power", + "description": "The controller's `power` capability.", + "type": "string" + }, + { + "const": "ingress", + "description": "The controller's `ingress` capability.", + "type": "string" + } + ] + }, + "ChallengeNonce": { + "description": "`nonce`: the gateway's 32 random bytes for one control connection.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "ChallengeParams": { + "description": "`registration/challenge` params.\n\n**Frozen.** This is the first frame of every control connection and the\nonly frame a daemon of any version must parse to learn it is stale: it\nonly ever gains fields, is decoded tolerantly (unknown members are\nignored), and a receiver reads `protocolVersion` with\n`challenge_protocol_version` before decoding anything else. The whole\nframe is\n`{\"jsonrpc\":\"2.0\",\"method\":\"registration/challenge\",\"params\":{\"protocolVersion\":N,\"nonce\":\"<64 hex>\"}}`.", + "properties": { + "nonce": { + "$ref": "#/definitions/ChallengeNonce" + }, + "protocolVersion": { + "description": "The gateway's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "protocolVersion", + "nonce" + ], + "type": "object" + }, + "ClosedParams": { + "additionalProperties": false, + "description": "`registration/closed` params: the gateway ends the session for good and\ncloses the socket after this frame. The daemon exits or parks, as\nconfigured, and does not reconnect.", + "properties": { + "message": { + "anyOf": [ + { + "$ref": "#/definitions/DescriptiveText" + } + ] + }, + "reason": { + "$ref": "#/definitions/ClosedReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "ClosedReason": { + "description": "Why the gateway ended the session for good.", + "oneOf": [ + { + "const": "superseded", + "description": "Another process with this daemon identity connected; the newest\nconnection wins and this daemon exits with its own status naming the\nshared identity.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The identity's environment was closed; the identity is spent.", + "type": "string" + } + ] + }, + "ControlConnectionId": { + "description": "`controlConnectionId` the gateway assigns to one control connection.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DaemonId": { + "description": "`daemonId` the gateway assigns to an admitted daemon identity.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "DaemonPublicKey": { + "description": "`daemonPublicKey`: a raw Ed25519 public key.", + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + "DescriptiveText": { + "description": "A short human-readable text: 1 to 1024 bytes without control characters\nother than line feed and tab. Never parsed.", + "maxLength": 1024, + "minLength": 1, + "pattern": "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "DiscoveryArtifact": { + "description": "One published envd archive.", + "properties": { + "file": { + "description": "Archive file name.", + "type": "string" + }, + "sha256": { + "allOf": [ + { + "$ref": "#/definitions/ArtifactSha256" + } + ], + "description": "Digest of the archive's bytes." + }, + "url": { + "description": "Absolute download URL; absent (or null) when the archive is served\nnext to the document.", + "type": "string" + } + }, + "required": [ + "file", + "sha256" + ], + "type": "object" + }, + "DiscoveryDocument": { + "description": "The discovery document.", + "properties": { + "artifacts": { + "additionalProperties": { + "$ref": "#/definitions/DiscoveryArtifact" + }, + "description": "Archives by Rust target triple.", + "type": "object" + }, + "builtAtMs": { + "format": "int64", + "type": "integer" + }, + "channel": { + "description": "Release channel, such as `stable` or `snapshot`.", + "type": "string" + }, + "gitSha": { + "type": "string" + }, + "protocolVersion": { + "description": "The published envd's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "version": { + "description": "Release version of the published envd.", + "type": "string" + } + }, + "required": [ + "version", + "gitSha", + "protocolVersion", + "artifacts" + ], + "type": "object" + }, + "DisplayName": { + "description": "A display name: 1 to 128 bytes without control characters.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "Ed25519Signature": { + "description": "`signature`: an Ed25519 signature over\n`signed_registration_message`.", + "pattern": "^[0-9a-f]{128}$", + "type": "string" + }, + "EnvironmentId": { + "description": "`environmentId` of a Lightspeed environment.", + "maxLength": 128, + "minLength": 1, + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$", + "type": "string" + }, + "ErrorData": { + "description": "The `data` member of an error object: the kind and its typed fields.\n\nKinds without fields are empty struct variants rather than unit\nvariants: serde does not apply `deny_unknown_fields` to the unit\nvariants of an internally tagged enum, and every kind must refuse\nunknown members.", + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "parseError", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "invalidRequest", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "methodNotFound", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "ceiling": { + "description": "The advertised ceiling the value exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "field": { + "description": "Wire path of the offending field, such as `limits.maxDurationMs`.", + "type": "string" + }, + "kind": { + "const": "invalidParams", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notInitialized", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "expected": { + "description": "The version of the peer that sent this error.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "versionMismatch", + "type": "string" + }, + "received": { + "description": "The version the requester stated.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind", + "expected", + "received" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "notFound", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/NotFoundReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "alreadyExists", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "failedPrecondition", + "type": "string" + }, + "reason": { + "$ref": "#/definitions/PreconditionReason" + } + }, + "required": [ + "kind", + "reason" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "permissionDenied", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/PermissionReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "capability": { + "$ref": "#/definitions/Capability" + }, + "kind": { + "const": "capabilityUnavailable", + "type": "string" + } + }, + "required": [ + "kind", + "capability" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "busy", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/BusyReason" + } + ] + }, + "retryAfterMs": { + "description": "Earliest useful retry, when the server knows it.", + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "deadlineExceeded", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "expired", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "aborted", + "type": "string" + }, + "reason": { + "anyOf": [ + { + "$ref": "#/definitions/AbortReason" + } + ] + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "payloadTooLarge", + "type": "string" + }, + "limitBytes": { + "description": "The byte limit the payload exceeded.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "limitEntries": { + "description": "The entry-count limit the result exceeded.", + "format": "uint32", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "processFailed", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "unavailable", + "type": "string" + }, + "retryAfterMs": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "kind": { + "const": "internal", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "GoodbyeParams": { + "additionalProperties": false, + "description": "`registration/goodbye` params: the daemon is stopping deliberately.", + "properties": { + "reason": { + "$ref": "#/definitions/GoodbyeReason" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "GoodbyeReason": { + "description": "Why the daemon is leaving.", + "oneOf": [ + { + "const": "shutdown", + "description": "`SIGTERM` or `SIGINT`.", + "type": "string" + } + ] + }, + "ImplementationInfo": { + "additionalProperties": false, + "description": "What a peer says about its own build: `clientInfo`, `serverInfo` and a\ndaemon's registration `implementation`. Descriptive only; the protocol\nversion alone decides whether two peers can talk, and nothing here\nselects, authenticates or gates. Every field is 1 to 256 bytes without control characters,\nchecked on decode and by every constructor, so a value that would be\nrefused can never be built or sent.", + "properties": { + "gitSha": { + "description": "Full or abbreviated git commit of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "name": { + "description": "Product name, such as `lightspeed-envd`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "target": { + "description": "Rust target triple of the build.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "version": { + "description": "Release version, such as `0.5.0`.", + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "Metadata": { + "additionalProperties": { + "maxLength": 256, + "minLength": 1, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$", + "type": "string" + }, + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "maxProperties": 32, + "propertyNames": { + "maxLength": 64, + "minLength": 1, + "not": { + "pattern": "^lightspeed\\." + }, + "pattern": "^[^\\u0000-\\u001F\\u007F-\\u009F]+$" + }, + "type": "object" + }, + "NotFoundReason": { + "description": "Why a handle, path or identity was not found.", + "oneOf": [ + { + "const": "expired", + "description": "Its retention window ended; a tombstone remembers it.", + "type": "string" + }, + { + "const": "daemonRestarted", + "description": "It belonged to an earlier daemon instance (see `daemonInstanceId`).", + "type": "string" + }, + { + "const": "unknownDaemon", + "description": "A daemon identity the gateway has never admitted presented no\nregistration key.", + "type": "string" + } + ] + }, + "OpenStreamParams": { + "additionalProperties": false, + "description": "`registration/openStream` params: the gateway asks the daemon to dial\none stream.", + "properties": { + "stream": { + "$ref": "#/definitions/StreamSpec" + }, + "token": { + "allOf": [ + { + "$ref": "#/definitions/SecretString" + } + ], + "description": "One-time bearer token for the stream dial; short-lived." + } + }, + "required": [ + "token", + "stream" + ], + "type": "object" + }, + "OpenStreamResponse": { + "additionalProperties": false, + "description": "`registration/openStream` response, sent once the stream socket is\nupgraded and served. A failed dial answers `unavailable`; the gateway\nfails the waiting route at once.", + "type": "object" + }, + "PermissionReason": { + "description": "Why a `permissionDenied` request was refused.", + "oneOf": [ + { + "const": "outsideRoot", + "description": "The path, or a symlink it passes through, resolves outside the\ndaemon's filesystem root.", + "type": "string" + }, + { + "const": "daemonState", + "description": "The path names or contains the daemon's state directory.", + "type": "string" + }, + { + "const": "filePermission", + "description": "The operating system refused access.", + "type": "string" + }, + { + "const": "invalidSignature", + "description": "The registration signature does not verify.", + "type": "string" + }, + { + "const": "invalidRegistrationKey", + "description": "The registration key is unknown.", + "type": "string" + }, + { + "const": "registrationKeyRevoked", + "description": "The registration key was revoked.", + "type": "string" + } + ] + }, + "PreconditionReason": { + "description": "The state that forbids a `failedPrecondition` request.", + "oneOf": [ + { + "const": "inputClosed", + "description": "The process has no open input: started without `pipe` or `pty`,\ninput already closed, or exited.", + "type": "string" + }, + { + "const": "ifMatchFailed", + "description": "The file's current `version` differs from `ifMatch`.", + "type": "string" + }, + { + "const": "createCondition", + "description": "`create: ifAbsent` met an existing file, or `create: ifPresent` a\nmissing one.", + "type": "string" + }, + { + "const": "notEmpty", + "description": "The directory is not empty and `recursive` was not set.", + "type": "string" + }, + { + "const": "isDirectory", + "description": "The path names a directory where a file is required.", + "type": "string" + }, + { + "const": "notDirectory", + "description": "A path component or the path names a non-directory where a directory\nis required.", + "type": "string" + }, + { + "const": "notRegularFile", + "description": "The path names a device, FIFO, socket or other non-regular file.", + "type": "string" + }, + { + "const": "symlinkLoop", + "description": "A symlink chain loops.", + "type": "string" + }, + { + "const": "symlinkRejected", + "description": "A capture met a symlink that leaves the selection, or one the\nselection's symlink policy rejects.", + "type": "string" + }, + { + "const": "filesystemRoot", + "description": "The filesystem root itself cannot be replaced or removed.", + "type": "string" + }, + { + "const": "crossDevice", + "description": "Rename across filesystems.", + "type": "string" + }, + { + "const": "noSpace", + "description": "No space left on the device.", + "type": "string" + }, + { + "const": "jobIdInUse", + "description": "A job id is in use by a job with another `requestId` or spec; choose\na new job id.", + "type": "string" + }, + { + "const": "operationIdInUse", + "description": "A transfer `operationId` is in use for different input.", + "type": "string" + }, + { + "const": "transferPhase", + "description": "The transfer action is not valid in the operation's current phase.", + "type": "string" + }, + { + "const": "transferOffset", + "description": "A transfer write's `offset` is not the digest's next expected offset.", + "type": "string" + }, + { + "const": "digestMismatch", + "description": "Uploaded bytes do not hash to their digest.", + "type": "string" + }, + { + "const": "environmentClosed", + "description": "The environment is closed; its identity or target is spent.", + "type": "string" + }, + { + "const": "identityInUse", + "description": "The daemon identity is bound to another universe.", + "type": "string" + }, + { + "const": "ownerMismatch", + "description": "The target belongs to another owner.", + "type": "string" + }, + { + "const": "templateDeprecated", + "description": "The template is deprecated and creates no new targets.", + "type": "string" + }, + { + "const": "ingressUnsupported", + "description": "`ingress/ensure` on a target whose template has no public ingress.", + "type": "string" + }, + { + "const": "targetNotActive", + "description": "A power change on a target whose `phase` is not `active`\n(provisioning, closing, closed or failed).", + "type": "string" + } + ] + }, + "ProtocolError": { + "additionalProperties": false, + "description": "A protocol error: one kind, its typed data and a message.\n\nSerializes as an `ErrorObject`. Displays as `{kind}: {message}`.", + "properties": { + "code": { + "description": "Fixed by `data.kind`; see `ProtocolErrorKind::code`.", + "format": "int32", + "type": "integer" + }, + "data": { + "$ref": "#/definitions/ErrorData" + }, + "message": { + "description": "Human-readable description. Never parsed.", + "type": "string" + } + }, + "required": [ + "code", + "message", + "data" + ], + "type": "object" + }, + "RegisterParams": { + "additionalProperties": false, + "description": "`registration/register` params. `registrationKey`, `displayName` and\n`metadata` are read only at a daemon identity's first admission; a\ndaemon sends the key until it is admitted once and never afterwards.\n\nThe gateway decodes them with `RegisterParams::decode`: build facts,\na display name or metadata out of bounds refuse admission with\n`invalidRequest`; anything else that does not decode is\n`invalidParams`.", + "properties": { + "daemonPublicKey": { + "$ref": "#/definitions/DaemonPublicKey" + }, + "displayName": { + "anyOf": [ + { + "$ref": "#/definitions/DisplayName" + } + ] + }, + "implementation": { + "$ref": "#/definitions/ImplementationInfo" + }, + "metadata": { + "$ref": "#/definitions/Metadata" + }, + "protocolVersion": { + "description": "The daemon's `ENVD_PROTOCOL_VERSION`.", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "registrationKey": { + "anyOf": [ + { + "$ref": "#/definitions/SecretString" + } + ] + }, + "signature": { + "allOf": [ + { + "$ref": "#/definitions/Ed25519Signature" + } + ], + "description": "Over `signed_registration_message` of the challenge nonce and the\ncanonical origin of the URL the daemon dialed." + } + }, + "required": [ + "protocolVersion", + "daemonPublicKey", + "signature", + "implementation" + ], + "type": "object" + }, + "RegisterResponse": { + "additionalProperties": false, + "description": "`registration/register` response: the daemon is admitted.", + "properties": { + "heartbeatIntervalMs": { + "description": "How often the gateway pings; see the liveness rule.", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "receipt": { + "$ref": "#/definitions/RegistrationReceipt" + } + }, + "required": [ + "receipt", + "heartbeatIntervalMs" + ], + "type": "object" + }, + "RegistrationIdentityMode": { + "description": "Registration-key policy for the daemon's environment.", + "oneOf": [ + { + "const": "persistent", + "description": "The environment outlives disconnects.", + "type": "string" + }, + { + "const": "ephemeral", + "description": "The environment closes after it stays disconnected past its grace.", + "type": "string" + } + ] + }, + "RegistrationReceipt": { + "additionalProperties": false, + "description": "What the daemon learns about the environment it serves. Written to the\nreceipt file as is; contains no secret.", + "properties": { + "admission": { + "$ref": "#/definitions/Admission" + }, + "controlConnectionId": { + "allOf": [ + { + "$ref": "#/definitions/ControlConnectionId" + } + ], + "description": "The gateway's id for this control connection." + }, + "daemonId": { + "$ref": "#/definitions/DaemonId" + }, + "environmentId": { + "$ref": "#/definitions/EnvironmentId" + }, + "identityMode": { + "$ref": "#/definitions/RegistrationIdentityMode" + } + }, + "required": [ + "environmentId", + "daemonId", + "controlConnectionId", + "identityMode", + "admission" + ], + "type": "object" + }, + "SecretString": { + "description": "A secret value carried on the wire, such as a `secretEnv` value, a\nregistration key or a stream token. It serializes as a plain string;\n`Debug` is redacted so structured logging never prints it.", + "type": "string" + }, + "StreamSpec": { + "description": "What the dialed stream carries.", + "oneOf": [ + { + "additionalProperties": false, + "description": "The data plane, served by the daemon as on any data connection.", + "properties": { + "kind": { + "const": "dataProtocol", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + } + }, + "description": "Every wire type of the registration plane, protocol version 4.", + "title": "Lightspeed environment registration channel", + "x-lightspeed-bootstrap": { + "buildFacts": { + "$ref": "#/definitions/BuildFacts" + }, + "discoveryDocument": { + "$ref": "#/definitions/DiscoveryDocument" + } + } +} diff --git a/crates/environment-protocol/fixtures/bootstrap/build-facts.json b/crates/environment-protocol/fixtures/bootstrap/build-facts.json new file mode 100644 index 000000000..33e5ecbcb --- /dev/null +++ b/crates/environment-protocol/fixtures/bootstrap/build-facts.json @@ -0,0 +1,7 @@ +{ + "name": "lightspeed-envd", + "version": "0.5.0", + "gitSha": "86f6adf3", + "target": "x86_64-unknown-linux-musl", + "protocolVersion": 4 +} diff --git a/crates/environment-protocol/fixtures/bootstrap/discovery-document.json b/crates/environment-protocol/fixtures/bootstrap/discovery-document.json new file mode 100644 index 000000000..5113877fb --- /dev/null +++ b/crates/environment-protocol/fixtures/bootstrap/discovery-document.json @@ -0,0 +1,14 @@ +{ + "version": "0.5.0", + "gitSha": "86f6adf3", + "channel": "stable", + "protocolVersion": 4, + "builtAtMs": 1761400000000, + "artifacts": { + "x86_64-unknown-linux-musl": { + "file": "lightspeed-envd-x86_64-unknown-linux-musl.tar.gz", + "sha256": "abababababababababababababababababababababababababababababababab", + "url": "https://releases.example.com/lightspeed-envd-x86_64-unknown-linux-musl.tar.gz" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/ingress.ensure.json b/crates/environment-protocol/fixtures/controller/ingress.ensure.json new file mode 100644 index 000000000..524079bee --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/ingress.ensure.json @@ -0,0 +1,33 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "targetId": "ls-3f2a" + } + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "phase": "active", + "power": "running", + "ingress": { + "endpoint": "https://env-1.apps.example.com" + } + } + }, + "error": { + "code": -32006, + "message": "this provider serves no public ingress", + "data": { + "kind": "capabilityUnavailable", + "capability": "ingress" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/ingress.remove.json b/crates/environment-protocol/fixtures/controller/ingress.remove.json new file mode 100644 index 000000000..5486656a9 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/ingress.remove.json @@ -0,0 +1,29 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "targetId": "ls-3f2a" + } + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "phase": "active", + "power": "running" + } + }, + "error": { + "code": -32004, + "message": "target ls-3f2a no longer exists", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/initialize.json b/crates/environment-protocol/fixtures/controller/initialize.json new file mode 100644 index 000000000..6492d8b24 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/initialize.json @@ -0,0 +1,30 @@ +{ + "params": { + "protocolVersion": 4, + "clientInfo": { + "name": "lightspeed-runtime", + "version": "0.5.0" + } + }, + "response": { + "protocolVersion": 4, + "plane": "controller", + "serverInfo": { + "name": "lightspeed-provider-incus", + "version": "0.5.0" + }, + "capabilities": { + "power": true, + "ingress": true + } + }, + "error": { + "code": -32005, + "message": "protocol version 5 is not supported; this peer speaks 4", + "data": { + "kind": "versionMismatch", + "expected": 4, + "received": 5 + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.close.json b/crates/environment-protocol/fixtures/controller/target.close.json new file mode 100644 index 000000000..66c8695a0 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.close.json @@ -0,0 +1,27 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "environmentId": "env_1" + } + } + }, + "response": { + "target": { + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "phase": "closed" + } + }, + "error": { + "code": -32019, + "message": "the Incus API is unreachable", + "data": { + "kind": "unavailable", + "retryAfterMs": 5000 + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.create.json b/crates/environment-protocol/fixtures/controller/target.create.json new file mode 100644 index 000000000..6ea8b5539 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.create.json @@ -0,0 +1,33 @@ +{ + "params": { + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "templateId": "ubuntu-24.04-small", + "displayName": "build box", + "idle": { + "pauseAfterMs": 600000, + "stopAfterMs": 3600000 + } + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "phase": "provisioning", + "message": "pulling image" + } + }, + "error": { + "code": -32007, + "message": "template ubuntu-24.04-small is deprecated", + "data": { + "kind": "failedPrecondition", + "reason": "templateDeprecated" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.get.json b/crates/environment-protocol/fixtures/controller/target.get.json new file mode 100644 index 000000000..19ed2292b --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.get.json @@ -0,0 +1,32 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "targetId": "ls-3f2a" + } + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "phase": "active", + "power": "running", + "ingress": { + "endpoint": "https://env-1.apps.example.com" + } + } + }, + "error": { + "code": -32004, + "message": "target ls-3f2a no longer exists", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/target.setPower.json b/crates/environment-protocol/fixtures/controller/target.setPower.json new file mode 100644 index 000000000..1601082b7 --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/target.setPower.json @@ -0,0 +1,31 @@ +{ + "params": { + "target": { + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "targetId": "ls-3f2a" + }, + "power": "paused" + }, + "response": { + "target": { + "targetId": "ls-3f2a", + "owner": { + "universeId": "u1", + "environmentId": "env_1" + }, + "phase": "active", + "power": "paused" + } + }, + "error": { + "code": -32007, + "message": "target ls-3f2a belongs to another environment", + "data": { + "kind": "failedPrecondition", + "reason": "ownerMismatch" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller/template.list.json b/crates/environment-protocol/fixtures/controller/template.list.json new file mode 100644 index 000000000..69d4f497c --- /dev/null +++ b/crates/environment-protocol/fixtures/controller/template.list.json @@ -0,0 +1,40 @@ +{ + "params": { + "universeId": "u1" + }, + "response": { + "templates": [ + { + "templateId": "ubuntu-24.04-small", + "displayName": "Ubuntu 24.04 (small)", + "description": "2 vCPU, 4 GiB", + "powerStates": [ + "running", + "paused", + "stopped" + ], + "publicIngress": true, + "idle": { + "pauseAfterMs": 900000, + "stopAfterMs": 7200000 + }, + "deprecated": true, + "resources": { + "vcpus": 2, + "memoryBytes": 4294967296, + "diskBytes": 21474836480 + }, + "metadata": { + "imageFingerprint": "a1b2c3" + } + } + ] + }, + "error": { + "code": -32019, + "message": "the Incus API is unreachable", + "data": { + "kind": "unavailable" + } + } +} diff --git a/crates/environment-protocol/fixtures/controller_create_target_params.json b/crates/environment-protocol/fixtures/controller_create_target_params.json deleted file mode 100644 index 2d2ffa85c..000000000 --- a/crates/environment-protocol/fixtures/controller_create_target_params.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "binding": { - "bindingId": "primary", - "universeId": "universe-1" - }, - "environmentId": "environment-1", - "incarnationId": "incarnation-1", - "requestId": "request-1", - "templateId": "lightspeed-dev-v1" -} diff --git a/crates/environment-protocol/fixtures/controller_create_target_response.json b/crates/environment-protocol/fixtures/controller_create_target_response.json deleted file mode 100644 index 10b9ddf56..000000000 --- a/crates/environment-protocol/fixtures/controller_create_target_response.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "target": { - "capabilities": { - "filesystemRead": true, - "filesystemSearch": true, - "filesystemGlob": true, - "filesystemRangedRead": true, - "filesystemWrite": true, - "jobCancel": true, - "jobDependencies": true, - "jobList": true, - "jobRead": true, - "jobQueueKeys": true, - "jobStart": true, - "jobWaitHint": false, - "network": true, - "processOutputNotifications": true, - "processOutputPolling": true, - "processPty": false, - "processStart": true, - "processStdin": true, - "processTerminate": true - }, - "defaultCwd": "/workspace", - "displayName": "lightspeed dev", - "metadata": { - "provider": "smolvm" - }, - "scope": { - "sessionId": "sandbox-123", - "type": "session" - }, - "status": "ready", - "targetId": "sandbox-123" - } -} diff --git a/crates/environment-protocol/fixtures/controller_initialize_params.json b/crates/environment-protocol/fixtures/controller_initialize_params.json deleted file mode 100644 index 55abb5153..000000000 --- a/crates/environment-protocol/fixtures/controller_initialize_params.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "clientName": "lightspeed-test", - "protocolVersion": 2 -} diff --git a/crates/environment-protocol/fixtures/controller_list_targets_response.json b/crates/environment-protocol/fixtures/controller_list_targets_response.json deleted file mode 100644 index 5cb884925..000000000 --- a/crates/environment-protocol/fixtures/controller_list_targets_response.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "targets": [ - { - "capabilities": { - "filesystemRead": true, - "filesystemSearch": true, - "filesystemGlob": true, - "filesystemRangedRead": true, - "filesystemWrite": true, - "jobCancel": true, - "jobDependencies": true, - "jobList": true, - "jobRead": true, - "jobQueueKeys": true, - "jobStart": true, - "jobWaitHint": false, - "network": true, - "processOutputNotifications": true, - "processOutputPolling": true, - "processPty": false, - "processStart": true, - "processStdin": true, - "processTerminate": true - }, - "defaultCwd": "/workspace", - "displayName": "lightspeed dev", - "metadata": { - "provider": "smolvm" - }, - "scope": { - "sessionId": "sandbox-123", - "type": "session" - }, - "status": "ready", - "targetId": "sandbox-123" - } - ] -} diff --git a/crates/environment-protocol/fixtures/data/env.getIdle.json b/crates/environment-protocol/fixtures/data/env.getIdle.json new file mode 100644 index 000000000..9bd9ac05e --- /dev/null +++ b/crates/environment-protocol/fixtures/data/env.getIdle.json @@ -0,0 +1,15 @@ +{ + "params": {}, + "response": { + "idleForMs": 0, + "runningProcessCount": 1, + "leftoverProcessGroupCount": 1 + }, + "error": { + "code": -32002, + "message": "initialize must be the first request on a connection", + "data": { + "kind": "notInitialized" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.createDirectory.json b/crates/environment-protocol/fixtures/data/fs.createDirectory.json new file mode 100644 index 000000000..705890f70 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.createDirectory.json @@ -0,0 +1,15 @@ +{ + "params": { + "path": "/workspace/build/out", + "recursive": true + }, + "response": {}, + "error": { + "code": -32006, + "message": "the filesystem is read-only", + "data": { + "kind": "capabilityUnavailable", + "capability": "filesystemWrite" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.getMetadata.json b/crates/environment-protocol/fixtures/data/fs.getMetadata.json new file mode 100644 index 000000000..d8c16aba5 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.getMetadata.json @@ -0,0 +1,24 @@ +{ + "params": { + "path": "/workspace/link", + "followSymlinks": true + }, + "response": { + "metadata": { + "kind": "file", + "sizeBytes": 1024, + "executable": true, + "modifiedAtMs": 1761400000000, + "createdAtMs": 1761300000000, + "version": "v1:1761400000000000000:1024:42" + } + }, + "error": { + "code": -32003, + "message": "/workspace/link resolves outside the filesystem root", + "data": { + "kind": "permissionDenied", + "reason": "outsideRoot" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.globFiles.json b/crates/environment-protocol/fixtures/data/fs.globFiles.json new file mode 100644 index 000000000..f10ee1502 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.globFiles.json @@ -0,0 +1,33 @@ +{ + "params": { + "root": "/workspace", + "filter": { + "include": [ + "**/SKILL.md" + ], + "ignoreCase": true, + "includeHidden": true, + "includeIgnored": true + }, + "limits": { + "maxMatches": 1000, + "maxFiles": 5000, + "maxDurationMs": 30000 + } + }, + "response": { + "matches": [ + "/workspace/.agents/skills/review/SKILL.md" + ], + "visitedFileCount": 310, + "elapsedMs": 12 + }, + "error": { + "code": -32017, + "message": "request cancelled", + "data": { + "kind": "aborted", + "reason": "cancelled" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.readDirectory.json b/crates/environment-protocol/fixtures/data/fs.readDirectory.json new file mode 100644 index 000000000..430060a53 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.readDirectory.json @@ -0,0 +1,40 @@ +{ + "params": { + "path": "/workspace", + "limit": 2, + "cursor": "c:src" + }, + "response": { + "entries": [ + { + "name": "src", + "metadata": { + "kind": "directory", + "sizeBytes": 4096, + "executable": true, + "modifiedAtMs": 1761400000000, + "version": "v1:1761400000000000000:4096:7" + } + }, + { + "name": "link", + "metadata": { + "kind": "symlink", + "sizeBytes": 6, + "modifiedAtMs": 1761400000000, + "symlinkTarget": "target", + "version": "v1:1761400000000000000:6:9" + } + } + ], + "nextCursor": "c:link", + "skippedEntryCount": 1 + }, + "error": { + "code": -32004, + "message": "/workspace/missing does not exist", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.readFile.json b/crates/environment-protocol/fixtures/data/fs.readFile.json new file mode 100644 index 000000000..e4deeccc3 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.readFile.json @@ -0,0 +1,21 @@ +{ + "params": { + "path": "/workspace/src/lib.rs", + "offset": 1024, + "maxBytes": 65536 + }, + "response": { + "data": "aGVsbG8=", + "fileSize": 4096, + "truncated": true, + "version": "v1:1761400000000000000:4096:42" + }, + "error": { + "code": -32007, + "message": "/dev/zero is not a regular file", + "data": { + "kind": "failedPrecondition", + "reason": "notRegularFile" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.remove.json b/crates/environment-protocol/fixtures/data/fs.remove.json new file mode 100644 index 000000000..e2a9d4e7c --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.remove.json @@ -0,0 +1,16 @@ +{ + "params": { + "path": "/workspace/build", + "recursive": true, + "force": true + }, + "response": {}, + "error": { + "code": -32003, + "message": "/var/lib/lightspeed-envd is the daemon state directory", + "data": { + "kind": "permissionDenied", + "reason": "daemonState" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.rename.json b/crates/environment-protocol/fixtures/data/fs.rename.json new file mode 100644 index 000000000..cbca3341d --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.rename.json @@ -0,0 +1,16 @@ +{ + "params": { + "source": "/workspace/a.rs", + "destination": "/workspace/b.rs", + "overwrite": true, + "ifMatch": "v1:1761400000000000000:4096:42" + }, + "response": {}, + "error": { + "code": -32009, + "message": "/workspace/b.rs exists", + "data": { + "kind": "alreadyExists" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.scan.json b/crates/environment-protocol/fixtures/data/fs.scan.json new file mode 100644 index 000000000..b4ad2ca79 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.scan.json @@ -0,0 +1,89 @@ +{ + "params": { + "roots": [ + { + "path": "/workspace", + "maxDepth": 0 + }, + { + "path": "/workspace/.agents/skills", + "ifNoneMatch": "scan:v1:9f2c" + }, + { + "path": "/home/dev/.agents/skills" + } + ], + "filter": { + "include": [ + "SKILL.md", + "*/SKILL.md" + ], + "maxDepth": 2 + }, + "readContent": true, + "includeDigests": true, + "followSymlinks": true, + "limits": { + "maxEntries": 4096, + "maxFileBytes": 65536, + "maxTotalBytes": 4194304, + "maxDurationMs": 2000 + } + }, + "response": { + "roots": [ + { + "kind": "complete", + "path": "/workspace", + "fingerprint": "scan:v1:0001" + }, + { + "kind": "unchanged", + "path": "/workspace/.agents/skills", + "fingerprint": "scan:v1:9f2c" + }, + { + "kind": "complete", + "path": "/home/dev/.agents/skills", + "canonicalPath": "/home/dev/dotfiles/skills", + "fingerprint": "scan:v1:77aa", + "entries": [ + { + "path": "review", + "content": { + "kind": "directory" + } + }, + { + "path": "review/SKILL.md", + "content": { + "kind": "file", + "sizeBytes": 5, + "digest": "sha256:abababababababababababababababababababababababababababababababab" + }, + "data": "aGVsbG8=" + } + ], + "skipped": [ + { + "path": "huge/SKILL.md", + "reason": "fileTooLarge" + }, + { + "reason": "nonUtf8Name" + } + ], + "skippedEntryCount": 2 + } + ] + }, + "error": { + "code": -32602, + "message": "roots has 40 entries; the ceiling is 32", + "data": { + "kind": "invalidParams", + "field": "roots", + "ceiling": 32 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.searchText.json b/crates/environment-protocol/fixtures/data/fs.searchText.json new file mode 100644 index 000000000..59f51f6eb --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.searchText.json @@ -0,0 +1,47 @@ +{ + "params": { + "root": "/workspace", + "pattern": "fn main", + "patternIgnoreCase": true, + "filter": { + "include": [ + "**/*.rs" + ], + "exclude": [ + "target/**" + ], + "maxDepth": 8 + }, + "limits": { + "maxMatches": 1000, + "maxFiles": 5000, + "maxBytes": 67108864, + "maxDurationMs": 30000, + "maxLineBytes": 4096 + } + }, + "response": { + "matches": [ + { + "path": "/workspace/src/main.rs", + "lineNumber": 3, + "line": "fn main() {" + } + ], + "searchedFileCount": 120, + "searchedBytes": 480000, + "skippedFileCount": 2, + "skippedEntryCount": 1, + "elapsedMs": 41, + "stopped": "matchLimit" + }, + "error": { + "code": -32602, + "message": "limits.maxDurationMs is 90000; the ceiling is 60000", + "data": { + "kind": "invalidParams", + "field": "limits.maxDurationMs", + "ceiling": 60000 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/fs.writeFile.json b/crates/environment-protocol/fixtures/data/fs.writeFile.json new file mode 100644 index 000000000..15ba5f557 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/fs.writeFile.json @@ -0,0 +1,20 @@ +{ + "params": { + "path": "/workspace/src/lib.rs", + "data": "aGVsbG8=", + "create": "ifPresent", + "ifMatch": "v1:1761400000000000000:4096:42", + "executable": true + }, + "response": { + "version": "v1:1761400001000000000:5:42" + }, + "error": { + "code": -32007, + "message": "/workspace/src/lib.rs changed since it was read", + "data": { + "kind": "failedPrecondition", + "reason": "ifMatchFailed" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/initialize.json b/crates/environment-protocol/fixtures/data/initialize.json new file mode 100644 index 000000000..e9bdeddce --- /dev/null +++ b/crates/environment-protocol/fixtures/data/initialize.json @@ -0,0 +1,97 @@ +{ + "params": { + "protocolVersion": 4, + "clientInfo": { + "name": "lightspeed-runtime", + "version": "0.5.0" + } + }, + "response": { + "protocolVersion": 4, + "plane": "data", + "daemonInstanceId": "envd_01jb8x7k2m", + "serverInfo": { + "name": "lightspeed-envd", + "version": "0.5.0", + "gitSha": "86f6adf3", + "target": "x86_64-unknown-linux-musl" + }, + "capabilities": { + "filesystem": { + "write": true + }, + "process": { + "pty": true + }, + "job": {} + }, + "limits": { + "maxConcurrentRequests": 32, + "maxWaitMs": 1800000, + "process": { + "maxReadBytes": 4194304, + "maxRetainedOutputBytes": 1048576, + "retentionAfterExitMs": 60000, + "unreadRetentionMs": 7200000, + "maxExitedRetainedBytes": 67108864, + "terminateGraceMs": 5000, + "maxInputQueueBytes": 1048576 + }, + "job": { + "maxReadBytes": 4194304, + "maxRequestJobs": 64, + "maxRetainedOutputBytes": 2097152, + "recordRetentionMs": 604800000, + "cancelGraceMs": 5000 + }, + "filesystem": { + "maxReadFileBytes": 8388608, + "maxWriteFileBytes": 8388608, + "maxDirectoryEntries": 1000, + "search": { + "maxMatches": 10000, + "maxFiles": 100000, + "maxBytes": 1073741824, + "maxDurationMs": 60000, + "maxLineBytes": 4096 + }, + "glob": { + "maxMatches": 100000, + "maxFiles": 1000000, + "maxDurationMs": 60000 + }, + "scan": { + "maxRoots": 32, + "maxPatterns": 32, + "maxEntries": 10000, + "maxFileBytes": 262144, + "maxTotalBytes": 4194304, + "maxDurationMs": 30000 + }, + "transfer": { + "maxActiveOperations": 8, + "maxContentChunkBytes": 4194304, + "maxInventoryPageEntries": 1000, + "receiptRetentionMs": 86400000, + "maxEntries": 100000, + "maxDepth": 64, + "maxFileBytes": 1099511627776, + "maxTotalBytes": 1099511627776, + "maxManifestBytes": 33554432, + "maxDurationMs": 86400000 + } + } + }, + "defaultCwd": "/workspace", + "homeDirectory": "/home/dev" + }, + "error": { + "code": -32005, + "message": "protocol version 3 is not supported; this peer speaks 4", + "data": { + "kind": "versionMismatch", + "expected": 4, + "received": 3 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/job.cancel.json b/crates/environment-protocol/fixtures/data/job.cancel.json new file mode 100644 index 000000000..d16db2602 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/job.cancel.json @@ -0,0 +1,43 @@ +{ + "params": { + "jobIds": [ + "build" + ], + "scope": "dependents", + "force": true + }, + "response": { + "jobs": [ + { + "kind": "job", + "summary": { + "jobId": "build", + "requestId": "jobreq-run1-turn2-call3", + "status": "running", + "createdAtMs": 1761400000000, + "startedAtMs": 1761400000200, + "cancelRequestedAtMs": 1761400005000 + } + }, + { + "kind": "error", + "jobId": "deploy", + "error": { + "code": -32004, + "message": "unknown job id: deploy", + "data": { + "kind": "notFound" + } + } + } + ] + }, + "error": { + "code": -32602, + "message": "jobIds must not be empty", + "data": { + "kind": "invalidParams", + "field": "jobIds" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/job.read.json b/crates/environment-protocol/fixtures/data/job.read.json new file mode 100644 index 000000000..811427576 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/job.read.json @@ -0,0 +1,115 @@ +{ + "params": { + "jobs": [ + { + "jobId": "build", + "tailBytes": 8192, + "maxBytes": 8192 + }, + { + "jobId": "setup", + "after": 512 + }, + { + "jobId": "gone" + } + ], + "maxBytes": 16384, + "waitMs": 0, + "returnOn": "exit" + }, + "response": { + "jobs": [ + { + "kind": "job", + "summary": { + "jobId": "build", + "requestId": "jobreq-run1-turn2-call3", + "name": "build", + "status": "failed", + "dependsOn": [ + "setup" + ], + "queueKey": "cargo", + "createdAtMs": 1761400000000, + "queuedAtMs": 1761400000100, + "startedAtMs": 1761400000200, + "finishedAtMs": 1761400009200, + "durationMs": 9000, + "exit": { + "kind": "code", + "code": 101 + }, + "leftovers": { + "disposition": "swept", + "processes": [ + { + "pid": 4242, + "command": "sleep 600" + } + ] + }, + "error": { + "code": -32018, + "message": "output capture failed", + "data": { + "kind": "processFailed" + } + } + }, + "output": { + "chunks": [ + { + "offset": 3000000, + "stream": "stderr", + "data": "ZXJyb3I=" + } + ], + "next": 3000005, + "end": 3000005 + } + }, + { + "kind": "job", + "summary": { + "jobId": "setup", + "requestId": "jobreq-run1-turn2-call3", + "name": "setup", + "status": "succeeded", + "createdAtMs": 1761400000000, + "startedAtMs": 1761400000050, + "finishedAtMs": 1761400003050, + "durationMs": 3000, + "exit": { + "kind": "code", + "code": 0 + } + }, + "output": { + "next": 512, + "end": 512 + } + }, + { + "kind": "error", + "jobId": "gone", + "error": { + "code": -32004, + "message": "unknown job id: gone", + "data": { + "kind": "notFound" + } + } + } + ] + }, + "error": { + "code": -32602, + "message": "waitMs is 3600000; the ceiling is 1800000", + "data": { + "kind": "invalidParams", + "field": "waitMs", + "ceiling": 1800000 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/job.start.json b/crates/environment-protocol/fixtures/data/job.start.json new file mode 100644 index 000000000..af185d018 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/job.start.json @@ -0,0 +1,75 @@ +{ + "params": { + "requestId": "jobreq-run1-turn2-call3", + "jobs": [ + { + "jobId": "setup", + "name": "setup", + "command": { + "argv": [ + "npm", + "ci" + ] + } + }, + { + "jobId": "build", + "command": { + "argv": [ + "cargo", + "build" + ], + "cwd": "/workspace", + "input": { + "kind": "bytes", + "data": "aGVsbG8=" + }, + "timeoutMs": 1800000 + }, + "dependsOn": [ + { + "kind": "name", + "name": "setup" + }, + { + "kind": "jobId", + "jobId": "lint" + } + ], + "dependencyPolicy": "allTerminal", + "queueKey": "cargo" + } + ] + }, + "response": { + "jobs": [ + { + "jobId": "setup", + "requestId": "jobreq-run1-turn2-call3", + "name": "setup", + "status": "running", + "createdAtMs": 1761400000000, + "startedAtMs": 1761400000050 + }, + { + "jobId": "build", + "requestId": "jobreq-run1-turn2-call3", + "status": "queued", + "dependsOn": [ + "setup", + "lint" + ], + "queueKey": "cargo", + "createdAtMs": 1761400000000 + } + ] + }, + "error": { + "code": -32007, + "message": "job id build names an earlier run; choose a new job id", + "data": { + "kind": "failedPrecondition", + "reason": "jobIdInUse" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/process.read.json b/crates/environment-protocol/fixtures/data/process.read.json new file mode 100644 index 000000000..74529eb76 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/process.read.json @@ -0,0 +1,63 @@ +{ + "params": { + "processId": "proc-7", + "after": 16384, + "ack": 16384, + "maxBytes": 65536, + "waitMs": 20000, + "returnOn": "anyOutput" + }, + "response": { + "output": { + "chunks": [ + { + "offset": 16384, + "stream": "stdout", + "data": "aGVsbG8=" + }, + { + "offset": 1048581, + "stream": "stderr", + "data": "ZmFpbA==" + } + ], + "next": 1048585, + "end": 1048585, + "gaps": [ + { + "offset": 16389, + "omittedBytes": 1032192, + "reason": "overflow" + } + ] + }, + "state": "exited", + "exit": { + "kind": "signal", + "signal": "SIGSEGV", + "coreDumped": true + }, + "stoppedBy": "terminate", + "pid": 4242, + "leftovers": { + "disposition": "kept", + "processes": [ + { + "pid": 4250, + "command": "postgres -D /data" + }, + { + "pid": 4251 + } + ] + } + }, + "error": { + "code": -32004, + "message": "unknown process id: proc-7", + "data": { + "kind": "notFound", + "reason": "expired" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/process.start.json b/crates/environment-protocol/fixtures/data/process.start.json new file mode 100644 index 000000000..0cf4495bc --- /dev/null +++ b/crates/environment-protocol/fixtures/data/process.start.json @@ -0,0 +1,38 @@ +{ + "params": { + "processId": "proc-7", + "command": { + "argv": [ + "sh", + "-lc", + "cat" + ], + "cwd": "/workspace", + "env": { + "RUST_LOG": "debug" + }, + "secretEnv": { + "API_TOKEN": "s3cr3t" + }, + "input": { + "kind": "bytes", + "data": "aGVsbG8=" + }, + "timeoutMs": 600000 + }, + "pty": { + "rows": 40, + "cols": 120 + } + }, + "response": { + "pid": 4242 + }, + "error": { + "code": -32009, + "message": "process id proc-7 is in use", + "data": { + "kind": "alreadyExists" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/process.terminate.json b/crates/environment-protocol/fixtures/data/process.terminate.json new file mode 100644 index 000000000..e0596a728 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/process.terminate.json @@ -0,0 +1,17 @@ +{ + "params": { + "processId": "proc-7", + "signal": "terminate" + }, + "response": { + "running": true + }, + "error": { + "code": -32004, + "message": "unknown process id: proc-7", + "data": { + "kind": "notFound", + "reason": "daemonRestarted" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/process.write.json b/crates/environment-protocol/fixtures/data/process.write.json new file mode 100644 index 000000000..efff05c73 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/process.write.json @@ -0,0 +1,18 @@ +{ + "params": { + "processId": "proc-7", + "data": "aGVsbG8K", + "closeInput": true + }, + "response": { + "acceptedBytes": 6 + }, + "error": { + "code": -32007, + "message": "process proc-7 has no open input", + "data": { + "kind": "failedPrecondition", + "reason": "inputClosed" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/rpc.cancelRequest.json b/crates/environment-protocol/fixtures/data/rpc.cancelRequest.json new file mode 100644 index 000000000..b3e9ca10a --- /dev/null +++ b/crates/environment-protocol/fixtures/data/rpc.cancelRequest.json @@ -0,0 +1,5 @@ +{ + "params": { + "id": 42 + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.abort.json b/crates/environment-protocol/fixtures/data/transfer.abort.json new file mode 100644 index 000000000..8229f7231 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.abort.json @@ -0,0 +1,22 @@ +{ + "params": { + "operationId": "op-2" + }, + "response": { + "operationId": "op-2", + "phase": "aborted", + "entryCount": 3, + "totalBytes": 2048, + "transferredBytes": 1024, + "reusedBytes": 512, + "deadlineAtMs": 1761400000000, + "receiptExpiresAtMs": 1761486400000 + }, + "error": { + "code": -32004, + "message": "unknown operation op-2", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.advance.json b/crates/environment-protocol/fixtures/data/transfer.advance.json new file mode 100644 index 000000000..c6f52bfba --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.advance.json @@ -0,0 +1,38 @@ +{ + "params": { + "operationId": "op-1" + }, + "response": { + "operationId": "op-1", + "phase": "failed", + "entryCount": 3, + "totalBytes": 2048, + "transferredBytes": 1024, + "reusedBytes": 512, + "deadlineAtMs": 1761400000000, + "skipped": [ + { + "path": "node_modules/.bin/tsc", + "reason": "symlink" + } + ], + "skippedEntryCount": 140, + "error": { + "code": -32017, + "message": "src/lib.rs changed while it was read", + "data": { + "kind": "aborted", + "reason": "sourceChanged" + } + }, + "receiptExpiresAtMs": 1761486400000 + }, + "error": { + "code": -32007, + "message": "advance is not valid in the complete phase", + "data": { + "kind": "failedPrecondition", + "reason": "transferPhase" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.append.json b/crates/environment-protocol/fixtures/data/transfer.append.json new file mode 100644 index 000000000..4270216ae --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.append.json @@ -0,0 +1,40 @@ +{ + "params": { + "operationId": "op-2", + "index": 0, + "entries": [ + { + "path": ".", + "content": { + "kind": "directory" + } + }, + { + "path": "README.md", + "content": { + "kind": "file", + "sizeBytes": 12, + "digest": "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd" + } + } + ], + "last": true + }, + "response": { + "operationId": "op-2", + "phase": "scanning", + "entryCount": 2, + "totalBytes": 12, + "transferredBytes": 0, + "reusedBytes": 0, + "deadlineAtMs": 1761400000000 + }, + "error": { + "code": -32007, + "message": "operation op-2 was begun with different input", + "data": { + "kind": "failedPrecondition", + "reason": "operationIdInUse" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.begin.json b/crates/environment-protocol/fixtures/data/transfer.begin.json new file mode 100644 index 000000000..7b548e881 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.begin.json @@ -0,0 +1,43 @@ +{ + "params": { + "operationId": "op-1", + "selection": { + "kind": "capture", + "source": "/workspace", + "filter": { + "exclude": [ + "node_modules/**", + ".git/**" + ] + }, + "symlinks": "skip", + "specialFiles": "skip" + }, + "limits": { + "maxEntries": 100000, + "maxDepth": 64, + "maxFileBytes": 1073741824, + "maxTotalBytes": 1073741824, + "maxManifestBytes": 33554432, + "maxDurationMs": 86400000 + } + }, + "response": { + "operationId": "op-1", + "phase": "scanning", + "entryCount": 0, + "totalBytes": 0, + "transferredBytes": 0, + "reusedBytes": 0, + "deadlineAtMs": 1761400000000 + }, + "error": { + "code": -32008, + "message": "8 transfers are active", + "data": { + "kind": "busy", + "reason": "transferCapacity", + "retryAfterMs": 1000 + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.commit.json b/crates/environment-protocol/fixtures/data/transfer.commit.json new file mode 100644 index 000000000..b78e69dd6 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.commit.json @@ -0,0 +1,22 @@ +{ + "params": { + "operationId": "op-2" + }, + "response": { + "operationId": "op-2", + "phase": "complete", + "entryCount": 3, + "totalBytes": 2048, + "transferredBytes": 1024, + "reusedBytes": 512, + "deadlineAtMs": 1761400000000, + "receiptExpiresAtMs": 1761486400000 + }, + "error": { + "code": -32015, + "message": "operation op-2 ran out of its 86400000 ms", + "data": { + "kind": "deadlineExceeded" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.getStatus.json b/crates/environment-protocol/fixtures/data/transfer.getStatus.json new file mode 100644 index 000000000..8ce8c3a14 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.getStatus.json @@ -0,0 +1,23 @@ +{ + "params": { + "operationId": "op-1" + }, + "response": { + "operationId": "op-1", + "phase": "complete", + "entryCount": 3, + "totalBytes": 2048, + "transferredBytes": 1024, + "reusedBytes": 512, + "deadlineAtMs": 1761400000000, + "receiptExpiresAtMs": 1761486400000 + }, + "error": { + "code": -32004, + "message": "transfer receipt op-1 expired", + "data": { + "kind": "notFound", + "reason": "expired" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.listMissing.json b/crates/environment-protocol/fixtures/data/transfer.listMissing.json new file mode 100644 index 000000000..2bc25ac9f --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.listMissing.json @@ -0,0 +1,20 @@ +{ + "params": { + "operationId": "op-2", + "limit": 64 + }, + "response": { + "digests": [ + "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd" + ], + "remainingCount": 3 + }, + "error": { + "code": -32007, + "message": "listMissing is not valid in the inventory phase", + "data": { + "kind": "failedPrecondition", + "reason": "transferPhase" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.read.json b/crates/environment-protocol/fixtures/data/transfer.read.json new file mode 100644 index 000000000..70d89110e --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.read.json @@ -0,0 +1,18 @@ +{ + "params": { + "operationId": "op-1", + "digest": "sha256:abababababababababababababababababababababababababababababababab", + "offset": 0, + "maxBytes": 4194304 + }, + "response": { + "data": "aGVsbG8=" + }, + "error": { + "code": -32004, + "message": "digest is not part of the inventory", + "data": { + "kind": "notFound" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.readInventory.json b/crates/environment-protocol/fixtures/data/transfer.readInventory.json new file mode 100644 index 000000000..5a6a26baf --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.readInventory.json @@ -0,0 +1,35 @@ +{ + "params": { + "operationId": "op-1", + "index": 0, + "limit": 128 + }, + "response": { + "entries": [ + { + "path": ".", + "content": { + "kind": "directory" + } + }, + { + "path": "bin/tool", + "content": { + "kind": "file", + "sizeBytes": 2048, + "executable": true, + "digest": "sha256:abababababababababababababababababababababababababababababababab" + } + } + ], + "nextIndex": 2 + }, + "error": { + "code": -32007, + "message": "the inventory is not sealed yet", + "data": { + "kind": "failedPrecondition", + "reason": "transferPhase" + } + } +} diff --git a/crates/environment-protocol/fixtures/data/transfer.write.json b/crates/environment-protocol/fixtures/data/transfer.write.json new file mode 100644 index 000000000..080567086 --- /dev/null +++ b/crates/environment-protocol/fixtures/data/transfer.write.json @@ -0,0 +1,19 @@ +{ + "params": { + "operationId": "op-2", + "digest": "sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + "offset": 0, + "data": "aGVsbG8gd29ybGQK" + }, + "response": { + "nextOffset": 12 + }, + "error": { + "code": -32007, + "message": "uploaded bytes do not hash to the digest", + "data": { + "kind": "failedPrecondition", + "reason": "digestMismatch" + } + } +} diff --git a/crates/environment-protocol/fixtures/data_initialize_params.json b/crates/environment-protocol/fixtures/data_initialize_params.json deleted file mode 100644 index 0e07e72db..000000000 --- a/crates/environment-protocol/fixtures/data_initialize_params.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "clientName": "lightspeed-test", - "protocolVersion": 2, - "resumeConnectionId": "conn-prev", - "scope": { - "sessionId": "sandbox-123", - "type": "session" - } -} diff --git a/crates/environment-protocol/fixtures/fs_read_file_response.json b/crates/environment-protocol/fixtures/fs_read_file_response.json deleted file mode 100644 index 5340bffb7..000000000 --- a/crates/environment-protocol/fixtures/fs_read_file_response.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "data": "aGVsbG8K" -} diff --git a/crates/environment-protocol/fixtures/job_read_response.json b/crates/environment-protocol/fixtures/job_read_response.json deleted file mode 100644 index b8656ee16..000000000 --- a/crates/environment-protocol/fixtures/job_read_response.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "jobs": [ - { - "artifacts": [ - { - "kind": "file", - "metadata": { - "role": "report" - }, - "path": "/workspace/result.txt" - } - ], - "outputChunks": [ - { - "chunk": "b2sK", - "seq": 1, - "stream": "stdout" - } - ], - "outputNextSeq": 2, - "summary": { - "createdAtMs": 1, - "dependencies": [ - "checkout" - ], - "exitCode": 0, - "finishedAtMs": 4, - "jobId": "tests", - "name": "tests", - "namespace": "session_1", - "orphanedDescendants": true, - "queuedAtMs": 2, - "queueKey": "repo", - "startedAtMs": 3, - "status": "succeeded" - } - } - ] -} diff --git a/crates/environment-protocol/fixtures/job_start_params.json b/crates/environment-protocol/fixtures/job_start_params.json deleted file mode 100644 index 64fd721b4..000000000 --- a/crates/environment-protocol/fixtures/job_start_params.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "jobs": [ - { - "argv": [ - "git", - "status" - ], - "cwd": "/workspace", - "dependencyPolicy": "allSucceeded", - "dependsOn": [ - { - "name": "setup" - } - ], - "env": { - "RUST_LOG": "debug" - }, - "name": "checkout", - "jobId": "checkout", - "queueKey": "repo", - "stdin": "aGVsbG8K", - "timeoutMs": 60000 - } - ], - "namespace": "session_1", - "requestId": "request-1" -} diff --git a/crates/environment-protocol/fixtures/process_read_response.json b/crates/environment-protocol/fixtures/process_read_response.json deleted file mode 100644 index 2141c3ff8..000000000 --- a/crates/environment-protocol/fixtures/process_read_response.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "chunks": [ - { - "chunk": "b2sK", - "seq": 1, - "stream": "stdout" - }, - { - "chunk": "d2Fybgo=", - "seq": 2, - "stream": "stderr" - } - ], - "closed": true, - "exitCode": 0, - "exited": true, - "leftoverProcesses": [ - { - "command": "python -m http.server 8080", - "pid": 91 - } - ], - "nextSeq": 3, - "pid": 90 -} diff --git a/crates/environment-protocol/fixtures/process_start_params.json b/crates/environment-protocol/fixtures/process_start_params.json deleted file mode 100644 index 9c6e5a190..000000000 --- a/crates/environment-protocol/fixtures/process_start_params.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "argv": [ - "sh", - "-lc", - "cat" - ], - "cwd": "/workspace", - "env": { - "RUST_LOG": "debug" - }, - "processId": "proc-1", - "stdin": "aW5wdXQK", - "timeoutMs": 60000, - "tty": false -} diff --git a/crates/environment-protocol/fixtures/registration/registration.challenge.json b/crates/environment-protocol/fixtures/registration/registration.challenge.json new file mode 100644 index 000000000..0716d92bb --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.challenge.json @@ -0,0 +1,6 @@ +{ + "params": { + "protocolVersion": 4, + "nonce": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f" + } +} diff --git a/crates/environment-protocol/fixtures/registration/registration.closed.json b/crates/environment-protocol/fixtures/registration/registration.closed.json new file mode 100644 index 000000000..975d0bdf1 --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.closed.json @@ -0,0 +1,6 @@ +{ + "params": { + "reason": "superseded", + "message": "another process with this daemon identity connected" + } +} diff --git a/crates/environment-protocol/fixtures/registration/registration.goodbye.json b/crates/environment-protocol/fixtures/registration/registration.goodbye.json new file mode 100644 index 000000000..3675fe277 --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.goodbye.json @@ -0,0 +1,5 @@ +{ + "params": { + "reason": "shutdown" + } +} diff --git a/crates/environment-protocol/fixtures/registration/registration.openStream.json b/crates/environment-protocol/fixtures/registration/registration.openStream.json new file mode 100644 index 000000000..31ac34c21 --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.openStream.json @@ -0,0 +1,16 @@ +{ + "params": { + "token": "tok_5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + "stream": { + "kind": "dataProtocol" + } + }, + "response": {}, + "error": { + "code": -32019, + "message": "dialing wss://gw.example.com/environment-gateway/data failed: connection refused", + "data": { + "kind": "unavailable" + } + } +} diff --git a/crates/environment-protocol/fixtures/registration/registration.register.json b/crates/environment-protocol/fixtures/registration/registration.register.json new file mode 100644 index 000000000..84f2e1294 --- /dev/null +++ b/crates/environment-protocol/fixtures/registration/registration.register.json @@ -0,0 +1,36 @@ +{ + "params": { + "protocolVersion": 4, + "registrationKey": "lsrk_0123456789abcdef", + "daemonPublicKey": "3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b3b", + "signature": "9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c9c", + "displayName": "ci-runner-7", + "metadata": { + "harbor.trialId": "trial-1" + }, + "implementation": { + "name": "lightspeed-envd", + "version": "0.5.0", + "gitSha": "86f6adf3", + "target": "x86_64-unknown-linux-musl" + } + }, + "response": { + "receipt": { + "environmentId": "env_1", + "daemonId": "daemon_3b3b", + "controlConnectionId": "connection_7", + "identityMode": "ephemeral", + "admission": "created" + }, + "heartbeatIntervalMs": 30000 + }, + "error": { + "code": -32003, + "message": "the registration signature does not verify", + "data": { + "kind": "permissionDenied", + "reason": "invalidSignature" + } + } +} diff --git a/crates/environment-protocol/src/bin/export-contract.rs b/crates/environment-protocol/src/bin/export-contract.rs new file mode 100644 index 000000000..b3f077732 --- /dev/null +++ b/crates/environment-protocol/src/bin/export-contract.rs @@ -0,0 +1,25 @@ +//! Writes the committed contract under `crates/environment-protocol/contract/`. +//! +//! Usage: `cargo run -p environment-protocol --features schema --bin export-contract [output-dir]` + +use std::{env, fs, path::PathBuf}; + +fn main() { + let out_dir = env::args() + .nth(1) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("contract")); + fs::create_dir_all(&out_dir).expect("create output directory"); + for plane in environment_protocol::contract::export() { + for (name, value) in [ + (plane.schema_file_name(), &plane.schema_bundle), + (plane.openrpc_file_name(), &plane.openrpc), + ] { + let mut text = serde_json::to_string_pretty(value).expect("serialize contract"); + text.push('\n'); + let path = out_dir.join(name); + fs::write(&path, text).expect("write contract"); + println!("wrote {}", path.display()); + } + } +} diff --git a/crates/environment-protocol/src/bootstrap.rs b/crates/environment-protocol/src/bootstrap.rs new file mode 100644 index 000000000..fe995ffda --- /dev/null +++ b/crates/environment-protocol/src/bootstrap.rs @@ -0,0 +1,70 @@ +//! The frozen bootstrap surface besides the `registration/challenge` frame: +//! the envd discovery document the gateway serves at +//! [`DISCOVERY_PATH`](crate::routes::DISCOVERY_PATH), and the build facts +//! `lightspeed-envd --print-build` prints. +//! +//! Daemons of every version read these to upgrade themselves, so they only +//! ever gain fields and are decoded tolerantly: unknown members are +//! ignored. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::bytes::hex_value; + +hex_value!( + /// `sha256` of a published archive: 64 lowercase hex digits. + ArtifactSha256, + 32, + "^[0-9a-f]{64}$" +); + +/// The discovery document. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct DiscoveryDocument { + /// Release version of the published envd. + pub version: String, + pub git_sha: String, + /// Release channel, such as `stable` or `snapshot`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub channel: Option, + /// The published envd's `ENVD_PROTOCOL_VERSION`. + pub protocol_version: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub built_at_ms: Option, + /// Archives by Rust target triple. + pub artifacts: BTreeMap, +} + +/// One published envd archive. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct DiscoveryArtifact { + /// Archive file name. + pub file: String, + /// Digest of the archive's bytes. + pub sha256: ArtifactSha256, + /// Absolute download URL; absent (or null) when the archive is served + /// next to the document. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, +} + +/// What `lightspeed-envd --print-build` prints, as one JSON object. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct BuildFacts { + /// Always `lightspeed-envd`. + pub name: String, + pub version: String, + pub git_sha: String, + /// Rust target triple. + pub target: String, + /// The binary's `ENVD_PROTOCOL_VERSION`. + pub protocol_version: u32, +} diff --git a/crates/environment-protocol/src/bytes.rs b/crates/environment-protocol/src/bytes.rs new file mode 100644 index 000000000..35b8b84cf --- /dev/null +++ b/crates/environment-protocol/src/bytes.rs @@ -0,0 +1,303 @@ +//! Byte encodings. +//! +//! Payload bytes are padded standard base64 ([`ByteChunk`]). Fixed-size +//! cryptographic values are canonical lowercase hex, decoded byte by byte +//! so malformed input fails cleanly and never panics. + +use std::fmt; + +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +/// JSON Schema pattern of padded standard base64. +pub const BASE64_PATTERN: &str = "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"; + +/// Payload bytes, always named `data` on the wire, as padded standard +/// base64. +#[derive(Clone, Default, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct ByteChunk( + #[cfg_attr( + feature = "schema", + schemars( + with = "String", + pattern(BASE64_PATTERN), + extend("contentEncoding" = "base64") + ) + )] + Vec, +); + +impl ByteChunk { + pub fn new(bytes: impl Into>) -> Self { + Self(bytes.into()) + } + + pub fn into_inner(self) -> Vec { + self.0 + } + + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for ByteChunk { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "ByteChunk({} bytes)", self.0.len()) + } +} + +impl From> for ByteChunk { + fn from(value: Vec) -> Self { + Self(value) + } +} + +impl From<&[u8]> for ByteChunk { + fn from(value: &[u8]) -> Self { + Self(value.to_vec()) + } +} + +impl Serialize for ByteChunk { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&BASE64_STANDARD.encode(&self.0)) + } +} + +impl<'de> Deserialize<'de> for ByteChunk { + fn deserialize>(deserializer: D) -> Result { + let encoded = String::deserialize(deserializer)?; + BASE64_STANDARD + .decode(encoded) + .map(Self) + .map_err(D::Error::custom) + } +} + +/// A secret value carried on the wire, such as a `secretEnv` value, a +/// registration key or a stream token. It serializes as a plain string; +/// `Debug` is redacted so structured logging never prints it. +#[derive(Clone, PartialEq, Eq)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "schema", schemars(with = "String"))] +pub struct SecretString(String); + +impl SecretString { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn expose(&self) -> &str { + &self.0 + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for SecretString { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SecretString()") + } +} + +impl From for SecretString { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for SecretString { + fn from(value: &str) -> Self { + Self(value.to_owned()) + } +} + +impl Serialize for SecretString { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for SecretString { + fn deserialize>(deserializer: D) -> Result { + String::deserialize(deserializer).map(Self) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum HexError { + #[error("hex value has {actual} digits; expected {expected}")] + Length { expected: usize, actual: usize }, + + #[error("hex value has an odd number of digits")] + OddLength, + + #[error("byte {index} is not a lowercase hex digit")] + Digit { index: usize }, +} + +/// Lowercase hex of `bytes`. +pub fn encode_hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut text = String::with_capacity(bytes.len() * 2); + for byte in bytes { + text.push(DIGITS[usize::from(byte >> 4)] as char); + text.push(DIGITS[usize::from(byte & 0x0f)] as char); + } + text +} + +/// Decode canonical lowercase hex of any even length. +pub fn decode_hex(value: &str) -> Result, HexError> { + let digits = value.as_bytes(); + if !digits.len().is_multiple_of(2) { + return Err(HexError::OddLength); + } + digits + .as_chunks::<2>() + .0 + .iter() + .enumerate() + .map(|(pair, [high, low])| { + let high = hex_digit(*high).ok_or(HexError::Digit { index: pair * 2 })?; + let low = hex_digit(*low).ok_or(HexError::Digit { + index: pair * 2 + 1, + })?; + Ok((high << 4) | low) + }) + .collect() +} + +/// Decode canonical lowercase hex of exactly `N` bytes. +pub fn decode_hex_array(value: &str) -> Result<[u8; N], HexError> { + if value.len() != N * 2 { + return Err(HexError::Length { + expected: N * 2, + actual: value.len(), + }); + } + let bytes = decode_hex(value)?; + let mut array = [0u8; N]; + array.copy_from_slice(&bytes); + Ok(array) +} + +fn hex_digit(digit: u8) -> Option { + match digit { + b'0'..=b'9' => Some(digit - b'0'), + b'a'..=b'f' => Some(digit - b'a' + 10), + _ => None, + } +} + +/// A fixed-size value that travels as canonical lowercase hex. +macro_rules! hex_value { + ($(#[$doc:meta])* $name:ident, $bytes:literal, $pattern:literal) => { + $(#[$doc])* + #[derive(Clone, Copy, PartialEq, Eq, Hash)] + #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] + pub struct $name( + #[cfg_attr( + feature = "schema", + schemars( + with = "String", + pattern($pattern) + ) + )] + [u8; $bytes], + ); + + impl $name { + pub const BYTES: usize = $bytes; + + pub fn from_bytes(bytes: [u8; $bytes]) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8; $bytes] { + &self.0 + } + + pub fn parse(value: &str) -> Result { + $crate::bytes::decode_hex_array::<$bytes>(value).map(Self) + } + + pub fn to_hex(&self) -> String { + $crate::bytes::encode_hex(&self.0) + } + } + + impl std::fmt::Debug for $name { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "{}({})", stringify!($name), self.to_hex()) + } + } + + impl std::fmt::Display for $name { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.to_hex()) + } + } + + impl std::str::FromStr for $name { + type Err = $crate::bytes::HexError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } + } + + impl serde::Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_hex()) + } + } + + impl<'de> serde::Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + Self::parse(&value).map_err(::custom) + } + } + }; +} + +pub(crate) use hex_value; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn byte_chunks_are_padded_standard_base64() { + let value = serde_json::to_value(ByteChunk::from(b"hello".as_slice())).unwrap(); + assert_eq!(value, serde_json::json!("aGVsbG8=")); + assert!(serde_json::from_value::(serde_json::json!("aGVsbG8")).is_err()); + } + + #[test] + fn hex_decoding_is_canonical_and_never_panics() { + assert_eq!(decode_hex("0001abff").unwrap(), [0, 1, 0xab, 0xff]); + assert_eq!(encode_hex(&[0, 1, 0xab, 0xff]), "0001abff"); + for bad in ["abc", "zz", "AB", "+1", "a\u{e9}b", "aaaaaaaaaaaaaaa\u{e9}"] { + assert!(decode_hex(bad).is_err(), "{bad:?}"); + } + assert!(matches!( + decode_hex_array::<2>("00"), + Err(HexError::Length { .. }) + )); + } +} diff --git a/crates/environment-protocol/src/contract.rs b/crates/environment-protocol/src/contract.rs new file mode 100644 index 000000000..0c6e269be --- /dev/null +++ b/crates/environment-protocol/src/contract.rs @@ -0,0 +1,369 @@ +//! Machine-readable contract export, one set of documents per plane: +//! +//! - `.schema.json`: a draft-07 JSON Schema bundle of every wire type +//! the plane uses, under `definitions`; +//! - `.openrpc.json`: an OpenRPC document of the plane's methods with +//! by-name params, the manifest facts as `x-lightspeed-*` members, and the +//! error kinds. +//! +//! The controller plane also names two types no method carries as a whole: +//! `x-lightspeed-data-route` (the data route refusal body and its bounds) +//! and `x-lightspeed-idle-policy` (the policy providers run). +//! +//! The committed copies under `crates/environment-protocol/contract/` are +//! checked by the `contract` integration test; regenerate them with +//! `cargo run -p environment-protocol --features schema --bin export-contract`. + +use std::collections::BTreeMap; + +use schemars::generate::SchemaSettings; +use serde_json::{Value, json}; + +use crate::{ + CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION, + bootstrap::{BuildFacts, DiscoveryDocument}, + error::{ProtocolError, ProtocolErrorKind}, + handshake::Plane, + idle::IdlePolicy, + limits::{MAX_DATA_ROUTE_HOLD_MS, MAX_MESSAGE_BYTES, RETRY_AFTER_MAX_SECS}, + manifest::{MessageKind, plane_methods}, + routes::DataRouteRefusal, +}; + +/// The exported documents of one plane. +pub struct PlaneContract { + pub plane: Plane, + pub schema_bundle: Value, + pub openrpc: Value, +} + +impl PlaneContract { + pub fn schema_file_name(&self) -> String { + format!("{}.schema.json", self.plane) + } + + pub fn openrpc_file_name(&self) -> String { + format!("{}.openrpc.json", self.plane) + } +} + +pub fn export() -> Vec { + [Plane::Data, Plane::Controller, Plane::Registration] + .into_iter() + .map(export_plane) + .collect() +} + +fn export_plane(plane: Plane) -> PlaneContract { + let mut generator = SchemaSettings::draft07().into_generator(); + let error_schema = to_value(generator.subschema_for::()); + let bootstrap = (plane == Plane::Registration).then(|| { + json!({ + "discoveryDocument": to_value(generator.subschema_for::()), + "buildFacts": to_value(generator.subschema_for::()), + }) + }); + let controller_types = (plane == Plane::Controller).then(|| { + ( + json!({ + "refusal": to_value(generator.subschema_for::()), + "maxHoldMs": MAX_DATA_ROUTE_HOLD_MS, + "retryAfterMaxSecs": RETRY_AFTER_MAX_SECS, + }), + to_value(generator.subschema_for::()), + ) + }); + let mut entries = Vec::new(); + for spec in plane_methods(plane) { + let schemas = (spec.schemas)(&mut generator); + entries.push(( + spec, + to_value(schemas.params), + schemas.response.map(to_value), + )); + } + let definitions: BTreeMap = generator + .take_definitions(true) + .into_iter() + .map(|(name, mut schema)| { + strip_null(&mut schema); + clean_descriptions(&mut schema); + (name, schema) + }) + .collect(); + + let methods: Vec = entries + .into_iter() + .map(|(spec, params, response)| { + let mut method = json!({ + "name": spec.name, + "summary": spec.summary, + "paramStructure": "by-name", + "params": by_name_params(¶ms, &definitions), + "x-lightspeed-kind": spec.kind.as_str(), + "x-lightspeed-direction": spec.direction.as_str(), + "x-lightspeed-family": spec.family.as_str(), + "x-lightspeed-params-type": spec.params_type, + }); + if let Some(activity) = spec.activity { + method["x-lightspeed-activity"] = json!(activity.as_str()); + } + if let Some(capability) = spec.capability { + method["x-lightspeed-capability"] = json!(capability); + } + if spec.abortable { + method["x-lightspeed-abortable"] = json!(true); + } + if let (MessageKind::Request, Some(response)) = (spec.kind, response) { + method["result"] = json!({"name": "result", "schema": response}); + method["x-lightspeed-result-type"] = json!(spec.response_type); + } + method + }) + .collect(); + + let (title, version) = match plane { + Plane::Data => ("Lightspeed environment data plane", ENVD_PROTOCOL_VERSION), + Plane::Controller => ( + "Lightspeed environment controller plane", + CONTROLLER_PROTOCOL_VERSION, + ), + Plane::Registration => ( + "Lightspeed environment registration channel", + ENVD_PROTOCOL_VERSION, + ), + }; + let error_kinds: Vec = ProtocolErrorKind::ALL + .into_iter() + .map(|kind| { + json!({ + "kind": kind.as_str(), + "code": kind.code(), + "retryable": kind.is_retryable(), + }) + }) + .collect(); + + let mut schema_bundle = json!({ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": title, + "description": format!("Every wire type of the {plane} plane, protocol version {version}."), + "definitions": definitions, + }); + if let Some(bootstrap) = &bootstrap { + schema_bundle["x-lightspeed-bootstrap"] = bootstrap.clone(); + } + if let Some((data_route, idle_policy)) = &controller_types { + schema_bundle["x-lightspeed-data-route"] = data_route.clone(); + schema_bundle["x-lightspeed-idle-policy"] = idle_policy.clone(); + } + + let mut openrpc = json!({ + "openrpc": "1.3.2", + "info": { + "title": title, + "version": version.to_string(), + "description": format!( + "JSON-RPC 2.0 over WebSocket text frames of at most {MAX_MESSAGE_BYTES} bytes. \ + Exact-match protocolVersion {version}. Errors carry code, message and \ + data.kind; see x-lightspeed-error-kinds." + ), + }, + "methods": methods, + "components": { + "schemas": schema_bundle["definitions"].clone(), + }, + "x-lightspeed-plane": plane.as_str(), + "x-lightspeed-error": error_schema, + "x-lightspeed-error-kinds": error_kinds, + }); + if let Some(bootstrap) = bootstrap { + openrpc["x-lightspeed-bootstrap"] = bootstrap; + } + if let Some((data_route, idle_policy)) = controller_types { + openrpc["x-lightspeed-data-route"] = data_route; + openrpc["x-lightspeed-idle-policy"] = idle_policy; + } + rewrite_refs(&mut openrpc, "#/definitions/", "#/components/schemas/"); + + PlaneContract { + plane, + schema_bundle, + openrpc, + } +} + +/// One OpenRPC content descriptor per property of the params object. +fn by_name_params(schema: &Value, definitions: &BTreeMap) -> Value { + let resolved = schema + .get("$ref") + .and_then(Value::as_str) + .and_then(|reference| reference.strip_prefix("#/definitions/")) + .and_then(|name| definitions.get(name)) + .unwrap_or(schema); + let required: Vec<&str> = resolved + .get("required") + .and_then(Value::as_array) + .map(|names| names.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + let Some(properties) = resolved.get("properties").and_then(Value::as_object) else { + return json!([]); + }; + Value::Array( + properties + .iter() + .map(|(name, schema)| { + json!({ + "name": name, + "required": required.contains(&name.as_str()), + "schema": schema, + }) + }) + .collect(), + ) +} + +fn to_value(schema: schemars::Schema) -> Value { + let mut value = serde_json::to_value(schema).expect("schema serializes"); + strip_null(&mut value); + clean_descriptions(&mut value); + value +} + +/// Descriptions come from rustdoc comments; implementers read them without +/// rustdoc, so intra-doc links become their plain text. +fn clean_descriptions(value: &mut Value) { + match value { + Value::Object(map) => { + for (key, entry) in map.iter_mut() { + match entry { + Value::String(text) if key == "description" => *text = strip_doc_links(text), + _ => clean_descriptions(entry), + } + } + } + Value::Array(entries) => { + for entry in entries { + clean_descriptions(entry); + } + } + _ => {} + } +} + +/// `[`X`](path)` and `[`X`]` become `` `X` ``, `[text](path)` becomes +/// `text`; brackets inside code spans are left alone. +fn strip_doc_links(text: &str) -> String { + let mut output = String::with_capacity(text.len()); + let mut rest = text; + while let Some(index) = rest.find(['`', '[']) { + output.push_str(&rest[..index]); + rest = &rest[index..]; + if let Some(after_tick) = rest.strip_prefix('`') { + let end = after_tick.find('`').map_or(rest.len(), |end| end + 2); + output.push_str(&rest[..end]); + rest = &rest[end..]; + continue; + } + let Some(close) = rest.find(']') else { + output.push_str(rest); + return output; + }; + let inner = &rest[1..close]; + let after = &rest[close + 1..]; + if let Some(target) = after.strip_prefix('(') + && let Some(target_end) = target.find(')') + { + output.push_str(inner); + rest = &target[target_end + 1..]; + } else if inner.len() >= 2 && inner.starts_with('`') && inner.ends_with('`') { + output.push_str(inner); + rest = after; + } else { + output.push('['); + rest = &rest[1..]; + } + } + output.push_str(rest); + output +} + +/// Absent options are skipped on the wire, never sent as `null`, so the +/// `null` alternatives the schema generator adds for `Option` fields are +/// removed. +fn strip_null(value: &mut Value) { + match value { + Value::Object(map) => { + if let Some(Value::Array(types)) = map.get_mut("type") { + types.retain(|kind| kind != "null"); + if types.len() == 1 { + let only = types.remove(0); + map.insert("type".to_owned(), only); + } + } + for key in ["anyOf", "oneOf"] { + if let Some(Value::Array(variants)) = map.get_mut(key) { + variants.retain(|variant| variant.get("type") != Some(&json!("null"))); + } + } + for entry in map.values_mut() { + strip_null(entry); + } + } + Value::Array(entries) => { + for entry in entries { + strip_null(entry); + } + } + _ => {} + } +} + +fn rewrite_refs(value: &mut Value, from: &str, to: &str) { + match value { + Value::Object(map) => { + for (key, entry) in map.iter_mut() { + if key == "$ref" + && let Value::String(reference) = entry + && let Some(rest) = reference.strip_prefix(from) + { + *reference = format!("{to}{rest}"); + } else { + rewrite_refs(entry, from, to); + } + } + } + Value::Array(entries) => { + for entry in entries { + rewrite_refs(entry, from, to); + } + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn doc_links_become_plain_text() { + assert_eq!( + strip_doc_links("see [`MAX_PATH_BYTES`] and [`X`](crate::x::X) or [the docs](a::b)"), + "see `MAX_PATH_BYTES` and `X` or the docs" + ); + assert_eq!( + strip_doc_links("`[...]` is a class; [1, 2] stays"), + "`[...]` is a class; [1, 2] stays" + ); + } + + #[test] + fn exported_descriptions_carry_no_doc_links() { + for plane in export() { + let text = plane.schema_bundle.to_string(); + assert!(!text.contains("](crate::"), "{}", plane.plane); + assert!(!text.contains("[`"), "{}", plane.plane); + } + } +} diff --git a/crates/environment-protocol/src/control/handshake.rs b/crates/environment-protocol/src/control/handshake.rs index 4895f7073..07ae9f7b7 100644 --- a/crates/environment-protocol/src/control/handshake.rs +++ b/crates/environment-protocol/src/control/handshake.rs @@ -1,46 +1,51 @@ -//! Controller-plane connection handshake. +//! Controller-plane `initialize`. +//! +//! The same handshake rules as the data plane apply: `initialize` first +//! (`notInitialized` otherwise), once (`invalidRequest` on a repeat), with +//! [`CONTROLLER_PROTOCOL_VERSION`](crate::CONTROLLER_PROTOCOL_VERSION) +//! (`versionMismatch` otherwise). `template/list`, `target/create`, +//! `target/get` and `target/close` are mandatory at this version; the +//! capabilities name the optional ones. use serde::{Deserialize, Serialize}; -use crate::shared::{CURRENT_PROTOCOL_VERSION, ImplementationInfo}; +use crate::{ + handshake::{ImplementationInfo, Plane}, + serde_util::is_false, +}; +/// `initialize` params on the controller plane. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ControllerInitializeParams { - #[serde(default = "default_protocol_version")] + /// The client's `CONTROLLER_PROTOCOL_VERSION`. pub protocol_version: u32, - pub client_name: String, + pub client_info: ImplementationInfo, } +/// `initialize` response on the controller plane. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ControllerInitializeResponse { pub protocol_version: u32, + /// Always `controller`. + pub plane: Plane, + pub server_info: ImplementationInfo, pub capabilities: ControllerCapabilities, - pub implementation: ImplementationInfo, } +/// Optional controller methods. Calling an absent one is +/// `capabilityUnavailable`. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ControllerCapabilities { - #[serde(default)] - pub list_templates: bool, - #[serde(default)] - pub list_targets: bool, - #[serde(default)] - pub create_target: bool, - #[serde(default)] - pub adopt_target: bool, - #[serde(default)] - pub get_target: bool, - #[serde(default)] - pub close_target: bool, - #[serde(default)] - pub set_target_power: bool, - #[serde(default)] + /// `target/setPower` is served. + #[serde(default, skip_serializing_if = "is_false")] + pub power: bool, + /// `ingress/ensure` and `ingress/remove` are served. + #[serde(default, skip_serializing_if = "is_false")] pub ingress: bool, } - -fn default_protocol_version() -> u32 { - CURRENT_PROTOCOL_VERSION -} diff --git a/crates/environment-protocol/src/control/ingress.rs b/crates/environment-protocol/src/control/ingress.rs index 230f2f93c..ae481553f 100644 --- a/crates/environment-protocol/src/control/ingress.rs +++ b/crates/environment-protocol/src/control/ingress.rs @@ -1,40 +1,30 @@ -//! Provider-owned public ingress lifecycle payloads. +//! `ingress/*`: public ingress of one target. Requires +//! `capabilities.ingress`. `target/close` removes ingress by contract, so +//! `ingress/remove` exists only to disable ingress on a target that stays. +//! Both answer [`TargetResponse`](crate::control::target::TargetResponse) +//! with the realized ingress in the observation. The runtime calls them +//! synchronously on a user's request and caches the observed endpoint; it +//! records no ingress intent and retries nothing. use serde::{Deserialize, Serialize}; -use crate::{control::targets::ProviderBindingContext, shared::ProviderTargetId}; +use crate::control::target::TargetRef; +/// `ingress/ensure` params: route the template's public ingress to the +/// target. Idempotent. A target whose template has no `publicIngress` is +/// `failedPrecondition` (`ingressUnsupported`). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct EnsureIngressParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, + pub target: TargetRef, } +/// `ingress/remove` params: stop routing public ingress to the target. +/// Idempotent. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RemoveIngressParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ProviderIngressStatus { - Ready, - Disabled, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct IngressResponse { - pub status: ProviderIngressStatus, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub public_endpoint: Option, + pub target: TargetRef, } diff --git a/crates/environment-protocol/src/control/methods.rs b/crates/environment-protocol/src/control/methods.rs deleted file mode 100644 index b36371e7e..000000000 --- a/crates/environment-protocol/src/control/methods.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! JSON-RPC method names for the provider controller plane. - -pub const INITIALIZE_METHOD: &str = "controller/initialize"; - -pub const LIST_TEMPLATES_METHOD: &str = "controller/listTemplates"; -pub const LIST_TARGETS_METHOD: &str = "controller/listTargets"; -pub const CREATE_TARGET_METHOD: &str = "controller/createTarget"; -pub const ADOPT_TARGET_METHOD: &str = "controller/adoptTarget"; -pub const GET_TARGET_METHOD: &str = "controller/getTarget"; -pub const CLOSE_TARGET_METHOD: &str = "controller/closeTarget"; -pub const SET_TARGET_POWER_METHOD: &str = "controller/setTargetPower"; -pub const ENSURE_INGRESS_METHOD: &str = "controller/ensureIngress"; -pub const REMOVE_INGRESS_METHOD: &str = "controller/removeIngress"; diff --git a/crates/environment-protocol/src/control/mod.rs b/crates/environment-protocol/src/control/mod.rs index 1b621996d..146c54431 100644 --- a/crates/environment-protocol/src/control/mod.rs +++ b/crates/environment-protocol/src/control/mod.rs @@ -1,6 +1,44 @@ -//! Controller-plane protocol for provider target lifecycle and connection discovery. +//! The controller plane: a provider controller creating, observing, +//! powering and closing targets for Lightspeed environments. The runtime +//! is the client; the provider is the server. Calls go from Lightspeed to +//! the provider only: the provider never calls Lightspeed. +//! +//! **Credential.** When the provider is registered with a bearer +//! credential, the runtime sends it as an `Authorization: Bearer` header on +//! the controller URL and on every data route; the provider checks it in +//! constant time before the WebSocket upgrade. Without a credential no +//! `Authorization` header is sent. +//! +//! **Bounded calls.** Every target method answers `{target: observation}` +//! promptly. Slow work (an image pull, a VM boot, a graceful shutdown) +//! continues in the background and is followed with `target/get`. +//! +//! **Ownership.** A target belongs to exactly one +//! [`TargetOwner`](target::TargetOwner), `{universeId, environmentId}`. +//! Every target method carries the owner; a provider resolves the target +//! from the owner when no `targetId` is given, and refuses a target that +//! belongs to another owner with `failedPrecondition` (`ownerMismatch`). +//! +//! **Provider obligations.** The provider owns its targets' power, idle +//! policy and ingress; Lightspeed decides only create and close. Every +//! provider: +//! +//! 1. makes `target/create` and `target/close` idempotent per owner; +//! 2. wakes a powered-down target when its data route is dialed, holding +//! the upgrade at most +//! [`MAX_DATA_ROUTE_HOLD_MS`](crate::limits::MAX_DATA_ROUTE_HOLD_MS) +//! before it answers a [`DataRouteRefusal`](crate::routes::DataRouteRefusal); +//! 3. decides idleness from envd `env/getIdle`, so running processes, jobs +//! and leftover process groups keep the target awake +//! ([`idle`](crate::idle)); +//! 4. never powers a target down while it relays a data connection to it; +//! 5. reports a target it removed on its own, such as by a lifetime limit +//! or an idle `closeAfterMs`, as `closed`, or answers `notFound` from +//! `target/get` once it no longer holds it; +//! 6. never snapshots or clones an envd state directory that holds a +//! daemon identity. pub mod handshake; pub mod ingress; -pub mod methods; -pub mod targets; +pub mod target; +pub mod template; diff --git a/crates/environment-protocol/src/control/target.rs b/crates/environment-protocol/src/control/target.rs new file mode 100644 index 000000000..c31bb6b6f --- /dev/null +++ b/crates/environment-protocol/src/control/target.rs @@ -0,0 +1,190 @@ +//! `target/*`: create, observe, power and close provider targets. + +use serde::{Deserialize, Serialize}; + +use crate::{ + idle::IdlePolicy, + ids::{EnvironmentId, ProviderTargetId, TemplateId, UniverseId}, + metadata::{DescriptiveText, DisplayName}, + routes::PublicUrl, +}; + +/// The Lightspeed environment a target belongs to. A provider owns at most +/// one target per owner. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetOwner { + pub universe_id: UniverseId, + pub environment_id: EnvironmentId, +} + +/// A reference to an existing target: its owner and, once known, its id. +/// Without `targetId` the provider resolves the owner's target itself. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetRef { + pub owner: TargetOwner, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_id: Option, +} + +/// What the provider observes about a target. Every target and ingress +/// method answers with one; Lightspeed records it for display only. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetObservation { + /// The provider's id for the target. Absent in two cases only: while + /// `phase` is `provisioning` and the provider has not assigned an id + /// yet (an asynchronous create answers before it has one; the runtime + /// records the id from the first observation that carries it), and in a + /// `closed` observation of an owner the provider holds no target for. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_id: Option, + /// Echoed from the request. + pub owner: TargetOwner, + pub phase: TargetPhase, + /// Present exactly when `phase` is `active`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub power: Option, + /// Why the target is `failed` or `unknown`, or what a transition waits + /// for. Providers build it with [`DescriptiveText::lossy`] from + /// whatever their backend reports. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, + /// Public ingress as realized; absent when disabled. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ingress: Option, +} + +/// Realized public ingress of a target. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct IngressObservation { + /// Public URL that reaches the target's application. + pub endpoint: PublicUrl, +} + +/// Lifecycle phase of a target. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum TargetPhase { + /// Being created; not yet usable. + Provisioning, + /// Exists; see `power`. + Active, + /// Being destroyed. + Closing, + /// Gone, and nothing of it remains. + Closed, + /// Broken beyond the provider's own recovery; see `message`. + Failed, + /// The provider cannot tell; see `message`. + Unknown, +} + +/// Steady power state of an active target, in provider-neutral terms. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum PowerState { + Running, + /// Execution halted, memory retained; resuming is fast. + Paused, + /// Execution state persisted, compute released; resuming restores it. + Suspended, + /// Powered off; disks retained. + Stopped, +} + +impl PowerState { + pub const ALL: [PowerState; 4] = [ + PowerState::Running, + PowerState::Paused, + PowerState::Suspended, + PowerState::Stopped, + ]; + + pub fn as_str(self) -> &'static str { + match self { + Self::Running => "running", + Self::Paused => "paused", + Self::Suspended => "suspended", + Self::Stopped => "stopped", + } + } +} + +impl std::fmt::Display for PowerState { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// `target/create` params. Idempotent per owner: when the owner already +/// has a target, the provider returns its observation and applies no +/// creation policy (template deprecation, image checks). Handing out a +/// target from a warm pool counts as a create. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CreateTargetParams { + pub owner: TargetOwner, + pub template_id: TemplateId, + /// Label for the target, where the provider shows one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display_name: Option, + /// Overrides the template's `idle` for this target. A policy that does + /// not validate ([`IdlePolicy::validate`]) is `invalidParams`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub idle: Option, +} + +/// `target/get` params. A pure read: it never creates or changes anything. +/// A target that no longer exists is `notFound`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GetTargetParams { + pub target: TargetRef, +} + +/// `target/setPower` params: move the target to `power`. Requires +/// `capabilities.power`. A target already in, or moving toward, that state +/// is observed without further action. Resuming may replace the underlying +/// host process. A state outside the template's `powerStates` is +/// `invalidParams`; a target whose `phase` is not `active` is +/// `failedPrecondition` (`targetNotActive`). The provider stays the owner +/// of power afterwards: its own idle policy may move the target again. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SetTargetPowerParams { + pub target: TargetRef, + /// One of the template's `powerStates`. + pub power: PowerState, +} + +/// `target/close` params: destroy the owner's target and every resource it +/// holds, public ingress included. Idempotent. Graceful within the +/// provider's own bound, then forced. Answers `closing` until the target is +/// gone, then `closed`; `closed` also when the provider holds no target for +/// the owner. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CloseTargetParams { + pub target: TargetRef, +} + +/// Response of every target method and both ingress methods. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TargetResponse { + pub target: TargetObservation, +} diff --git a/crates/environment-protocol/src/control/targets.rs b/crates/environment-protocol/src/control/targets.rs deleted file mode 100644 index 742c20434..000000000 --- a/crates/environment-protocol/src/control/targets.rs +++ /dev/null @@ -1,250 +0,0 @@ -//! Controller-plane target lifecycle payloads. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; - -use crate::shared::{EnvironmentCapabilities, EnvironmentPath, EnvironmentScope, ProviderTargetId}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProviderBindingContext { - pub universe_id: String, - pub binding_id: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentTemplate { - pub template_id: String, - pub display_name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(default)] - pub public_ingress: bool, - #[serde(default)] - pub deprecated: bool, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListTemplatesParams { - pub binding: ProviderBindingContext, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListTemplatesResponse { - pub templates: Vec, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListTargetsParams { - pub binding: ProviderBindingContext, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListTargetsResponse { - pub targets: Vec, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreateTargetParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub template_id: String, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreateTargetResponse { - pub target: ProviderTargetSummary, -} - -/// Explicit ownership transfer of an existing provider target into a -/// Lightspeed-managed binding namespace. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AdoptTargetParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - /// Provider-native reference to the existing target. Incus uses - /// `/` and defaults the project to `default`. - pub source_target: String, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AdoptTargetResponse { - pub target: ProviderTargetSummary, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetTargetParams { - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetTargetResponse { - pub target: ProviderTargetSummary, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CloseTargetParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, - #[serde(default)] - pub force: bool, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CloseTargetResponse { - pub target_id: ProviderTargetId, - pub status: ProviderTargetStatus, -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SandboxTargetSpec { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub template: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub image: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cwd: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub env: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub labels: BTreeMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_options: Option, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProviderTargetSummary { - pub target_id: ProviderTargetId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub display_name: Option, - pub status: ProviderTargetStatus, - pub scope: EnvironmentScope, - pub capabilities: EnvironmentCapabilities, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_cwd: Option, - /// Power states this target can be moved to with `controller/setTargetPower`. - /// Empty when the provider offers no power control beyond create/close. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub power_states: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ProviderTargetStatus { - Creating, - Starting, - Ready, - /// VMM alive with execution frozen; RAM stays resident, resume is instant. - Paused, - /// Execution state saved to disk and the VMM released; resume restores it. - Suspended, - Stopped, - Closing, - Closed, - Failed, - Unknown, -} - -impl ProviderTargetStatus { - /// The power state a steady-state observation corresponds to, or `None` - /// while the target is transitioning or gone. - pub fn power_state(self) -> Option { - match self { - Self::Ready => Some(PowerState::Running), - Self::Paused => Some(PowerState::Paused), - Self::Suspended => Some(PowerState::Suspended), - Self::Stopped => Some(PowerState::Stopped), - Self::Creating - | Self::Starting - | Self::Closing - | Self::Closed - | Self::Failed - | Self::Unknown => None, - } - } -} - -/// Requested steady power state of a provider target. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum PowerState { - Running, - Paused, - Suspended, - Stopped, -} - -impl PowerState { - pub const ALL: [PowerState; 4] = [ - PowerState::Running, - PowerState::Paused, - PowerState::Suspended, - PowerState::Stopped, - ]; - - pub fn as_str(self) -> &'static str { - match self { - Self::Running => "running", - Self::Paused => "paused", - Self::Suspended => "suspended", - Self::Stopped => "stopped", - } - } -} - -impl std::fmt::Display for PowerState { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.as_str()) - } -} - -/// Converge one target to a steady power state. Idempotent by inventory: a -/// target already in `power` (or transitioning toward it) is reported as -/// observed without further action; resuming may replace the underlying -/// host process. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SetTargetPowerParams { - pub request_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub binding: ProviderBindingContext, - pub target_id: ProviderTargetId, - pub power: PowerState, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SetTargetPowerResponse { - pub target: ProviderTargetSummary, -} diff --git a/crates/environment-protocol/src/control/template.rs b/crates/environment-protocol/src/control/template.rs new file mode 100644 index 000000000..56ab76caa --- /dev/null +++ b/crates/environment-protocol/src/control/template.rs @@ -0,0 +1,74 @@ +//! `template/list`: the templates a universe may create targets from. + +use serde::{Deserialize, Serialize}; + +use crate::{ + control::target::PowerState, + idle::IdlePolicy, + ids::{TemplateId, UniverseId}, + metadata::{DescriptiveText, DisplayName, Metadata}, + serde_util::is_false, +}; + +/// `template/list` params: the universe whose targets the templates would +/// create. A provider may offer each universe a different set. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListTemplatesParams { + pub universe_id: UniverseId, +} + +/// `template/list` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListTemplatesResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub templates: Vec, +} + +/// A template targets are created from. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct EnvironmentTemplate { + pub template_id: TemplateId, + pub display_name: DisplayName, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Power states `target/setPower` can move this template's targets to. + /// Empty when the provider offers no power control beyond create and + /// close. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub power_states: Vec, + /// Targets of this template can serve public ingress. + #[serde(default, skip_serializing_if = "is_false")] + pub public_ingress: bool, + /// Idle policy targets get when `target/create` names none; absent when + /// the provider applies none by default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub idle: Option, + /// Creates no new targets; existing targets keep working. + #[serde(default, skip_serializing_if = "is_false")] + pub deprecated: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resources: Option, + /// Provider-specific display facts, such as an image fingerprint. + #[serde(default, skip_serializing_if = "Metadata::is_empty")] + pub metadata: Metadata, +} + +/// Compute a template's targets get, in provider-neutral units. Each +/// field is absent when the provider does not fix it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TemplateResources { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vcpus: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub disk_bytes: Option, +} diff --git a/crates/environment-protocol/src/data/events.rs b/crates/environment-protocol/src/data/events.rs deleted file mode 100644 index 0f7fefe96..000000000 --- a/crates/environment-protocol/src/data/events.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! Data-plane notifications. - -use serde::{Deserialize, Serialize}; - -use crate::{ - data::process::{ProcessOutputChunk, ProcessOutputStream}, - shared::{ByteChunk, ProcessId}, -}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessOutputNotification { - pub process_id: ProcessId, - pub seq: u64, - pub stream: ProcessOutputStream, - pub chunk: ByteChunk, -} - -impl ProcessOutputNotification { - pub fn from_chunk(process_id: ProcessId, chunk: ProcessOutputChunk) -> Self { - Self { - process_id, - seq: chunk.seq, - stream: chunk.stream, - chunk: chunk.chunk, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessExitedNotification { - pub process_id: ProcessId, - pub seq: u64, - pub exit_code: i32, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessClosedNotification { - pub process_id: ProcessId, - pub seq: u64, -} diff --git a/crates/environment-protocol/src/data/exec.rs b/crates/environment-protocol/src/data/exec.rs new file mode 100644 index 000000000..bde6edfe8 --- /dev/null +++ b/crates/environment-protocol/src/data/exec.rs @@ -0,0 +1,373 @@ +//! The execution primitive processes and jobs share: the command, its +//! input, its output stream and its outcome. +//! +//! **Output positions.** A handle's output is one byte stream: stdout, +//! stderr and the PTY interleaved in arrival order. Positions are byte +//! offsets into it. `end` is the number of bytes produced so far; a page +//! cut at `maxBytes` loses nothing because the next read starts at `next`; +//! a tail read starts at `end - tailBytes`. Bytes the daemon no longer +//! retains are reported as typed [`OutputGap`]s, never skipped silently. +//! +//! **Settled exit.** `state` becomes `exited` only once the root process +//! has exited and its output has settled (every reader reached end of file +//! or was discarded), so `end` is final from then on. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::{ + bytes::{ByteChunk, SecretString}, + paths::EnvironmentPath, + serde_util::{is_false, is_zero_u32}, +}; + +/// What to run. Shared by `process/start` and `job/start`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CommandSpec { + /// Program and arguments; `argv[0]` is resolved through `PATH`. Not + /// empty. + pub argv: Vec, + /// Working directory; the daemon's `defaultCwd` when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cwd: Option, + /// Variables added to the daemon's scrubbed environment. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, + /// Like `env`, but every value is redacted from the command's output. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub secret_env: BTreeMap, + /// Standard input; none by default. + #[serde(default, skip_serializing_if = "ExecInput::is_none")] + pub input: ExecInput, + /// Stop deadline from start. On expiry the daemon terminates the process + /// group (`SIGTERM`, the advertised grace, then `SIGKILL`) and records + /// `stoppedBy: timeout`. Absent means no deadline. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + +/// Standard input of a command. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ExecInput { + /// `/dev/null`: the command reads end of file. Model-facing tools use + /// this. + None {}, + /// These bytes, then end of file. Written by a writer task after the + /// output readers, the exit watcher and the deadline are in place. + Bytes { data: ByteChunk }, + /// A persistent pipe fed by `process/write` until closed. Processes + /// only; `job/start` refuses it with `invalidParams`. + Pipe {}, +} + +impl Default for ExecInput { + fn default() -> Self { + Self::None {} + } +} + +impl ExecInput { + pub fn is_none(&self) -> bool { + matches!(self, Self::None {}) + } +} + +/// Terminal size of a PTY. Each dimension is 1 to 4096. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PtySize { + pub rows: u32, + pub cols: u32, +} + +impl PtySize { + pub const MAX_DIMENSION: u32 = 4096; +} + +/// Which descriptor produced a chunk. Jobs never produce `pty`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum OutputStream { + Stdout, + Stderr, + /// The PTY master, which merges stdout and stderr. + Pty, +} + +/// Contiguous output bytes from one stream. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OutputChunk { + /// Byte offset of the chunk's first byte in the combined output. + pub offset: u64, + pub stream: OutputStream, + pub data: ByteChunk, +} + +/// Output bytes in `[offset, offset + omittedBytes)` that a page cannot +/// return. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OutputGap { + pub offset: u64, + pub omitted_bytes: u64, + pub reason: GapReason, +} + +/// Why a range of output cannot be returned. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum GapReason { + /// Never delivered: dropped from the middle because retained output + /// exceeded the handle's cap. The head and the newest bytes are kept. + Overflow, + /// Already delivered through the daemon cursor (or acknowledged) and + /// released since. + Expired, +} + +/// One page of a handle's output. Returned by `process/read` and in every +/// `job/read` entry. A page holds at most +/// [`MAX_PAGE_CHUNKS`](crate::limits::MAX_PAGE_CHUNKS) chunks and gaps +/// together (for `job/read`, across all entries); one that reaches it ends +/// early, and `next` says where to continue. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OutputPage { + /// In offset order, at most `maxBytes` of `data` in total. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub chunks: Vec, + /// Offset to read from next: past the last returned byte or gap. + pub next: u64, + /// Bytes produced so far; final once the handle has exited. + pub end: u64, + /// Unreturnable ranges between the requested start and `next`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub gaps: Vec, +} + +/// When a waiting read returns. Every mode also returns when `waitMs` +/// elapses, when the handle reaches a settled exit, or at once when there +/// is nothing left to wait for. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ReturnOn { + /// Collect for the whole window; return early only when unread output + /// reaches `maxBytes`. + Window, + /// Return as soon as any output past the start position exists. + AnyOutput, + /// Return only at the settled exit (for `job/read`: once every entry is + /// terminal), even if more than `maxBytes` of output accumulates. + Exit, +} + +/// Lifecycle state of a process. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ExecState { + Running, + /// The root exited and its output settled. + Exited, +} + +/// How the root process ended. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ExitStatus { + /// Normal exit with this status code. + Code { code: i32 }, + /// Killed by this signal. + Signal { + signal: SignalName, + #[serde(default, skip_serializing_if = "is_false")] + core_dumped: bool, + }, +} + +/// Why the daemon stopped a process or job that did not exit on its own. +/// Set only when the daemon signalled a root that was still running. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum StoppedBy { + /// `timeoutMs` expired: `SIGTERM`, the grace, then `SIGKILL`. + Timeout, + /// `process/terminate` with `terminate`, or `job/cancel` without + /// `force`: `SIGTERM`, the grace, then `SIGKILL`. + Terminate, + /// `process/terminate` with `kill`, or `job/cancel` with `force`: + /// `SIGKILL` at once. + Kill, +} + +/// A POSIX signal name such as `SIGSEGV` or `SIGRTMIN+3`: `SIG` followed by +/// 1 to 16 uppercase letters, digits or `+`. Names, not numbers, because +/// numbers differ between platforms. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct SignalName( + #[cfg_attr(feature = "schema", schemars(pattern(SIGNAL_NAME_PATTERN)))] String, +); + +/// JSON Schema pattern of [`SignalName`]. +pub const SIGNAL_NAME_PATTERN: &str = "^SIG[A-Z0-9+]{1,16}$"; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +#[error("signal name must be SIG followed by 1..=16 uppercase letters, digits or '+': {0:?}")] +pub struct SignalNameError(pub String); + +impl SignalName { + pub fn new(name: impl Into) -> Result { + let name = name.into(); + let valid = name.strip_prefix("SIG").is_some_and(|rest| { + (1..=16).contains(&rest.len()) + && rest + .bytes() + .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'+') + }); + if valid { + Ok(Self(name)) + } else { + Err(SignalNameError(name)) + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for SignalName { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(formatter) + } +} + +impl Serialize for SignalName { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for SignalName { + fn deserialize>(deserializer: D) -> Result { + let name = String::deserialize(deserializer)?; + Self::new(name).map_err(serde::de::Error::custom) + } +} + +/// Processes of a command's group still alive after its root exited. +/// Present only when there is at least one. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct LeftoverReport { + pub disposition: LeftoverDisposition, + /// At most [`MAX_LEFTOVER_PROCESSES`](crate::limits::MAX_LEFTOVER_PROCESSES), + /// lowest pid first; never empty. + pub processes: Vec, + /// Leftovers beyond the listed ones. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub omitted_process_count: u32, +} + +/// What the daemon did with leftovers. The policies differ on purpose. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum LeftoverDisposition { + /// Left running (processes): they keep the environment's lifetime and + /// are reported until they exit. + Kept, + /// Terminated when the job finished (jobs); the submitter should fix + /// its script's teardown. + Swept, +} + +/// One leftover process. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct LeftoverProcess { + pub pid: u32, + /// Command line when the platform exposes it, otherwise the executable + /// name, cut to + /// [`MAX_LEFTOVER_COMMAND_BYTES`](crate::limits::MAX_LEFTOVER_COMMAND_BYTES) + /// on a character boundary; absent when neither could be read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub command: Option, +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn exit_status_names_signals() { + let status = ExitStatus::Signal { + signal: SignalName::new("SIGSEGV").unwrap(), + core_dumped: true, + }; + assert_eq!( + serde_json::to_value(&status).unwrap(), + json!({"kind": "signal", "signal": "SIGSEGV", "coreDumped": true}) + ); + assert!(SignalName::new("SIGRTMIN+3").is_ok()); + for bad in ["SEGV", "SIG", "sigsegv", "SIGSEGV!"] { + assert!(SignalName::new(bad).is_err(), "{bad}"); + } + } + + #[test] + fn input_defaults_to_none_and_is_omitted() { + let spec: CommandSpec = serde_json::from_value(json!({"argv": ["true"]})).unwrap(); + assert!(spec.input.is_none()); + assert_eq!( + serde_json::to_value(&spec).unwrap(), + json!({"argv": ["true"]}) + ); + let piped: CommandSpec = + serde_json::from_value(json!({"argv": ["cat"], "input": {"kind": "pipe"}})).unwrap(); + assert_eq!(piped.input, ExecInput::Pipe {}); + assert!(serde_json::from_value::(json!({"argv": [], "stdin": "x"})).is_err()); + } + + #[test] + fn fieldless_input_kinds_reject_unknown_members() { + for kind in ["none", "pipe"] { + assert!(serde_json::from_value::(json!({"kind": kind})).is_ok()); + assert!( + serde_json::from_value::(json!({"kind": kind, "data": "aGk="})).is_err(), + "{kind}" + ); + } + } +} diff --git a/crates/environment-protocol/src/data/fs.rs b/crates/environment-protocol/src/data/fs.rs index 8ff51c0f9..075ec6075 100644 --- a/crates/environment-protocol/src/data/fs.rs +++ b/crates/environment-protocol/src/data/fs.rs @@ -1,232 +1,529 @@ -//! Filesystem method payloads. +//! `fs/*`: path-based filesystem methods. +//! +//! Every path is an absolute [`EnvironmentPath`] resolved under the +//! confinement rule stated on that type. Methods that change the +//! filesystem need `capabilities.filesystem.write`; without it they answer +//! `capabilityUnavailable` (`filesystemWrite`). +//! +//! **Pattern grammar** ([`TreeFilter`]), shared by `fs/searchText`, +//! `fs/globFiles`, `fs/scan` and transfer captures. Patterns match paths +//! relative to the walk's root, gitignore-style: `*` and `?` match within +//! one segment, `**` spans segments, `[...]` is a character class and +//! `{a,b}` an alternation. A pattern without `/` matches the entry's name +//! at any depth; a leading `/` anchors it to the root. Walks always prune +//! VCS metadata directories (`.git`, `.hg`, `.svn`, `.jj`) and the daemon +//! state directory; by default they also respect ignore files (`.gitignore`, +//! `.ignore`, git excludes) and skip hidden entries. An entry a walk cannot +//! represent or read (a name that is not UTF-8, a directory that cannot be +//! listed) is skipped and counted in the result's `skippedEntryCount`; +//! it never ends the walk. +//! +//! **Two entry shapes, on purpose.** [`FileMetadata`] describes one path as +//! it is now (`fs/getMetadata`, `fs/readDirectory`): kind including +//! symlinks, times and a `version` for preconditions. [`TreeContent`] +//! describes what a tree holds (`fs/scan` entries, transfer inventories): +//! directories and files with size, mode and digest, the facts two trees +//! are compared by. Trees never hold symlink entries: scans skip or follow +//! links and captures expand or skip them. use serde::{Deserialize, Serialize}; -use crate::shared::{ByteChunk, EnvironmentPath}; +use crate::{ + bytes::ByteChunk, + ids::{DirectoryCursor, validate_opaque_id}, + paths::{ContentDigest, EntryName, EnvironmentPath}, + serde_util::{is_false, is_zero_u32}, +}; -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Kind of a filesystem entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct ReadFileParams { - pub path: EnvironmentPath, - /// Byte offset to start reading from. Environment endpoints advertising - /// `filesystem_ranged_read` honor it; others ignore it and return the - /// full file. +pub enum FileKind { + File, + Directory, + Symlink, + /// Device, FIFO, socket or another special file. + Other, +} + +/// One file-entry shape for `fs/getMetadata` and `fs/readDirectory`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct FileMetadata { + pub kind: FileKind, + /// Size in bytes; for a symlink, the length of its target text. + pub size_bytes: u64, + /// A regular file with any execute bit set; always false for + /// directories, symlinks and special files. + #[serde(default, skip_serializing_if = "is_false")] + pub executable: bool, + pub modified_at_ms: i64, + /// Absent when the filesystem records no birth time. #[serde(default, skip_serializing_if = "Option::is_none")] - pub offset: Option, - /// Maximum bytes to return. Environment endpoints advertising `filesystem_ranged_read` - /// truncate at the source and set `truncated`. + pub created_at_ms: Option, + /// Target text of a symlink, when it is UTF-8. #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_bytes: Option, + pub symlink_target: Option, + /// Changes whenever the entry's content or identity changes; compare it + /// with `ifMatch`. + pub version: FileVersion, +} + +/// Opaque version token of a file's content and identity. Equal tokens for +/// one path mean the content is unchanged. 1 to 255 bytes without control +/// characters. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct FileVersion( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 255), pattern(crate::ids::NO_CONTROL_PATTERN)) + )] + String, +); + +impl FileVersion { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + validate_opaque_id("FileVersion", &value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Serialize for FileVersion { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for FileVersion { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(serde::de::Error::custom) + } } +/// What a tree entry is: a scan entry or a transfer inventory entry. +/// Digests identify raw file bytes, not paths or modes. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum TreeContent { + Directory {}, + File { + size_bytes: u64, + #[serde(default, skip_serializing_if = "is_false")] + executable: bool, + /// Scans: present when `includeDigests` was set. Transfer + /// inventories: always present; an appended file entry without it + /// is `invalidParams`. + #[serde(default, skip_serializing_if = "Option::is_none")] + digest: Option, + }, +} +/// `fs/readFile` params. Reads a regular file only; anything else is +/// `failedPrecondition` (`notRegularFile`) and is never opened for a +/// blocking read. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadFileParams { + pub path: EnvironmentPath, + /// First byte to return; 0 by default. + #[serde(default, skip_serializing_if = "crate::serde_util::is_zero_u64")] + pub offset: u64, + /// Most bytes to return, at most `limits.filesystem.maxReadFileBytes`. + pub max_bytes: u64, +} + +/// `fs/readFile` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReadFileResponse { pub data: ByteChunk, - /// Total size of the file in the environment, independent of the returned range. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub file_size: Option, - /// True when the returned data is a strict prefix range of the file. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] + /// Size of the whole file when it was read. + pub file_size: u64, + /// Bytes exist after `offset` plus the length of `data`. + #[serde(default, skip_serializing_if = "is_false")] pub truncated: bool, + pub version: FileVersion, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// When `fs/writeFile` may write. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] +pub enum WriteCondition { + /// Create or replace. + #[default] + Always, + /// Only create; an existing entry is `failedPrecondition` + /// (`createCondition`). + IfAbsent, + /// Only replace; a missing file is `failedPrecondition` + /// (`createCondition`). + IfPresent, +} + +impl WriteCondition { + pub fn is_default(&self) -> bool { + *self == Self::Always + } +} + +/// `fs/writeFile` params. The write is staged in the target's directory and +/// published by an atomic rename: readers see the old or the new content, +/// never a mix. An existing target keeps its permissions. A symlink at +/// `path` whose target stays inside the root is followed like any path: the +/// final target is replaced and the link itself is kept (a dangling link +/// creates its target, subject to `create`); a link that leaves the root is +/// `permissionDenied` (`outsideRoot`). A special file is +/// `failedPrecondition` (`notRegularFile`), a directory +/// `failedPrecondition` (`isDirectory`). The parent of the final target +/// must exist. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct WriteFileParams { pub path: EnvironmentPath, + /// At most `limits.filesystem.maxWriteFileBytes`; larger files go + /// through `transfer/*` with a `writeFile` selection. pub data: ByteChunk, + #[serde(default, skip_serializing_if = "WriteCondition::is_default")] + pub create: WriteCondition, + /// Write only when the current file's version equals this token; + /// otherwise `failedPrecondition` (`ifMatchFailed`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub if_match: Option, + /// Set the execute bits (the read bits' pattern). Without it a new file + /// gets the default mode and an existing file keeps its mode. + #[serde(default, skip_serializing_if = "is_false")] + pub executable: bool, } -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct WriteFileResponse {} +/// `fs/writeFile` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WriteFileResponse { + /// Version of the written file. + pub version: FileVersion, +} +/// `fs/createDirectory` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct CreateDirectoryParams { pub path: EnvironmentPath, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub recursive: Option, + /// Create missing parents, and succeed when the directory exists. + #[serde(default, skip_serializing_if = "is_false")] + pub recursive: bool, } +/// `fs/createDirectory` response. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct CreateDirectoryResponse {} +/// `fs/getMetadata` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GetMetadataParams { pub path: EnvironmentPath, + /// Describe the symlink's target instead of the link (subject to + /// confinement). + #[serde(default, skip_serializing_if = "is_false")] + pub follow_symlinks: bool, } +/// `fs/getMetadata` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GetMetadataResponse { - pub is_directory: bool, - pub is_file: bool, - pub is_symlink: bool, - pub created_at_ms: i64, - pub modified_at_ms: i64, + pub metadata: FileMetadata, } +/// `fs/readDirectory` params. Pages in name order. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReadDirectoryParams { + /// A directory, or a symlink to one inside the root. pub path: EnvironmentPath, + /// Most entries per page, 1 to `limits.filesystem.maxDirectoryEntries`. + pub limit: u32, + /// `nextCursor` of the previous page; a cursor this daemon did not mint + /// is `invalidParams`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cursor: Option, } +/// `fs/readDirectory` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReadDirectoryEntry { - pub file_name: String, - pub is_directory: bool, - pub is_file: bool, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadDirectoryResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub entries: Vec, + /// Opaque cursor of the next page; absent on the last page. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_cursor: Option, + /// Entries in this page's range skipped because their names are not + /// UTF-8. Entries that vanish while the page is built are left out and + /// not counted. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_entry_count: u32, } +/// One `fs/readDirectory` entry. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReadDirectoryResponse { - pub entries: Vec, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DirectoryEntry { + pub name: EntryName, + /// The entry itself, not a symlink's target. Entries that vanish while + /// the page is built are left out. + pub metadata: FileMetadata, } +/// `fs/remove` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RemoveParams { + /// A symlink is removed, never followed. pub path: EnvironmentPath, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub recursive: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub force: Option, + /// Remove a directory and its contents. + #[serde(default, skip_serializing_if = "is_false")] + pub recursive: bool, + /// Succeed when the path does not exist. + #[serde(default, skip_serializing_if = "is_false")] + pub force: bool, } +/// `fs/remove` response. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RemoveResponse {} +/// `fs/rename` params. Atomic within one filesystem; across filesystems it +/// is `failedPrecondition` (`crossDevice`). The entry keeps its mode. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CopyParams { - pub source_path: EnvironmentPath, - pub destination_path: EnvironmentPath, - #[serde(default)] - pub recursive: bool, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RenameParams { + pub source: EnvironmentPath, + pub destination: EnvironmentPath, + /// Replace an existing destination file; otherwise an existing + /// destination is `alreadyExists`. A directory destination is never + /// replaced. + #[serde(default, skip_serializing_if = "is_false")] + pub overwrite: bool, + /// Rename only when the source's version equals this token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub if_match: Option, } +/// `fs/rename` response. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CopyResponse {} +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RenameResponse {} + +/// Which entries a walk visits. The default visits everything except +/// hidden and ignored entries. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TreeFilter { + /// Select entries matching any pattern; empty selects every entry. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub include: Vec, + /// Drop entries matching any pattern; a matching directory is pruned. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude: Vec, + /// Match patterns case-insensitively. + #[serde(default, skip_serializing_if = "is_false")] + pub ignore_case: bool, + /// Visit entries whose name starts with `.`. + #[serde(default, skip_serializing_if = "is_false")] + pub include_hidden: bool, + /// Visit entries that ignore files exclude. + #[serde(default, skip_serializing_if = "is_false")] + pub include_ignored: bool, + /// Deepest entry depth visited: the root's direct children are depth + /// 1, so `1` selects direct children only. Unbounded when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_depth: Option, +} + +impl TreeFilter { + pub fn is_default(&self) -> bool { + *self == Self::default() + } +} -/// Bounded recursive text search executed natively by the environment endpoint. -/// -/// The environment endpoint performs traversal and matching locally and returns only bounded -/// matches plus scan statistics, so a broad search does not become per-file -/// transfer over the data plane. Every limit is mandatory: an endpoint must stop -/// scanning when any bound is reached and report why. +/// `fs/searchText` params: a bounded recursive regex search run by the +/// daemon. A file larger than the remaining byte budget is skipped and +/// counted; it never ends the walk. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchTextParams { pub root: EnvironmentPath, - /// Regular expression (Rust `regex` dialect, the same dialect the - /// caller-side fallback uses). + /// Regular expression in the Rust `regex` crate's syntax, matched per + /// line. pub pattern: String, - /// Optional glob filter applied to the root-relative path or file name. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub include: Option, - #[serde(default)] - pub case_sensitive: bool, - /// Maximum directory depth below the root, matching the generic - /// traversal's semantics; absent means unbounded depth. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_depth: Option, + /// Match `pattern` case-insensitively. `filter.ignoreCase` governs the + /// path patterns instead. + #[serde(default, skip_serializing_if = "is_false")] + pub pattern_ignore_case: bool, + #[serde(default, skip_serializing_if = "TreeFilter::is_default")] + pub filter: TreeFilter, pub limits: SearchTextLimits, } +/// Caller quotas of one search; each at most its advertised ceiling. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchTextLimits { - pub max_matches: u64, - /// Maximum number of files searched (not merely visited by traversal). - pub max_files: u64, - /// Maximum cumulative bytes searched across files. + pub max_matches: u32, + /// Files searched, not merely visited. + pub max_files: u32, + /// Bytes actually read across files. pub max_bytes: u64, pub max_duration_ms: u64, + /// Longer matching lines are cut and marked `lineTruncated`. + pub max_line_bytes: u32, } +/// `fs/searchText` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchTextResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub matches: Vec, + pub searched_file_count: u32, + pub searched_bytes: u64, + /// Files skipped because they exceeded the remaining byte budget, are + /// binary or could not be read. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_file_count: u32, + /// Entries the walk could not represent or list: names that are not + /// UTF-8 and directories that could not be read. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_entry_count: u32, + pub elapsed_ms: u64, + /// Absent when the walk finished within its limits. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stopped: Option, +} + +/// One matching line. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SearchTextMatch { pub path: EnvironmentPath, - pub line_number: u64, + /// 1-based. + pub line_number: u32, + /// The matching line without its terminator, cut at `maxLineBytes` on a + /// character boundary. pub line: String, + #[serde(default, skip_serializing_if = "is_false")] + pub line_truncated: bool, } -/// Why a search stopped before exhausting the tree. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Why a search stopped before finishing its walk. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] pub enum SearchTextStop { + /// `maxMatches` matches were found. MatchLimit, + /// `maxFiles` files were searched. FileLimit, + /// `maxBytes` bytes were read. ByteLimit, + /// `maxDurationMs` ran out. TimeLimit, + /// The next match would push the encoded matches past + /// [`MAX_RESULT_BYTES`](crate::limits::MAX_RESULT_BYTES). + ResponseLimit, } +/// `fs/globFiles` params: bounded recursive enumeration of non-directory +/// entries matching `filter.include`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SearchTextResponse { - pub matches: Vec, - pub files_searched: u64, - pub bytes_searched: u64, - pub elapsed_ms: u64, - /// Absent when the search exhausted the tree within its limits. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stopped: Option, -} - -/// Bounded recursive file enumeration executed natively by the environment endpoint. -/// -/// Pattern semantics mirror the caller-side generic glob tool: a pattern -/// starting with `/` matches the caller-space absolute path; otherwise the -/// root-relative path matches, and a pattern without `/` also matches bare -/// file names. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GlobFilesParams { pub root: EnvironmentPath, - pub pattern: String, - /// Maximum directory depth below the root, matching the generic - /// traversal's semantics; absent means unbounded depth. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_depth: Option, + /// `include` must not be empty. + pub filter: TreeFilter, pub limits: GlobFilesLimits, } +/// Caller quotas of one enumeration; each at most its advertised ceiling. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GlobFilesLimits { - pub max_matches: u64, - /// Maximum directory entries (files and directories) visited by the - /// traversal. - pub max_entries: u64, + pub max_matches: u32, + /// Non-directory entries visited. + pub max_files: u32, pub max_duration_ms: u64, } -/// Why an enumeration stopped before exhausting the tree. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum GlobFilesStop { - MatchLimit, - EntryLimit, - TimeLimit, -} - +/// `fs/globFiles` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct GlobFilesResponse { + /// In walk order (sorted by path). + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub matches: Vec, - pub entries_visited: u64, + pub visited_file_count: u32, + /// Entries the walk could not represent or list: names that are not + /// UTF-8 and directories that could not be read. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_entry_count: u32, pub elapsed_ms: u64, - /// Absent when the enumeration exhausted the tree within its limits. #[serde(default, skip_serializing_if = "Option::is_none")] pub stopped: Option, } + +/// Why an enumeration stopped before finishing its walk. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum GlobFilesStop { + /// `maxMatches` paths were found. + MatchLimit, + /// `maxFiles` non-directory entries were visited. + FileLimit, + /// `maxDurationMs` ran out. + TimeLimit, + /// The next match would push the encoded matches past + /// [`MAX_RESULT_BYTES`](crate::limits::MAX_RESULT_BYTES). + ResponseLimit, +} diff --git a/crates/environment-protocol/src/data/handshake.rs b/crates/environment-protocol/src/data/handshake.rs index 2a59d46e2..5def4fa5c 100644 --- a/crates/environment-protocol/src/data/handshake.rs +++ b/crates/environment-protocol/src/data/handshake.rs @@ -1,41 +1,230 @@ -//! Data-plane connection handshake. +//! Data-plane `initialize`. +//! +//! `initialize` is the first request on every data connection; any other +//! method before it is `notInitialized`, and a second `initialize` is +//! `invalidRequest`. A `protocolVersion` other than +//! [`ENVD_PROTOCOL_VERSION`](crate::ENVD_PROTOCOL_VERSION) is +//! `versionMismatch`. The connection stays open after all three. use serde::{Deserialize, Serialize}; -use crate::shared::{ - CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentConnectionId, EnvironmentScope, - ImplementationInfo, +use crate::{ + handshake::{ImplementationInfo, Plane}, + ids::DaemonInstanceId, + paths::EnvironmentPath, + serde_util::is_false, }; +/// `initialize` params on the data plane. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct InitializeParams { - #[serde(default = "default_protocol_version")] + /// The client's `ENVD_PROTOCOL_VERSION`. pub protocol_version: u32, - pub client_name: String, - pub scope: EnvironmentScope, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub resume_connection_id: Option, + pub client_info: ImplementationInfo, } +/// `initialize` response on the data plane. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct InitializeResponse { pub protocol_version: u32, - pub connection_id: EnvironmentConnectionId, - pub capabilities: EnvironmentCapabilities, + /// Always `data`. + pub plane: Plane, + /// Random per daemon start; a different value than on an earlier + /// connection means the daemon restarted and the earlier instance's + /// process handles are gone. + pub daemon_instance_id: DaemonInstanceId, + pub server_info: ImplementationInfo, + pub capabilities: DataCapabilities, + pub limits: DataLimits, + /// The daemon's working directory: the default `cwd` of commands. + pub default_cwd: EnvironmentPath, + /// Home directory of the user commands run as, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub home_directory: Option, +} + +/// Capability families. A present family means its base methods exist; a +/// family's fields are facts that vary per instance. A request to an +/// absent family, or one needing an absent fact, is +/// `capabilityUnavailable`. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DataCapabilities { + /// `fs/*` (search, glob and scan included) and `transfer/*`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filesystem: Option, + /// `process/*`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_cwd: Option, - /// Home directory of the endpoint execution user. + pub process: Option, + /// `job/*`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub home_directory: Option, - pub implementation: ImplementationInfo, + pub job: Option, } +/// The `filesystem` family: `fs/*` and `transfer/*`. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InitializedParams {} +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct FilesystemCapabilities { + /// Methods that change the filesystem are served; false on a daemon + /// started read-only. + #[serde(default, skip_serializing_if = "is_false")] + pub write: bool, +} + +/// The `process` family: `process/*`. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ProcessCapabilities { + /// `process/start` accepts `pty`. + #[serde(default, skip_serializing_if = "is_false")] + pub pty: bool, +} + +/// The `job` family: `job/*`. It has no instance facts. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobCapabilities {} + +/// Operational ceilings this daemon enforces. A request above a ceiling is +/// `invalidParams` naming the field and the ceiling; nothing is clamped. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DataLimits { + /// Requests in flight per connection before `busy`; at least + /// [`MIN_CONCURRENT_REQUESTS`](crate::limits::MIN_CONCURRENT_REQUESTS). + pub max_concurrent_requests: u32, + /// Largest `waitMs` of `process/read` and `job/read`. + pub max_wait_ms: u64, + pub process: ProcessLimits, + pub job: JobLimits, + pub filesystem: FilesystemLimits, +} + +/// Ceilings of `process/*`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ProcessLimits { + /// Largest `maxBytes` of one `process/read`. + pub max_read_bytes: u64, + /// Output a process retains; beyond it the oldest delivered bytes are + /// released (`expired` gaps), then the undelivered middle (`overflow` + /// gaps). + pub max_retained_output_bytes: u64, + /// How long an exited process stays readable after a read first + /// returned its settled exit with nothing left unread. + pub retention_after_exit_ms: u64, + /// How long an exited process whose output was never fully read stays + /// readable. + pub unread_retention_ms: u64, + /// Output retained across all exited processes; beyond it the oldest + /// are evicted early. + pub max_exited_retained_bytes: u64, + /// Grace between `SIGTERM` and `SIGKILL` for `terminate` and timeouts. + pub terminate_grace_ms: u64, + /// Bytes queued for a process's input before `process/write` accepts + /// fewer than it was sent. + pub max_input_queue_bytes: u64, +} + +/// Ceilings of `job/*`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobLimits { + /// Largest `maxBytes` of one `job/read`, across its entries. + pub max_read_bytes: u64, + /// Most jobs one `job/start`, `job/read` or `job/cancel` names, and most + /// entries a `job/cancel` response lists. + pub max_request_jobs: u32, + /// Output a job retains: the head and the tail, each half of this; the + /// middle is an `overflow` gap. + pub max_retained_output_bytes: u64, + /// How long a terminal job's record and output are kept. + pub record_retention_ms: u64, + /// Grace between `SIGTERM` and `SIGKILL` for a non-forced cancel and + /// timeouts. + pub cancel_grace_ms: u64, +} + +/// Ceilings of `fs/*` and `transfer/*`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct FilesystemLimits { + /// Largest `maxBytes` of `fs/readFile`. + pub max_read_file_bytes: u64, + /// Largest `data` of `fs/writeFile`. + pub max_write_file_bytes: u64, + /// Largest `limit` of `fs/readDirectory`. + pub max_directory_entries: u32, + pub search: SearchCeilings, + pub glob: GlobCeilings, + pub scan: ScanCeilings, + pub transfer: TransferCeilings, +} + +/// Ceilings of `fs/searchText` quotas, field for field. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SearchCeilings { + pub max_matches: u32, + pub max_files: u32, + pub max_bytes: u64, + pub max_duration_ms: u64, + pub max_line_bytes: u32, +} + +/// Ceilings of `fs/globFiles` quotas, field for field. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GlobCeilings { + pub max_matches: u32, + pub max_files: u32, + pub max_duration_ms: u64, +} + +/// Ceilings of `fs/scan`: root and pattern counts, then its quotas. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanCeilings { + pub max_roots: u32, + pub max_patterns: u32, + pub max_entries: u32, + pub max_file_bytes: u64, + pub max_total_bytes: u64, + pub max_duration_ms: u64, +} -fn default_protocol_version() -> u32 { - CURRENT_PROTOCOL_VERSION +/// Ceilings of `transfer/*`: capacity, page and chunk sizes, receipt +/// retention, then the per-operation quotas. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TransferCeilings { + /// Operations not yet complete, aborted or failed, across connections; + /// a begin beyond it is `busy` (`transferCapacity`). + pub max_active_operations: u32, + pub max_content_chunk_bytes: u64, + pub max_inventory_page_entries: u32, + /// How long a finished operation's receipt is answered. + pub receipt_retention_ms: u64, + pub max_entries: u32, + pub max_depth: u32, + pub max_file_bytes: u64, + pub max_total_bytes: u64, + pub max_manifest_bytes: u64, + pub max_duration_ms: u64, } diff --git a/crates/environment-protocol/src/data/idle.rs b/crates/environment-protocol/src/data/idle.rs index 4ebd0bbfa..e92bb1e3e 100644 --- a/crates/environment-protocol/src/data/idle.rs +++ b/crates/environment-protocol/src/data/idle.rs @@ -1,41 +1,41 @@ -//! Daemon activity report used by idle power policy. +//! `env/getIdle`: the daemon's activity report for idle power policy. //! -//! The daemon reports a monotonic idle *duration*, never a wall-clock -//! timestamp: after a freeze, stateful stop, or snapshot restore the guest -//! clock is stale until it resyncs, while a monotonic duration stays correct. +//! The daemon reports a monotonic idle duration, never a wall-clock +//! timestamp: after a freeze, stateful stop or snapshot restore the guest +//! clock is stale until it resyncs, while a monotonic duration stays +//! correct. The policy is the provider's +//! ([`IdlePolicy`](crate::idle::IdlePolicy)); it decides on `idleForMs` +//! alone, and the counts are diagnostics. use serde::{Deserialize, Serialize}; +use crate::serde_util::is_zero_u32; + +/// `env/getIdle` params. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct IdleParams {} +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GetIdleParams {} +/// `env/getIdle` response. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct IdleResponse { - /// Milliseconds since the last data-plane request or since the last - /// process/job finished, whichever is later. Zero while anything runs. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GetIdleResponse { + /// Milliseconds since the last `use` request, or since running work + /// last ended, whichever is later. Zero while any process or job runs + /// or any leftover process group has live members; the clock restarts + /// when the last of them ends. pub idle_for_ms: u64, - /// Processes started through `process/start` that have not exited. - pub running_processes: u32, - /// Jobs that are running or cancelling. - pub running_jobs: u32, - /// Process groups of finished commands that still have members alive: - /// services and other leftovers a command started and did not stop. - /// They are not running work, so they never block idle power policy; - /// a policy may prefer freezing over stopping while the count is - /// non-zero. - #[serde(default, skip_serializing_if = "is_zero")] - pub leftover_process_groups: u32, -} - -fn is_zero(value: &u32) -> bool { - *value == 0 -} - -impl IdleResponse { - /// True when nothing is executing inside the environment. - pub fn is_quiescent(&self) -> bool { - self.running_processes == 0 && self.running_jobs == 0 - } + /// Processes started with `process/start` whose root has not exited. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub running_process_count: u32, + /// Jobs that are `running`. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub running_job_count: u32, + /// Process groups of exited processes that still have members alive: + /// services a command started and left running. They hold `idleForMs` + /// at zero like running work. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub leftover_process_group_count: u32, } diff --git a/crates/environment-protocol/src/data/inventory.rs b/crates/environment-protocol/src/data/inventory.rs deleted file mode 100644 index 42d9a58f0..000000000 --- a/crates/environment-protocol/src/data/inventory.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! Portable filesystem inventories. Digests identify raw file bytes, not paths or modes. -use crate::shared::{ByteChunk, EnvironmentPath}; -use serde::{Deserialize, Serialize}; - -pub const MAX_INVENTORY_PAGE: usize = 128; -pub const MAX_CONTENT_CHUNK: usize = 256 * 1024; -pub const MAX_INVENTORY_PATH_BYTES: usize = 4096; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "kind", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum InventoryContent { - Directory, - File { - size_bytes: u64, - executable: bool, - digest: String, - }, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InventoryEntry { - /// Empty for the selected root; otherwise a strict relative slash-separated path. - pub path: String, - pub content: InventoryContent, -} - -/// Total operation quotas, separate from wire chunk and page limits. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct InventoryLimits { - pub max_entries: u32, - pub max_depth: u32, - pub max_file_bytes: u64, - pub max_total_bytes: u64, - pub max_manifest_bytes: u64, - pub max_duration_ms: u64, -} -impl Default for InventoryLimits { - fn default() -> Self { - Self { - max_entries: 100_000, - max_depth: 64, - max_file_bytes: 1024_u64.pow(4), - max_total_bytes: 1024_u64.pow(4), - max_manifest_bytes: 32 * 1024 * 1024, - max_duration_ms: 24 * 60 * 60 * 1000, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ScanDigestAlgorithm { - Sha256, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "kind", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum ScanContent { - Directory, - File { - size_bytes: u64, - executable: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - digest: Option, - }, -} - -/// Small, bounded observation for catalogs. Large inventories use transfer sessions. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScanParams { - pub roots: Vec, - /// Filename-only patterns without `/` or `**` select direct children only. - /// Other patterns permit recursive traversal within the operation quotas. - #[serde(default)] - pub include_patterns: Vec, - #[serde(default)] - pub read_content: bool, - /// Follow aliases only inside the endpoint filesystem access scope. - #[serde(default)] - pub follow_symlinks: bool, - #[serde(default)] - pub digest_algorithm: Option, - #[serde(default)] - pub limits: InventoryLimits, - pub if_none_match: Option, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScanEntry { - /// Resolved absolute identity, while root/path retain the requested spelling. - pub canonical_path: EnvironmentPath, - pub root: EnvironmentPath, - pub path: String, - pub content: ScanContent, - pub data: Option, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScanDiagnostic { - pub root: EnvironmentPath, - pub error: crate::error::EnvironmentProtocolError, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ScanResponse { - pub fingerprint: Option, - pub unchanged: bool, - pub complete: bool, - pub entries: Vec, - pub diagnostics: Vec, -} diff --git a/crates/environment-protocol/src/data/job.rs b/crates/environment-protocol/src/data/job.rs new file mode 100644 index 000000000..30f5e8a04 --- /dev/null +++ b/crates/environment-protocol/src/data/job.rs @@ -0,0 +1,302 @@ +//! `job/*`: durable batch commands with dependencies and queues. +//! +//! A job is identified by its `jobId` on the daemon together with the +//! `requestId` of the `job/start` that created it. Its record and output +//! survive daemon restarts; a job that was queued or running when the +//! daemon stopped is `interrupted`. When a job's root process finishes, the +//! daemon sweeps the job's process group and reports what it swept. + +use serde::{Deserialize, Serialize}; + +use crate::{ + data::exec::{CommandSpec, ExitStatus, LeftoverReport, OutputPage, ReturnOn, StoppedBy}, + error::ProtocolError, + ids::{JobId, JobName, JobRequestId, QueueKey}, + serde_util::{is_false, is_zero_u32}, +}; + +/// `job/start` params. All-or-nothing: either every job is accepted (or +/// replayed) or none is. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct StartJobsParams { + /// Idempotency key of this start. Retrying with the same request id and + /// identical jobs replays the existing jobs; reusing a job id with + /// another request id or spec is `failedPrecondition` (`jobIdInUse`). + pub request_id: JobRequestId, + /// 1 to `limits.job.maxRequestJobs` jobs. + pub jobs: Vec, +} + +/// One job of a `job/start`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobSpec { + pub job_id: JobId, + /// Local name other jobs of the same `job/start` can depend on; unique + /// within the request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// `input` must be `none` or `bytes`. + pub command: CommandSpec, + /// Jobs that must finish first. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub depends_on: Vec, + /// What "finish" means for `dependsOn`; `allSucceeded` by default. + #[serde(default, skip_serializing_if = "JobDependencyPolicy::is_default")] + pub dependency_policy: JobDependencyPolicy, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, +} + +/// A dependency reference. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum JobDependency { + /// An existing job on this daemon, or one in the same request. + JobId { job_id: JobId }, + /// The `name` of a job in the same `job/start`. + Name { name: JobName }, +} + +/// What a job waits for from its `dependsOn` jobs. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum JobDependencyPolicy { + /// Run once every dependency `succeeded`; any other terminal status + /// makes this job `dependencyFailed`. + #[default] + AllSucceeded, + /// Run once every dependency is terminal, whatever its status. + AllTerminal, +} + +impl JobDependencyPolicy { + pub fn is_default(&self) -> bool { + *self == Self::AllSucceeded + } +} + +/// `job/start` response: one summary per requested job, in request order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct StartJobsResponse { + /// Never empty. + pub jobs: Vec, +} + +/// `job/read` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadJobsParams { + /// 1 to `limits.job.maxRequestJobs` entries, each with its own position. + pub jobs: Vec, + /// Most `data` bytes across all entries, at most + /// `limits.job.maxReadBytes`, filled in request order: an entry reached + /// after the budget is spent returns its summary and an empty page at + /// its start position. 0 reads only summaries. + pub max_bytes: u64, + /// How long to wait, at most `limits.maxWaitMs`. 0 polls. + pub wait_ms: u64, + /// `anyOutput`: return when any entry has output past its position or + /// changed status; `exit`: when every entry is terminal; `window`: wait + /// the whole window. + pub return_on: ReturnOn, +} + +/// One entry of a `job/read`. Job reads never consume: without `after` and +/// `tailBytes` the page starts at offset 0. One job may be named by several +/// entries, such as a head and a tail page. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobReadRequest { + pub job_id: JobId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub after: Option, + /// Start at `end - tailBytes`; excludes `after`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tail_bytes: Option, + /// Most `data` bytes for this entry; the request's `maxBytes` still + /// bounds the total. Absent: whatever of the request's budget remains. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_bytes: Option, +} + +/// `job/read` response: one entry per requested job, in request order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadJobsResponse { + /// Never empty. + pub jobs: Vec, +} + +/// A `job/read` entry: the job's summary and output page, or why it could +/// not be read (`notFound` for an unknown or garbage-collected job). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum JobReadEntry { + Job { + summary: Box, + output: OutputPage, + }, + Error { + job_id: JobId, + error: ProtocolError, + }, +} + +/// `job/cancel` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CancelJobsParams { + /// 1 to `limits.job.maxRequestJobs` ids. + pub job_ids: Vec, + /// `job` by default. + #[serde(default, skip_serializing_if = "JobCancelScope::is_default")] + pub scope: JobCancelScope, + /// Kill at once instead of `SIGTERM`, `limits.job.cancelGraceMs`, then + /// `SIGKILL`. + #[serde(default, skip_serializing_if = "is_false")] + pub force: bool, +} + +/// Which jobs a `job/cancel` affects. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum JobCancelScope { + /// Only the named jobs. + #[default] + Job, + /// The named jobs and every job that transitively depends on them. + Dependents, +} + +impl JobCancelScope { + pub fn is_default(&self) -> bool { + *self == Self::Job + } +} + +/// `job/cancel` response: one entry per affected job, named jobs first in +/// request order, then dependents, at most `limits.job.maxRequestJobs` +/// entries in total. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CancelJobsResponse { + /// Never empty. + pub jobs: Vec, + /// Dependents cancelled as well but not listed, beyond the entry limit. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub unlisted_job_count: u32, +} + +/// A job summary, or why the named job could not be found. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum JobSummaryEntry { + Job { summary: Box }, + Error { job_id: JobId, error: ProtocolError }, +} + +/// What a job is and how it ended. Instants are the guest's wall clock. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct JobSummary { + pub job_id: JobId, + pub request_id: JobRequestId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub status: JobStatus, + /// Resolved ids of the jobs this one depends on. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub depends_on: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, + pub created_at_ms: i64, + /// When the job became runnable and waited for its queue. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queued_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at_ms: Option, + /// When a `job/cancel` reached the job while it ran. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cancel_requested_at_ms: Option, + /// Monotonic run time from start to finish, immune to guest clock jumps. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + /// How the root process ended, once it ran and exited. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit: Option, + /// Set when the daemon stopped the running job: `timeout` for + /// `timedOut`, `terminate` or `kill` for a cancel without or with + /// `force`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stopped_by: Option, + /// What the daemon swept from the job's group at completion + /// (`disposition: swept`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub leftovers: Option, + /// Why a `failed` or `interrupted` job produced no exit, or an + /// infrastructure failure while it ran. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Job lifecycle. `queued` and `running` are live; the rest are terminal. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum JobStatus { + /// Accepted; waiting for dependencies or its queue. + Queued, + Running, + /// Exited with code 0. + Succeeded, + /// Exited otherwise, or could not start. + Failed, + /// Its `timeoutMs` expired and the daemon stopped it. + TimedOut, + /// Stopped or never started because of `job/cancel`. + Cancelled, + /// A dependency did not finish as the dependency policy requires. + DependencyFailed, + /// The daemon stopped while the job was queued or running. + Interrupted, +} + +impl JobStatus { + pub fn is_terminal(self) -> bool { + !matches!(self, Self::Queued | Self::Running) + } +} diff --git a/crates/environment-protocol/src/data/jobs.rs b/crates/environment-protocol/src/data/jobs.rs deleted file mode 100644 index 7db917d75..000000000 --- a/crates/environment-protocol/src/data/jobs.rs +++ /dev/null @@ -1,240 +0,0 @@ -//! Durable job method payloads. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; - -use crate::shared::{ByteChunk, EnvironmentPath, JobId, SecretString}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StartJobsParams { - pub namespace: String, - pub request_id: String, - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct JobStartSpec { - pub job_id: JobId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - pub argv: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cwd: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub env: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub secret_env: BTreeMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stdin: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout_ms: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub depends_on: Vec, - #[serde(default)] - pub dependency_policy: JobDependencyPolicy, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub queue_key: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct JobDependency { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub job_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, -} - -impl JobDependency { - pub fn job_id(job_id: impl Into) -> Self { - Self { - job_id: Some(job_id.into()), - name: None, - } - } - - pub fn name(name: impl Into) -> Self { - Self { - job_id: None, - name: Some(name.into()), - } - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum JobDependencyPolicy { - #[default] - AllSucceeded, - AllTerminal, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct StartJobsResponse { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListJobsParams { - pub namespace: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub limit: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListJobsResponse { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReadJobsParams { - pub namespace: String, - pub jobs: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub after_seq: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_bytes: Option, - #[serde(default)] - pub include_artifacts: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub wait_ms: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ReadJobsResponse { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct JobReadResult { - pub summary: JobSummary, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub output_chunks: Vec, - pub output_next_seq: u64, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub artifacts: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CancelJobsParams { - pub namespace: String, - pub jobs: Vec, - #[serde(default)] - pub scope: JobCancelScope, - #[serde(default)] - pub force: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CancelJobsResponse { - pub jobs: Vec, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum JobCancelScope { - #[default] - Job, - Dependents, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub struct JobSummary { - pub namespace: String, - pub job_id: JobId, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - pub status: JobStatus, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub dependencies: Vec, - pub created_at_ms: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub queued_at_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub started_at_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub finished_at_ms: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub exit_code: Option, - /// True when the job's root process exited while descendants it spawned - /// were still running. The environment daemon terminates them best-effort at job - /// completion; a submitting agent should fix its script's teardown. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub orphaned_descendants: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub failure: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub queue_key: Option, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub enum JobStatus { - Accepted, - Queued, - Running, - Succeeded, - Failed, - CancelRequested, - Cancelled, - TimedOut, - DependencyFailed, - Interrupted, - Lost, -} - -impl JobStatus { - pub fn is_terminal(self) -> bool { - matches!( - self, - Self::Succeeded - | Self::Failed - | Self::Cancelled - | Self::TimedOut - | Self::DependencyFailed - | Self::Interrupted - | Self::Lost - ) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct JobOutputChunk { - pub seq: u64, - pub stream: JobOutputStream, - pub chunk: ByteChunk, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub enum JobOutputStream { - Stdout, - Stderr, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub struct JobArtifact { - pub path: EnvironmentPath, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub kind: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, -} diff --git a/crates/environment-protocol/src/data/methods.rs b/crates/environment-protocol/src/data/methods.rs deleted file mode 100644 index 3f9647c80..000000000 --- a/crates/environment-protocol/src/data/methods.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! JSON-RPC method names for the environment data plane. - -pub const INITIALIZE_METHOD: &str = "initialize"; -pub const INITIALIZED_METHOD: &str = "initialized"; - -pub const FS_READ_FILE_METHOD: &str = "fs/readFile"; -pub const FS_WRITE_FILE_METHOD: &str = "fs/writeFile"; -pub const FS_CREATE_DIRECTORY_METHOD: &str = "fs/createDirectory"; -pub const FS_GET_METADATA_METHOD: &str = "fs/getMetadata"; -pub const FS_READ_DIRECTORY_METHOD: &str = "fs/readDirectory"; -pub const FS_REMOVE_METHOD: &str = "fs/remove"; -pub const FS_COPY_METHOD: &str = "fs/copy"; -pub const FS_SEARCH_TEXT_METHOD: &str = "fs/searchText"; -pub const FS_GLOB_FILES_METHOD: &str = "fs/globFiles"; - -pub const PROCESS_START_METHOD: &str = "process/start"; -pub const PROCESS_READ_METHOD: &str = "process/read"; -pub const PROCESS_WRITE_METHOD: &str = "process/write"; -pub const PROCESS_TERMINATE_METHOD: &str = "process/terminate"; -pub const PROCESS_RESIZE_METHOD: &str = "process/resize"; -pub const PROCESS_OUTPUT_METHOD: &str = "process/output"; -pub const PROCESS_EXITED_METHOD: &str = "process/exited"; -pub const PROCESS_CLOSED_METHOD: &str = "process/closed"; - -pub const JOB_START_METHOD: &str = "job/start"; -pub const JOB_LIST_METHOD: &str = "job/list"; -pub const JOB_READ_METHOD: &str = "job/read"; -pub const JOB_CANCEL_METHOD: &str = "job/cancel"; - -pub const ENV_IDLE_METHOD: &str = "env/idle"; - -pub const FS_MATERIALIZE_METHOD: &str = "fs/materialize"; -pub const FS_CAPTURE_METHOD: &str = "fs/capture"; - -pub const FS_SCAN_METHOD: &str = "fs/scan"; -pub const FS_TRANSFER_METHOD: &str = "fs/transfer"; diff --git a/crates/environment-protocol/src/data/mod.rs b/crates/environment-protocol/src/data/mod.rs index f4dcb3340..5e143ce19 100644 --- a/crates/environment-protocol/src/data/mod.rs +++ b/crates/environment-protocol/src/data/mod.rs @@ -1,13 +1,11 @@ -//! Data-plane protocol for one selected environment execution scope. +//! The data plane: one envd serving one environment's filesystem, +//! processes and jobs. The runtime is the client; envd is the server. -pub mod events; +pub mod exec; pub mod fs; pub mod handshake; pub mod idle; -pub mod jobs; -pub mod methods; +pub mod job; pub mod process; +pub mod scan; pub mod transfer; - -pub mod inventory; -pub mod transfer_session; diff --git a/crates/environment-protocol/src/data/process.rs b/crates/environment-protocol/src/data/process.rs index 57b69c643..fac452aea 100644 --- a/crates/environment-protocol/src/data/process.rs +++ b/crates/environment-protocol/src/data/process.rs @@ -1,203 +1,167 @@ -//! Process method payloads. +//! `process/*`: interactive processes read through byte-offset pages. //! -//! A process is started as the leader of its own process group and read -//! through a daemon-owned cursor: `process/read` without `afterSeq` returns -//! the output produced since the previous cursor read and advances the -//! cursor, so a handle used from a later connection never re-reads output. -//! The root's natural exit never sweeps the group; whatever it left running -//! is reported and keeps the environment's lifetime. - -use std::collections::BTreeMap; +//! A process runs as the leader of its own process group. Its root's +//! natural exit never sweeps the group: whatever it left running is +//! reported in `leftovers` and keeps the environment's lifetime. Each +//! process has a daemon-owned cursor; a read without `after` starts there, +//! and the cursor advances to `next` only after the response frame was +//! written to the connection. use serde::{Deserialize, Serialize}; -use crate::shared::{ByteChunk, EnvironmentPath, ProcessId, SecretString}; - +use crate::{ + bytes::ByteChunk, + data::exec::{ + CommandSpec, ExecState, ExitStatus, LeftoverReport, OutputPage, PtySize, ReturnOn, + StoppedBy, + }, + error::ProtocolError, + ids::ProcessId, + serde_util::is_false, +}; + +/// `process/start` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct StartProcessParams { + /// Caller-chosen handle, reserved before any side effect: a start whose + /// id is already in use (running, retained or remembered by a tombstone) + /// is `alreadyExists` and runs nothing. pub process_id: ProcessId, - pub argv: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cwd: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub env: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub secret_env: BTreeMap, - /// One-shot standard input, written and closed at start. Plain-pipe - /// processes otherwise read end of file from `/dev/null`; interactive - /// input requires `tty`, whose master is the process's input. + pub command: CommandSpec, + /// Run under a PTY of this size. The PTY master is the input: `input` + /// must then be `none` or `bytes`, and `process/write` is always + /// accepted while the process runs. Requires + /// `capabilities.process.pty`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub stdin: Option, - /// Kill deadline. Absent means the call never kills a running process. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout_ms: Option, - #[serde(default)] - pub tty: bool, + pub pty: Option, } +/// `process/start` response, sent once the process runs. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct StartProcessResponse { - pub process_id: ProcessId, + /// OS pid of the root process, which is also its process group id. + pub pid: u32, } +/// `process/read` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReadProcessParams { pub process_id: ProcessId, - /// Explicit re-read from a sequence number over the retained output. It - /// does not move the daemon's cursor; omit it to read after the last - /// delivered chunk and advance the cursor. + /// Start offset. Absent: the daemon cursor, which this read advances to + /// `next` once its response is written. Present: a non-consuming read + /// that leaves the cursor alone. #[serde(default, skip_serializing_if = "Option::is_none")] - pub after_seq: Option, + pub after: Option, + /// Start at `end - tailBytes` instead (clamped at 0). Non-consuming; + /// excludes `after`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub max_bytes: Option, - /// Collect output for this long, returning early only when the process - /// exits or `max_bytes` is reached. Absent blocks until exit. + pub tail_bytes: Option, + /// Acknowledge every byte before this offset first: the cursor moves to + /// it (never backwards) whether or not this read's response arrives. A + /// caller that tracks positions reads with `after` and acknowledges + /// with `ack` for exactly-once delivery. #[serde(default, skip_serializing_if = "Option::is_none")] - pub wait_ms: Option, + pub ack: Option, + /// Most `data` bytes to return, at most `limits.process.maxReadBytes`. + /// 0 reads only the state. + pub max_bytes: u64, + /// How long to wait, at most `limits.maxWaitMs`. 0 polls. + pub wait_ms: u64, + pub return_on: ReturnOn, } +/// `process/read` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProcessOutputChunk { - pub seq: u64, - pub stream: ProcessOutputStream, - pub chunk: ByteChunk, -} - -/// A process still running in the command's group after the root exited. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct LeftoverProcess { - pub pid: u32, - /// Command line when the platform exposes it, otherwise the executable - /// name; may be empty when neither could be read. - #[serde(default, skip_serializing_if = "String::is_empty")] - pub command: String, -} - -/// Why a process that did not exit on its own stopped. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ProcessTermination { - /// The start deadline (`timeoutMs`) expired and the group was killed. - TimedOut, - /// `process/terminate` with `kill` ended the group. - Killed, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ReadProcessResponse { - pub chunks: Vec, - pub next_seq: u64, - pub exited: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub exit_code: Option, - pub closed: bool, + pub output: OutputPage, + pub state: ExecState, + /// How the root ended; present once `state` is `exited`, unless `error` + /// explains why it could not be observed. #[serde(default, skip_serializing_if = "Option::is_none")] - pub failure: Option, + pub exit: Option, /// Set when the daemon stopped the process; absent on a natural exit. #[serde(default, skip_serializing_if = "Option::is_none")] - pub termination: Option, - /// OS pid of the root process, which is also its process group id. + pub stopped_by: Option, + /// Infrastructure failure after start (output capture or wait failed). #[serde(default, skip_serializing_if = "Option::is_none")] - pub pid: Option, - /// Bytes dropped from the middle of the retained output since the last - /// cursor read because the retained buffer overflowed. The gap sits at - /// the sequence discontinuity between two returned chunks. - #[serde(default, skip_serializing_if = "is_zero")] - pub omitted_bytes: u64, - /// Processes still running in the command's group after the root - /// exited, sampled when this read was answered. Empty until the exit is - /// observed and on platforms without process enumeration. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub leftover_processes: Vec, -} - -fn is_zero(value: &u64) -> bool { - *value == 0 + pub error: Option, + /// OS pid of the root process. + pub pid: u32, + /// Group members alive after the root exited (`disposition: kept`), + /// sampled when this response was built. Absent while running, when + /// none remain, and on platforms without process enumeration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub leftovers: Option, } +/// `process/write` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct WriteProcessParams { pub process_id: ProcessId, - /// Bytes for the process's input. Requires an open input (a PTY); an - /// absent or empty chunk with `closeStdin: false` is a wait and is - /// accepted whether or not the process has exited. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub chunk: Option, - #[serde(default)] - pub close_stdin: bool, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum WriteProcessStatus { - Accepted, - /// The entry is gone: the process finished and its retention elapsed. - UnknownProcess, - /// Input was given but the process has no open input, or has exited. - StdinClosed, - Starting, + /// Bytes for the process's input pipe or PTY. + #[serde(default, skip_serializing_if = "ByteChunk::is_empty")] + pub data: ByteChunk, + /// Close the input after `data`: end of file on a pipe, `VEOF` on a + /// PTY. Applied only when every byte of `data` was accepted. + #[serde(default, skip_serializing_if = "is_false")] + pub close_input: bool, } +/// `process/write` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct WriteProcessResponse { - pub status: WriteProcessStatus, + /// Leading bytes of `data` queued to the process's writer. Fewer than + /// sent (possibly 0) means the input queue is full: resend the rest + /// later. + pub accepted_bytes: u64, } /// Signal delivered by `process/terminate`. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] pub enum ProcessSignal { - /// `SIGINT` to the process group; changes no daemon state, the next - /// read observes whatever the process did. + /// `SIGINT` to the foreground process group (the terminal's under a + /// PTY, otherwise the process group). Records nothing; the next read + /// shows what the process did. Interrupt, - /// Kill the whole group; the process is recorded as killed. + /// `SIGTERM` to the group, then `SIGKILL` after + /// `limits.process.terminateGraceMs` if members remain. Records + /// `stoppedBy: terminate`. + Terminate, + /// `SIGKILL` to the group. Records `stoppedBy: kill`. Kill, } +/// `process/terminate` params. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct TerminateProcessParams { pub process_id: ProcessId, pub signal: ProcessSignal, } +/// `process/terminate` response, sent once the signal was delivered (any +/// escalation continues in the background). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct TerminateProcessResponse { - /// True when the process was still running when the signal was sent. + /// True when the root process was still running when the signal was + /// sent. + #[serde(default, skip_serializing_if = "is_false")] pub running: bool, } - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TerminalSize { - pub rows: u16, - pub cols: u16, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ResizeProcessParams { - pub process_id: ProcessId, - pub size: TerminalSize, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ResizeProcessResponse {} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ProcessOutputStream { - Stdout, - Stderr, - Pty, -} diff --git a/crates/environment-protocol/src/data/scan.rs b/crates/environment-protocol/src/data/scan.rs new file mode 100644 index 000000000..b2aad6894 --- /dev/null +++ b/crates/environment-protocol/src/data/scan.rs @@ -0,0 +1,231 @@ +//! `fs/scan`: a small, bounded observation of several roots, for catalogs +//! such as skill and prompt discovery. An observation, not use: it never +//! resets the idle clock. Large trees go through `transfer/*`. +//! +//! Each root is reported on its own, so one bad entry or one failed root +//! never hides the others. + +use serde::{Deserialize, Serialize}; + +use crate::{ + bytes::ByteChunk, + data::fs::{TreeContent, TreeFilter}, + error::ProtocolError, + ids::validate_opaque_id, + paths::{EnvironmentPath, TreePath}, + serde_util::{is_false, is_zero_u32}, +}; + +/// `fs/scan` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanParams { + /// 1 to `limits.filesystem.scan.maxRoots` roots. + pub roots: Vec, + /// Applied below every root; at most `limits.filesystem.scan.maxPatterns` + /// patterns in total. + #[serde(default, skip_serializing_if = "TreeFilter::is_default")] + pub filter: TreeFilter, + /// Return the bytes of every selected file in `data`. + #[serde(default, skip_serializing_if = "is_false")] + pub read_content: bool, + /// Return each selected file's `digest`. + #[serde(default, skip_serializing_if = "is_false")] + pub include_digests: bool, + /// Follow symlinks whose targets stay inside the filesystem root; others + /// are skipped (`symlinkOutsideRoot`). Without it symlinks are skipped + /// (`symlink`). + #[serde(default, skip_serializing_if = "is_false")] + pub follow_symlinks: bool, + /// Caller quotas; each at most its ceiling. + pub limits: ScanLimits, +} + +/// One root of a scan. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanRoot { + pub path: EnvironmentPath, + /// The root's previous `fingerprint`: when the selection is unchanged + /// the result is `unchanged` and carries no entries. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub if_none_match: Option, + /// Overrides `filter.maxDepth` for this root. `0` examines only the + /// root itself: a `complete` result without entries proves it is an + /// existing directory, which is how discovery checks a working + /// directory in the same call. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_depth: Option, +} + +/// Caller quotas of one scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanLimits { + /// Selected entries per root; more fail that root (`payloadTooLarge` + /// with `limitEntries`). + pub max_entries: u32, + /// Larger selected files are skipped (`fileTooLarge`). + pub max_file_bytes: u64, + /// Content bytes returned across all roots, in root order: a root whose + /// content would exceed the rest fails (`payloadTooLarge` with + /// `limitBytes`) and later roots are still scanned. Independently, a + /// root whose encoded result (entries, skips and base64 content) would + /// push the scan's results past + /// [`MAX_RESULT_BYTES`](crate::limits::MAX_RESULT_BYTES) fails the same + /// way. + pub max_total_bytes: u64, + /// Walk time per root; exhausting it fails that root + /// (`deadlineExceeded`). + pub max_duration_ms: u64, +} + +/// `fs/scan` response: one result per requested root, in request order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanResponse { + /// Never empty. + pub roots: Vec, +} + +/// The outcome of one root. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ScanRootResult { + /// Every selected entry, plus what was skipped and why. + Complete { + path: EnvironmentPath, + /// Resolved absolute path of the root when it differs from `path` + /// (the root is a symlink). + #[serde(default, skip_serializing_if = "Option::is_none")] + canonical_path: Option, + fingerprint: ScanFingerprint, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + entries: Vec, + /// The first + /// [`MAX_LISTED_SKIPS`](crate::limits::MAX_LISTED_SKIPS) skipped + /// entries, in walk order. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + skipped: Vec, + /// Total number of skipped entries, listed or not; present whenever + /// it is not zero. + #[serde(default, skip_serializing_if = "is_zero_u32")] + skipped_entry_count: u32, + }, + /// The selection matches `ifNoneMatch`. + Unchanged { + path: EnvironmentPath, + fingerprint: ScanFingerprint, + }, + /// The root does not exist. + Missing { path: EnvironmentPath }, + /// The root could not be scanned; no entries are returned for it. + Failed { + path: EnvironmentPath, + error: ProtocolError, + }, +} + +/// One selected entry below a root. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanEntry { + /// Relative to the root as requested. + pub path: TreePath, + pub content: TreeContent, + /// The file's bytes when `readContent` was set. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +/// An entry a scan left out. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ScanSkip { + /// Absent when the entry's name is not UTF-8. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + pub reason: ScanSkipReason, +} + +/// Why a scan left an entry out. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ScanSkipReason { + /// Larger than `maxFileBytes`. + FileTooLarge, + /// A symlink, and `followSymlinks` is not set. + Symlink, + /// A symlink whose target leaves the filesystem root, or a link cycle. + SymlinkOutsideRoot, + /// A device, FIFO, socket or another special file. + SpecialFile, + /// The name is not UTF-8. + NonUtf8Name, + /// The entry vanished or could not be read during the walk. + Unreadable, +} + +/// Opaque fingerprint of one root's selection, prefixed `scan:v1:`, at most +/// 255 bytes without control characters. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct ScanFingerprint( + #[cfg_attr( + feature = "schema", + schemars(length(min = 9, max = 255), pattern(SCAN_FINGERPRINT_PATTERN)) + )] + String, +); + +/// JSON Schema pattern of [`ScanFingerprint`]. +pub const SCAN_FINGERPRINT_PATTERN: &str = "^scan:v1:[^\\u0000-\\u001F\\u007F-\\u009F]+$"; + +impl ScanFingerprint { + pub const PREFIX: &'static str = "scan:v1:"; + + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if !value.starts_with(Self::PREFIX) + || value.len() == Self::PREFIX.len() + || validate_opaque_id("ScanFingerprint", &value).is_err() + { + return Err(ScanFingerprintError(value)); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("scan fingerprint must start with scan:v1: and be at most 255 bytes: {0:?}")] +pub struct ScanFingerprintError(pub String); + +impl Serialize for ScanFingerprint { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for ScanFingerprint { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(serde::de::Error::custom) + } +} diff --git a/crates/environment-protocol/src/data/transfer.rs b/crates/environment-protocol/src/data/transfer.rs index cad1f5feb..0b8c130fc 100644 --- a/crates/environment-protocol/src/data/transfer.rs +++ b/crates/environment-protocol/src/data/transfer.rs @@ -1,111 +1,397 @@ -//! Bounded inline filesystem copies. No VFS identities or storage access are required. -use crate::shared::{ByteChunk, EnvironmentPath}; +//! `transfer/*`: one logical tree or file transfer as many bounded calls. +//! +//! No storage or VFS identities cross this boundary: entries are tree paths +//! and content digests. An operation is created by `transfer/begin` and +//! keyed by its caller-chosen `operationId`; retrying any call with the same +//! input is safe. Phases: +//! +//! - **capture** (environment to caller): `scanning` (advance until the +//! inventory is sealed) → `inventory` (page entries, read content) → +//! `complete` after `transfer/commit`. +//! - **materialize** and **writeFile** (caller to environment): +//! `inventory` (append the incoming inventory) → `scanning` (the daemon +//! reads destination metadata and hashes only files whose size matches an +//! incoming entry) → `content` (upload `transfer/listMissing` digests) → +//! `staging` (advance) → `ready` → `complete` after `transfer/commit`, +//! which publishes the staged tree atomically. +//! +//! A failure of the operation's own work (walking, hashing, staging, +//! publishing, a source changing, a digest mismatch, its duration bound) +//! moves it to `failed` with its `error`, and every later call except +//! `transfer/getStatus` and `transfer/abort` answers that error; nothing is +//! committed. A call refused before it acts (invalid params, a phase or +//! offset precondition, an unknown digest, `busy`) leaves the operation +//! unchanged, so a retried or misdirected call cannot fail it. A completed, aborted or failed operation's +//! receipt is answered for `limits.filesystem.transfer.receiptRetentionMs` +//! after it finished, across daemon restarts. A completed capture's +//! inventory stays pageable with `transfer/readInventory` for as long as its +//! receipt, so a retried capture answers the tree it first captured; its +//! content is no longer readable. + use serde::{Deserialize, Serialize}; -/// Hard ceilings for one operation, before base64/JSON framing. -pub const MAX_TRANSFER_BYTES: u64 = 8 * 1024 * 1024; -pub const MAX_TRANSFER_ENTRIES: u32 = 1024; -pub const MAX_TRANSFER_DEPTH: u32 = 32; -pub const MAX_TRANSFER_DURATION_MS: u64 = 30_000; -pub const MAX_TRANSFER_PATH_BYTES: usize = 4096; +use crate::{ + bytes::ByteChunk, + data::fs::{FileVersion, TreeContent, TreeFilter, WriteCondition}, + error::ProtocolError, + ids::TransferOperationId, + paths::{ContentDigest, EnvironmentPath, TreePath}, + serde_util::{is_false, is_zero_u32}, +}; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// What one operation transfers. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum TransferSelection { + /// Read a file or tree out of the environment. + Capture { + source: EnvironmentPath, + #[serde(default, skip_serializing_if = "TreeFilter::is_default")] + filter: TreeFilter, + #[serde(default, skip_serializing_if = "SymlinkPolicy::is_default")] + symlinks: SymlinkPolicy, + #[serde(default, skip_serializing_if = "SpecialFilePolicy::is_default")] + special_files: SpecialFilePolicy, + }, + /// Write a tree into the environment, replacing `destination` atomically + /// at commit. + Materialize { + destination: EnvironmentPath, + #[serde(default, skip_serializing_if = "TransferOnExisting::is_default")] + on_existing: TransferOnExisting, + }, + /// Write exactly one file with `fs/writeFile` semantics: staged, + /// atomic, an existing target keeps its permissions, an in-root symlink + /// at `destination` is followed. + WriteFile { + destination: EnvironmentPath, + #[serde(default, skip_serializing_if = "WriteCondition::is_default")] + create: WriteCondition, + #[serde(default, skip_serializing_if = "Option::is_none")] + if_match: Option, + #[serde(default, skip_serializing_if = "is_false")] + executable: bool, + }, +} + +/// Symlinks met during a capture. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum SymlinkPolicy { + /// Capture a link whose target stays inside the selection as the entry + /// it points to; a link leaving the selection or a cycle fails the + /// operation (`symlinkRejected`). A dangling link inside the selection + /// is left out and reported in `skipped` as `symlink`. + #[default] + ExpandInside, + /// Leave every link out and report it in `skipped`. + Skip, + /// Fail on the first link. + Reject, +} + +impl SymlinkPolicy { + pub fn is_default(&self) -> bool { + *self == Self::ExpandInside + } +} + +/// Devices, FIFOs and sockets met during a capture. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] +pub enum SpecialFilePolicy { + /// Fail the operation (`notRegularFile`). + #[default] + Reject, + /// Leave them out and report them in `skipped`. + Skip, +} + +impl SpecialFilePolicy { + pub fn is_default(&self) -> bool { + *self == Self::Reject + } +} + +/// What a materialize does with an existing destination. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum TransferOnExisting { + /// Refuse at begin (`alreadyExists`). + Error, + /// Replace it atomically at commit. + #[default] + Replace, +} + +impl TransferOnExisting { + pub fn is_default(&self) -> bool { + *self == Self::Replace + } +} + +/// Caller quotas of one operation; each at most its advertised ceiling. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct TransferLimits { - /// Counts the selected root, including an empty directory. + /// Entries, the selected root included. pub max_entries: u32, - /// Selected root has depth zero. + /// The selected root has depth 0. pub max_depth: u32, pub max_file_bytes: u64, pub max_total_bytes: u64, - /// Cooperative deadline checked between filesystem operations and byte chunks. - /// Does not interrupt an in-flight kernel syscall. + /// Encoded inventory size. + pub max_manifest_bytes: u64, + /// From begin to commit. pub max_duration_ms: u64, } +/// One inventory entry. Parents precede children. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "camelCase")] -pub enum TransferContent { - Directory, - File { data: ByteChunk, executable: bool }, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct InventoryEntry { + /// Relative to the selected root; `.` is the root itself. + pub path: TreePath, + /// A file entry always carries its `digest`. + pub content: TreeContent, } +/// `transfer/begin` params. Idempotent: a repeat with identical input +/// returns the operation's status; different input under the same id is +/// `failedPrecondition` (`operationIdInUse`). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TransferEntry { - /// Root is "". Others are strict relative '/' paths; parents must precede children. - /// UTF-8 names only; no empty, '.', '..', NUL or backslash components. - pub path: String, - pub content: TransferContent, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BeginTransferParams { + pub operation_id: TransferOperationId, + pub selection: TransferSelection, + pub limits: TransferLimits, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum TransferOnExisting { - Error, - #[default] - Replace, +/// Params of `transfer/advance`, `transfer/getStatus`, `transfer/commit` +/// and `transfer/abort`: each names only the operation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TransferOperationParams { + pub operation_id: TransferOperationId, } +/// The operation's status: the response of `transfer/begin`, +/// `transfer/advance`, `transfer/getStatus`, `transfer/append`, +/// `transfer/commit` and `transfer/abort`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TransferStatus { + pub operation_id: TransferOperationId, + pub phase: TransferPhase, + /// Entries in the inventory so far. + pub entry_count: u32, + /// File bytes in the inventory so far. + pub total_bytes: u64, + /// Content bytes moved over the wire. + pub transferred_bytes: u64, + /// Content bytes satisfied from files already at the destination. + pub reused_bytes: u64, + /// The first + /// [`MAX_LISTED_SKIPS`](crate::limits::MAX_LISTED_SKIPS) entries a + /// capture left out under its policies. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub skipped: Vec, + /// Total number of entries left out, listed or not; present whenever it + /// is not zero. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub skipped_entry_count: u32, + /// Why the operation failed; present exactly when `phase` is `failed`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + /// When `maxDurationMs` runs out for an active operation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deadline_at_ms: Option, + /// When the receipt of a finished (complete, aborted or failed) + /// operation stops being answered; afterwards calls are `notFound` + /// (`expired`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub receipt_expires_at_ms: Option, +} + +/// Where an operation stands; see the module docs for the order per +/// selection kind. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct MaterializeParams { - pub destination: EnvironmentPath, - pub entries: Vec, - pub limits: TransferLimits, - #[serde(default)] - pub on_existing: TransferOnExisting, +pub enum TransferPhase { + /// The daemon walks a tree: the capture source, or the destination of a + /// materialize. Advance until the phase changes. + Scanning, + /// Capture: the inventory is sealed and pageable. Materialize: waiting + /// for appended entries. + Inventory, + /// Waiting for the content of `transfer/listMissing` digests. + Content, + /// The daemon assembles the staged tree. Advance until `ready`. + Staging, + /// Ready for `transfer/commit`. + Ready, + Complete, + Aborted, + /// Stopped by an error; see `error`. + Failed, } +impl TransferPhase { + pub fn is_finished(self) -> bool { + matches!(self, Self::Complete | Self::Aborted | Self::Failed) + } +} + +/// An entry a capture left out. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TransferSkip { + /// Absent when the entry's name is not UTF-8. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + pub reason: TransferSkipReason, +} + +/// Why a capture left an entry out. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct MaterializeResponse { - pub destination: EnvironmentPath, - pub entries: u32, - pub bytes: u64, - /// Reserved for compatibility with early daemons. Current daemons clean retired - /// staging trees asynchronously and return None. - pub retired_directory: Option, +pub enum TransferSkipReason { + /// `symlinks: skip`, or a dangling link under `symlinks: expandInside`. + Symlink, + /// `specialFiles: skip`. + SpecialFile, + /// The name is not UTF-8; such entries are always skipped. + NonUtf8Name, } +/// `transfer/readInventory` params: page a sealed inventory by index. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CaptureParams { - pub source: EnvironmentPath, - pub limits: TransferLimits, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadTransferInventoryParams { + pub operation_id: TransferOperationId, + /// Index of the first entry. + pub index: u32, + /// 1 to `limits.filesystem.transfer.maxInventoryPageEntries`. + pub limit: u32, } +/// `transfer/readInventory` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CaptureResponse { - pub source: EnvironmentPath, - /// Complete selected tree only; failures never return a partial inventory. - pub entries: Vec, - pub bytes: u64, -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn binary_inventory_roundtrips_and_legacy_capabilities_default_false() { - let entry = TransferEntry { - path: "".into(), - content: TransferContent::File { - data: ByteChunk::from(vec![0, 255, 128]), - executable: true, - }, - }; - let json = serde_json::to_value(&entry).unwrap(); - assert_eq!(json["content"]["data"], "AP+A"); - assert_eq!( - serde_json::from_value::(json).unwrap(), - entry - ); - let capabilities: crate::shared::EnvironmentCapabilities = - serde_json::from_str("{}").unwrap(); - assert!(!capabilities.filesystem_capture); - assert!(!capabilities.filesystem_materialize); - assert!(serde_json::from_str::(r#"{"source":"."}"#).is_err()); - } +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadTransferInventoryResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub entries: Vec, + /// Index of the next page; absent after the last entry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_index: Option, +} + +/// `transfer/append` params: add incoming inventory entries in order. +/// Retrying an identical page is safe. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AppendTransferInventoryParams { + pub operation_id: TransferOperationId, + /// Index of the page's first entry: the number of entries appended + /// before it. + pub index: u32, + /// At most `limits.filesystem.transfer.maxInventoryPageEntries`. + pub entries: Vec, + /// This page completes the inventory. + #[serde(default, skip_serializing_if = "is_false")] + pub last: bool, +} + +/// `transfer/listMissing` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListTransferMissingParams { + pub operation_id: TransferOperationId, + /// Most digests to return, 1 to + /// `limits.filesystem.transfer.maxInventoryPageEntries`. + pub limit: u32, +} + +/// `transfer/listMissing` response: digests whose content the daemon still +/// needs. The set shrinks as uploads complete; call again rather than +/// paging. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListTransferMissingResponse { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub digests: Vec, + /// Missing digests beyond this page. + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub remaining_count: u32, +} + +/// `transfer/read` params: read a captured file's content by digest. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadTransferContentParams { + pub operation_id: TransferOperationId, + pub digest: ContentDigest, + pub offset: u64, + /// 1 to `limits.filesystem.transfer.maxContentChunkBytes`. + pub max_bytes: u64, +} + +/// `transfer/read` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ReadTransferContentResponse { + pub data: ByteChunk, + /// Bytes exist after `offset` plus the length of `data`, as for + /// `fs/readFile`. + #[serde(default, skip_serializing_if = "is_false")] + pub truncated: bool, +} + +/// `transfer/write` params: upload content for a missing digest. Chunks for +/// one digest arrive in order; an identical chunk may be retried. +/// Different digests may be uploaded concurrently. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WriteTransferContentParams { + pub operation_id: TransferOperationId, + pub digest: ContentDigest, + /// Must equal the digest's next expected offset. + pub offset: u64, + /// At most `limits.filesystem.transfer.maxContentChunkBytes`. + pub data: ByteChunk, +} + +/// `transfer/write` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WriteTransferContentResponse { + /// The digest's next expected offset; equal to its size once complete + /// and verified. + pub next_offset: u64, } diff --git a/crates/environment-protocol/src/data/transfer_session.rs b/crates/environment-protocol/src/data/transfer_session.rs deleted file mode 100644 index a1d93ad67..000000000 --- a/crates/environment-protocol/src/data/transfer_session.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! One logical transfer, many bounded exchanges. No storage or VFS identities cross this boundary. -use super::{ - inventory::{InventoryEntry, InventoryLimits}, - transfer::TransferOnExisting, -}; -use crate::shared::{ByteChunk, EnvironmentPath}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "direction", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum TransferSelection { - Capture { - source: EnvironmentPath, - }, - Materialize { - destination: EnvironmentPath, - #[serde(default)] - on_existing: TransferOnExisting, - }, - /// Write exactly one file, rejecting directories and preserving existing - /// file permissions. Uses the same bounded chunks and durable receipts. - /// A separate direction makes older endpoints reject unsupported semantics. - WriteFile { - destination: EnvironmentPath, - }, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "action", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum TransferRequest { - /// Idempotent until operation expiry, including completed receipts after daemon restart. - /// IDs must not be reused for different input. - Begin { - operation_id: String, - selection: TransferSelection, - #[serde(default)] - limits: InventoryLimits, - }, - /// Advances bounded scan/staging work. Inspect status until ready, then commit. - Advance { - operation_id: String, - }, - Inventory { - operation_id: String, - offset: u32, - }, - /// Parent-before-child inventory pages. Retrying the identical page is safe. - Append { - operation_id: String, - offset: u32, - entries: Vec, - last: bool, - }, - Missing { - operation_id: String, - offset: u32, - }, - Read { - operation_id: String, - digest: String, - offset: u64, - }, - /// Sequential chunks; an identical chunk may be retried. Only complete verified files are reusable. - Write { - operation_id: String, - digest: String, - offset: u64, - data: ByteChunk, - }, - Commit { - operation_id: String, - }, - Status { - operation_id: String, - }, - Abort { - operation_id: String, - }, -} -impl TransferRequest { - pub fn operation_id(&self) -> &str { - match self { - Self::Begin { operation_id, .. } - | Self::Advance { operation_id } - | Self::Inventory { operation_id, .. } - | Self::Append { operation_id, .. } - | Self::Missing { operation_id, .. } - | Self::Read { operation_id, .. } - | Self::Write { operation_id, .. } - | Self::Commit { operation_id } - | Self::Status { operation_id } - | Self::Abort { operation_id } => operation_id, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub enum TransferPhase { - Scanning, - Inventory, - Content, - Staging, - Ready, - Complete, - Aborted, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(rename_all = "camelCase")] -pub struct TransferStatus { - pub operation_id: String, - pub phase: TransferPhase, - pub entries: u32, - pub bytes: u64, - pub transferred_bytes: u64, - pub reused_bytes: u64, -} -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "result", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum TransferResponse { - Status(TransferStatus), - Inventory { - entries: Vec, - next_offset: Option, - }, - Missing { - digests: Vec, - next_offset: Option, - }, - Chunk { - data: ByteChunk, - eof: bool, - }, - Written { - next_offset: u64, - }, -} diff --git a/crates/environment-protocol/src/error.rs b/crates/environment-protocol/src/error.rs index f2e29b069..36561c1a3 100644 --- a/crates/environment-protocol/src/error.rs +++ b/crates/environment-protocol/src/error.rs @@ -1,38 +1,854 @@ -//! Shared error payloads for environment protocol methods. +//! The one error vocabulary of every plane. +//! +//! An error travels as a JSON-RPC error object whose `code` is a fixed +//! function of its kind and whose `data` carries the kind and its typed +//! fields: +//! +//! ```json +//! {"code": -32004, "message": "unknown process id: p1", "data": {"kind": "notFound", "reason": "expired"}} +//! ``` +//! +//! `data.kind` is authoritative: receivers branch on it, never on the code +//! or the message. Retryability is a property of the kind +//! ([`ProtocolErrorKind::is_retryable`]) and is never sent. The same error +//! object is embedded in results that report per-entry failures (`job/read` +//! entries, `fs/scan` roots, a failed transfer's status). + +use std::fmt; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use thiserror::Error; -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +use crate::limits::MAX_ERROR_MESSAGE_BYTES; + +/// Closed set of error kinds. Each kind has one meaning and one code. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ProtocolErrorKind { + /// The frame is not JSON. Answered with a null id. + ParseError, + /// The frame is JSON but not a valid request: wrong `jsonrpc`, a batch, + /// a missing or malformed member. A second `initialize` is also + /// `invalidRequest`. + InvalidRequest, + /// No method of that name exists on this plane. + MethodNotFound, + /// The params do not decode, or a value is out of range or above an + /// advertised ceiling. + InvalidParams, + /// A method other than `initialize` arrived before a successful + /// `initialize`. + NotInitialized, + /// The peer stated another protocol version. + VersionMismatch, + /// The named handle, path or target does not exist. + NotFound, + /// The thing to create already exists. + AlreadyExists, + /// The request is well-formed but the current state forbids it; `reason` + /// says which state. + FailedPrecondition, + /// The caller may not do this: confinement, credentials, signatures. + PermissionDenied, + /// The endpoint lacks the capability the request needs; an endpoint + /// never silently ignores a parameter it cannot honour. + CapabilityUnavailable, + /// A limit on concurrent work was reached; retry later. + Busy, + /// A server-side duration bound ran out. + DeadlineExceeded, + /// A credential or token is past its validity. + Expired, + /// The operation stopped before completing: cancelled, or its input + /// changed underneath it. Retrying from the start may succeed. + Aborted, + /// A message, payload or result is larger than allowed. + PayloadTooLarge, + /// A process could not be started or supervised. + ProcessFailed, + /// A dependency of the server is temporarily unreachable. + Unavailable, + /// A defect or unexpected failure inside the server. + Internal, +} + +impl ProtocolErrorKind { + /// The JSON-RPC `code` of this kind. Framing kinds use the reserved + /// codes; domain kinds reuse the public API's numbers where the meaning + /// matches (`permissionDenied`, `notFound`, `alreadyExists`, + /// `payloadTooLarge`). + pub fn code(self) -> i32 { + match self { + Self::ParseError => -32700, + Self::InvalidRequest => -32600, + Self::MethodNotFound => -32601, + Self::InvalidParams => -32602, + Self::Internal => -32603, + Self::NotInitialized => -32002, + Self::PermissionDenied => -32003, + Self::NotFound => -32004, + Self::VersionMismatch => -32005, + Self::CapabilityUnavailable => -32006, + Self::FailedPrecondition => -32007, + Self::Busy => -32008, + Self::AlreadyExists => -32009, + Self::PayloadTooLarge => -32013, + Self::DeadlineExceeded => -32015, + Self::Expired => -32016, + Self::Aborted => -32017, + Self::ProcessFailed => -32018, + Self::Unavailable => -32019, + } + } + + /// The kind a code stands for, for error objects that carry no `data` + /// (a non-conforming peer or an intermediary). + pub fn from_code(code: i64) -> Option { + Self::ALL + .into_iter() + .find(|kind| i64::from(kind.code()) == code) + } + + /// True when retrying the same request later can succeed without any + /// change: `busy`, `unavailable` and `aborted`. + pub fn is_retryable(self) -> bool { + matches!(self, Self::Busy | Self::Unavailable | Self::Aborted) + } + + pub fn as_str(self) -> &'static str { + match self { + Self::ParseError => "parseError", + Self::InvalidRequest => "invalidRequest", + Self::MethodNotFound => "methodNotFound", + Self::InvalidParams => "invalidParams", + Self::NotInitialized => "notInitialized", + Self::VersionMismatch => "versionMismatch", + Self::NotFound => "notFound", + Self::AlreadyExists => "alreadyExists", + Self::FailedPrecondition => "failedPrecondition", + Self::PermissionDenied => "permissionDenied", + Self::CapabilityUnavailable => "capabilityUnavailable", + Self::Busy => "busy", + Self::DeadlineExceeded => "deadlineExceeded", + Self::Expired => "expired", + Self::Aborted => "aborted", + Self::PayloadTooLarge => "payloadTooLarge", + Self::ProcessFailed => "processFailed", + Self::Unavailable => "unavailable", + Self::Internal => "internal", + } + } + + pub const ALL: [Self; 19] = [ + Self::ParseError, + Self::InvalidRequest, + Self::MethodNotFound, + Self::InvalidParams, + Self::NotInitialized, + Self::VersionMismatch, + Self::NotFound, + Self::AlreadyExists, + Self::FailedPrecondition, + Self::PermissionDenied, + Self::CapabilityUnavailable, + Self::Busy, + Self::DeadlineExceeded, + Self::Expired, + Self::Aborted, + Self::PayloadTooLarge, + Self::ProcessFailed, + Self::Unavailable, + Self::Internal, + ]; +} + +impl fmt::Display for ProtocolErrorKind { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Why a handle, path or identity was not found. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum NotFoundReason { + /// Its retention window ended; a tombstone remembers it. + Expired, + /// It belonged to an earlier daemon instance (see `daemonInstanceId`). + DaemonRestarted, + /// A daemon identity the gateway has never admitted presented no + /// registration key. + UnknownDaemon, +} + +/// The state that forbids a `failedPrecondition` request. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum PreconditionReason { + /// The process has no open input: started without `pipe` or `pty`, + /// input already closed, or exited. + InputClosed, + /// The file's current `version` differs from `ifMatch`. + IfMatchFailed, + /// `create: ifAbsent` met an existing file, or `create: ifPresent` a + /// missing one. + CreateCondition, + /// The directory is not empty and `recursive` was not set. + NotEmpty, + /// The path names a directory where a file is required. + IsDirectory, + /// A path component or the path names a non-directory where a directory + /// is required. + NotDirectory, + /// The path names a device, FIFO, socket or other non-regular file. + NotRegularFile, + /// A symlink chain loops. + SymlinkLoop, + /// A capture met a symlink that leaves the selection, or one the + /// selection's symlink policy rejects. + SymlinkRejected, + /// The filesystem root itself cannot be replaced or removed. + FilesystemRoot, + /// Rename across filesystems. + CrossDevice, + /// No space left on the device. + NoSpace, + /// A job id is in use by a job with another `requestId` or spec; choose + /// a new job id. + JobIdInUse, + /// A transfer `operationId` is in use for different input. + OperationIdInUse, + /// The transfer action is not valid in the operation's current phase. + TransferPhase, + /// A transfer write's `offset` is not the digest's next expected offset. + TransferOffset, + /// Uploaded bytes do not hash to their digest. + DigestMismatch, + /// The environment is closed; its identity or target is spent. + EnvironmentClosed, + /// The daemon identity is bound to another universe. + IdentityInUse, + /// The target belongs to another owner. + OwnerMismatch, + /// The template is deprecated and creates no new targets. + TemplateDeprecated, + /// `ingress/ensure` on a target whose template has no public ingress. + IngressUnsupported, + /// A power change on a target whose `phase` is not `active` + /// (provisioning, closing, closed or failed). + TargetNotActive, +} + +/// Why a `permissionDenied` request was refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum PermissionReason { + /// The path, or a symlink it passes through, resolves outside the + /// daemon's filesystem root. + OutsideRoot, + /// The path names or contains the daemon's state directory. + DaemonState, + /// The operating system refused access. + FilePermission, + /// The registration signature does not verify. + InvalidSignature, + /// The registration key is unknown. + InvalidRegistrationKey, + /// The registration key was revoked. + RegistrationKeyRevoked, +} + +/// What limit a `busy` request hit. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum BusyReason { + /// The connection's concurrent request limit. + RequestLimit, + /// The daemon's limit on active transfer operations. + TransferCapacity, + /// A rate limit on the caller's identity. + RateLimited, + /// A registration key's limit on active environments. + CapacityExhausted, +} + +/// Why an operation was `aborted`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub struct EnvironmentProtocolError { - pub code: EnvironmentProtocolErrorCode, +pub enum AbortReason { + /// `$/cancelRequest` named the request. + Cancelled, + /// A file or directory changed while it was being read. + SourceChanged, +} + +/// A capability a request needed and the endpoint lacks. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum Capability { + /// The `filesystem` family. + Filesystem, + /// `capabilities.filesystem.write`: the filesystem is writable. + FilesystemWrite, + /// The `process` family. + Process, + /// `capabilities.process.pty`. + ProcessPty, + /// The `job` family. + Job, + /// The controller's `power` capability. + Power, + /// The controller's `ingress` capability. + Ingress, +} + +/// The `data` member of an error object: the kind and its typed fields. +/// +/// Kinds without fields are empty struct variants rather than unit +/// variants: serde does not apply `deny_unknown_fields` to the unit +/// variants of an internally tagged enum, and every kind must refuse +/// unknown members. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum ErrorData { + ParseError {}, + InvalidRequest {}, + MethodNotFound {}, + InvalidParams { + /// Wire path of the offending field, such as `limits.maxDurationMs`. + #[serde(default, skip_serializing_if = "Option::is_none")] + field: Option, + /// The advertised ceiling the value exceeded. + #[serde(default, skip_serializing_if = "Option::is_none")] + ceiling: Option, + }, + NotInitialized {}, + VersionMismatch { + /// The version of the peer that sent this error. + expected: u32, + /// The version the requester stated. + received: u32, + }, + NotFound { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + }, + AlreadyExists {}, + FailedPrecondition { + reason: PreconditionReason, + }, + PermissionDenied { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + }, + CapabilityUnavailable { + capability: Capability, + }, + Busy { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + /// Earliest useful retry, when the server knows it. + #[serde(default, skip_serializing_if = "Option::is_none")] + retry_after_ms: Option, + }, + DeadlineExceeded {}, + Expired {}, + Aborted { + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + }, + PayloadTooLarge { + /// The byte limit the payload exceeded. + #[serde(default, skip_serializing_if = "Option::is_none")] + limit_bytes: Option, + /// The entry-count limit the result exceeded. + #[serde(default, skip_serializing_if = "Option::is_none")] + limit_entries: Option, + }, + ProcessFailed {}, + Unavailable { + #[serde(default, skip_serializing_if = "Option::is_none")] + retry_after_ms: Option, + }, + Internal {}, +} + +impl ErrorData { + pub fn kind(&self) -> ProtocolErrorKind { + match self { + Self::ParseError {} => ProtocolErrorKind::ParseError, + Self::InvalidRequest {} => ProtocolErrorKind::InvalidRequest, + Self::MethodNotFound {} => ProtocolErrorKind::MethodNotFound, + Self::InvalidParams { .. } => ProtocolErrorKind::InvalidParams, + Self::NotInitialized {} => ProtocolErrorKind::NotInitialized, + Self::VersionMismatch { .. } => ProtocolErrorKind::VersionMismatch, + Self::NotFound { .. } => ProtocolErrorKind::NotFound, + Self::AlreadyExists {} => ProtocolErrorKind::AlreadyExists, + Self::FailedPrecondition { .. } => ProtocolErrorKind::FailedPrecondition, + Self::PermissionDenied { .. } => ProtocolErrorKind::PermissionDenied, + Self::CapabilityUnavailable { .. } => ProtocolErrorKind::CapabilityUnavailable, + Self::Busy { .. } => ProtocolErrorKind::Busy, + Self::DeadlineExceeded {} => ProtocolErrorKind::DeadlineExceeded, + Self::Expired {} => ProtocolErrorKind::Expired, + Self::Aborted { .. } => ProtocolErrorKind::Aborted, + Self::PayloadTooLarge { .. } => ProtocolErrorKind::PayloadTooLarge, + Self::ProcessFailed {} => ProtocolErrorKind::ProcessFailed, + Self::Unavailable { .. } => ProtocolErrorKind::Unavailable, + Self::Internal {} => ProtocolErrorKind::Internal, + } + } + + /// The fieldless data of a kind, for kinds whose fields are all + /// optional. Kinds with required fields get their typed constructor on + /// [`ProtocolError`] instead. + fn bare(kind: ProtocolErrorKind) -> Option { + Some(match kind { + ProtocolErrorKind::ParseError => Self::ParseError {}, + ProtocolErrorKind::InvalidRequest => Self::InvalidRequest {}, + ProtocolErrorKind::MethodNotFound => Self::MethodNotFound {}, + ProtocolErrorKind::InvalidParams => Self::InvalidParams { + field: None, + ceiling: None, + }, + ProtocolErrorKind::NotInitialized => Self::NotInitialized {}, + ProtocolErrorKind::NotFound => Self::NotFound { reason: None }, + ProtocolErrorKind::AlreadyExists => Self::AlreadyExists {}, + ProtocolErrorKind::PermissionDenied => Self::PermissionDenied { reason: None }, + ProtocolErrorKind::Busy => Self::Busy { + reason: None, + retry_after_ms: None, + }, + ProtocolErrorKind::DeadlineExceeded => Self::DeadlineExceeded {}, + ProtocolErrorKind::Expired => Self::Expired {}, + ProtocolErrorKind::Aborted => Self::Aborted { reason: None }, + ProtocolErrorKind::ProcessFailed => Self::ProcessFailed {}, + ProtocolErrorKind::Unavailable => Self::Unavailable { + retry_after_ms: None, + }, + ProtocolErrorKind::Internal => Self::Internal {}, + ProtocolErrorKind::PayloadTooLarge => Self::PayloadTooLarge { + limit_bytes: None, + limit_entries: None, + }, + ProtocolErrorKind::VersionMismatch + | ProtocolErrorKind::FailedPrecondition + | ProtocolErrorKind::CapabilityUnavailable => return None, + }) + } +} + +/// A JSON-RPC error object as it appears on the wire. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ErrorObject { + /// Fixed by `data.kind`; see [`ProtocolErrorKind::code`]. + pub code: i32, + /// Human-readable description. Never parsed. pub message: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub details: Option, + pub data: ErrorData, } -impl EnvironmentProtocolError { - pub fn new(code: EnvironmentProtocolErrorCode, message: impl Into) -> Self { +/// A protocol error: one kind, its typed data and a message. +/// +/// Serializes as an [`ErrorObject`]. Displays as `{kind}: {message}`. +#[derive(Clone, Debug, PartialEq, Eq, Error, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "schema", schemars(with = "ErrorObject"))] +#[serde(from = "ErrorObject", into = "ErrorObject")] +#[error("{}: {message}", self.kind())] +pub struct ProtocolError { + pub data: ErrorData, + pub message: String, +} + +impl From for ProtocolError { + /// `data.kind` is authoritative; `code` is not consulted. + fn from(object: ErrorObject) -> Self { Self { - code, - message: message.into(), - details: None, + data: object.data, + message: object.message, } } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum EnvironmentProtocolErrorCode { - InvalidRequest, - Unauthorized, - Forbidden, - NotFound, - Conflict, - Unsupported, - CapabilityUnavailable, - ProcessFailed, - Timeout, - TransportClosed, - Internal, +impl From for ErrorObject { + fn from(error: ProtocolError) -> Self { + Self { + code: error.code(), + message: error.message, + data: error.data, + } + } +} + +impl ProtocolError { + /// An error of this kind. A message longer than + /// [`MAX_ERROR_MESSAGE_BYTES`] is cut on a character boundary, so errors + /// embedded in results stay bounded. + pub fn new(data: ErrorData, message: impl Into) -> Self { + let mut message = message.into(); + let max = MAX_ERROR_MESSAGE_BYTES as usize; + if message.len() > max { + let mut end = max; + while !message.is_char_boundary(end) { + end -= 1; + } + message.truncate(end); + } + Self { data, message } + } + + pub fn kind(&self) -> ProtocolErrorKind { + self.data.kind() + } + + pub fn code(&self) -> i32 { + self.kind().code() + } + + pub fn is_retryable(&self) -> bool { + self.kind().is_retryable() + } + + /// Decode an error object tolerantly: a conforming object keeps its + /// typed data; an object without valid `data` (from a non-conforming + /// peer or an intermediary) maps its code to a kind, falling back to + /// `internal`. + pub fn from_error_value(value: &Value) -> Self { + if let Ok(object) = serde_json::from_value::(value.clone()) { + return object.into(); + } + let message = value + .get("message") + .and_then(Value::as_str) + .unwrap_or("JSON-RPC error") + .to_owned(); + let data = value + .get("code") + .and_then(Value::as_i64) + .and_then(ProtocolErrorKind::from_code) + .and_then(ErrorData::bare) + .unwrap_or(ErrorData::Internal {}); + Self { data, message } + } + + pub fn parse_error(message: impl Into) -> Self { + Self::new(ErrorData::ParseError {}, message) + } + + pub fn invalid_request(message: impl Into) -> Self { + Self::new(ErrorData::InvalidRequest {}, message) + } + + pub fn method_not_found(method: &str) -> Self { + Self::new( + ErrorData::MethodNotFound {}, + format!("method not found: {method}"), + ) + } + + pub fn invalid_params(message: impl Into) -> Self { + Self::new( + ErrorData::InvalidParams { + field: None, + ceiling: None, + }, + message, + ) + } + + /// `invalidParams` naming the offending field by its wire path. + pub fn invalid_field(field: impl Into, message: impl Into) -> Self { + Self::new( + ErrorData::InvalidParams { + field: Some(field.into()), + ceiling: None, + }, + message, + ) + } + + /// `invalidParams` for a value above an advertised ceiling. + pub fn above_ceiling(field: impl Into, value: u64, ceiling: u64) -> Self { + let field = field.into(); + let message = format!("{field} is {value}; the ceiling is {ceiling}"); + Self::new( + ErrorData::InvalidParams { + field: Some(field), + ceiling: Some(ceiling), + }, + message, + ) + } + + pub fn not_initialized() -> Self { + Self::new( + ErrorData::NotInitialized {}, + "initialize must be the first request on a connection", + ) + } + + pub fn version_mismatch(expected: u32, received: u32) -> Self { + Self::new( + ErrorData::VersionMismatch { expected, received }, + format!("protocol version {received} is not supported; this peer speaks {expected}"), + ) + } + + pub fn not_found(message: impl Into) -> Self { + Self::new(ErrorData::NotFound { reason: None }, message) + } + + pub fn not_found_because(reason: NotFoundReason, message: impl Into) -> Self { + Self::new( + ErrorData::NotFound { + reason: Some(reason), + }, + message, + ) + } + + pub fn already_exists(message: impl Into) -> Self { + Self::new(ErrorData::AlreadyExists {}, message) + } + + pub fn failed_precondition(reason: PreconditionReason, message: impl Into) -> Self { + Self::new(ErrorData::FailedPrecondition { reason }, message) + } + + pub fn permission_denied(reason: Option, message: impl Into) -> Self { + Self::new(ErrorData::PermissionDenied { reason }, message) + } + + pub fn capability_unavailable(capability: Capability, message: impl Into) -> Self { + Self::new(ErrorData::CapabilityUnavailable { capability }, message) + } + + pub fn busy( + reason: Option, + retry_after_ms: Option, + message: impl Into, + ) -> Self { + Self::new( + ErrorData::Busy { + reason, + retry_after_ms, + }, + message, + ) + } + + pub fn deadline_exceeded(message: impl Into) -> Self { + Self::new(ErrorData::DeadlineExceeded {}, message) + } + + pub fn expired(message: impl Into) -> Self { + Self::new(ErrorData::Expired {}, message) + } + + pub fn aborted(reason: Option, message: impl Into) -> Self { + Self::new(ErrorData::Aborted { reason }, message) + } + + /// `aborted` for a request named by `$/cancelRequest`. + pub fn cancelled() -> Self { + Self::aborted(Some(AbortReason::Cancelled), "request cancelled") + } + + pub fn payload_too_large(limit_bytes: u64, message: impl Into) -> Self { + Self::new( + ErrorData::PayloadTooLarge { + limit_bytes: Some(limit_bytes), + limit_entries: None, + }, + message, + ) + } + + /// `payloadTooLarge` for a result with more entries than allowed. + pub fn too_many_entries(limit_entries: u32, message: impl Into) -> Self { + Self::new( + ErrorData::PayloadTooLarge { + limit_bytes: None, + limit_entries: Some(limit_entries), + }, + message, + ) + } + + pub fn process_failed(message: impl Into) -> Self { + Self::new(ErrorData::ProcessFailed {}, message) + } + + pub fn unavailable(message: impl Into) -> Self { + Self::new( + ErrorData::Unavailable { + retry_after_ms: None, + }, + message, + ) + } + + pub fn internal(message: impl Into) -> Self { + Self::new(ErrorData::Internal {}, message) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn errors_serialize_as_json_rpc_error_objects_with_the_kind_code() { + let error = ProtocolError::not_found_because(NotFoundReason::Expired, "unknown process"); + let value = serde_json::to_value(&error).unwrap(); + assert_eq!( + value, + json!({"code": -32004, "message": "unknown process", "data": {"kind": "notFound", "reason": "expired"}}) + ); + assert_eq!( + serde_json::from_value::(value).unwrap(), + error + ); + assert_eq!(error.to_string(), "notFound: unknown process"); + } + + #[test] + fn every_kind_has_a_distinct_code_that_maps_back() { + let mut codes = std::collections::BTreeSet::new(); + for kind in ProtocolErrorKind::ALL { + assert!(codes.insert(kind.code()), "duplicate code for {kind}"); + assert_eq!(ProtocolErrorKind::from_code(kind.code().into()), Some(kind)); + } + } + + #[test] + fn retryability_follows_the_kind() { + let retryable: Vec<_> = ProtocolErrorKind::ALL + .into_iter() + .filter(|kind| kind.is_retryable()) + .collect(); + assert_eq!( + retryable, + [ + ProtocolErrorKind::Busy, + ProtocolErrorKind::Aborted, + ProtocolErrorKind::Unavailable + ] + ); + } + + #[test] + fn foreign_error_objects_map_reserved_codes_and_fall_back_to_internal() { + let parsed = ProtocolError::from_error_value(&json!({"code": -32601, "message": "nope"})); + assert_eq!(parsed.kind(), ProtocolErrorKind::MethodNotFound); + let parsed = ProtocolError::from_error_value(&json!({"code": "weird"})); + assert_eq!(parsed.kind(), ProtocolErrorKind::Internal); + assert_eq!(parsed.message, "JSON-RPC error"); + } + + #[test] + fn unknown_data_members_are_rejected() { + let value = + json!({"code": -32004, "message": "m", "data": {"kind": "notFound", "extra": 1}}); + assert!(serde_json::from_value::(value).is_err()); + } + + /// One error of every kind, with every optional field set where the kind + /// has one. + fn one_of_each_kind() -> Vec { + vec![ + ProtocolError::parse_error("m"), + ProtocolError::invalid_request("m"), + ProtocolError::method_not_found("x/y"), + ProtocolError::above_ceiling("waitMs", 2, 1), + ProtocolError::not_initialized(), + ProtocolError::version_mismatch(3, 2), + ProtocolError::not_found_because(NotFoundReason::DaemonRestarted, "m"), + ProtocolError::already_exists("m"), + ProtocolError::failed_precondition(PreconditionReason::TargetNotActive, "m"), + ProtocolError::permission_denied(Some(PermissionReason::OutsideRoot), "m"), + ProtocolError::capability_unavailable(Capability::ProcessPty, "m"), + ProtocolError::busy(Some(BusyReason::RateLimited), Some(500), "m"), + ProtocolError::deadline_exceeded("m"), + ProtocolError::expired("m"), + ProtocolError::aborted(Some(AbortReason::SourceChanged), "m"), + ProtocolError::new( + ErrorData::PayloadTooLarge { + limit_bytes: Some(1), + limit_entries: Some(2), + }, + "m", + ), + ProtocolError::process_failed("m"), + ProtocolError::new( + ErrorData::Unavailable { + retry_after_ms: Some(10), + }, + "m", + ), + ProtocolError::internal("m"), + ] + } + + #[test] + fn every_kind_round_trips_with_its_code() { + let errors = one_of_each_kind(); + let kinds: Vec<_> = errors.iter().map(ProtocolError::kind).collect(); + assert_eq!(kinds, ProtocolErrorKind::ALL); + for error in errors { + let value = serde_json::to_value(&error).unwrap(); + assert_eq!(value["code"], json!(error.code())); + assert_eq!(value["data"]["kind"], json!(error.kind().as_str())); + assert_eq!( + serde_json::from_value::(value).unwrap(), + error + ); + } + } + + #[test] + fn every_kind_rejects_unknown_data_members() { + for error in one_of_each_kind() { + let mut value = serde_json::to_value(&error).unwrap(); + value["data"]["unexpectedMember"] = json!(1); + assert!( + serde_json::from_value::(value).is_err(), + "{} accepts an unknown member", + error.kind() + ); + } + } + + #[test] + fn fieldless_kinds_serialize_as_the_kind_alone() { + assert_eq!( + serde_json::to_value(ErrorData::NotInitialized {}).unwrap(), + json!({"kind": "notInitialized"}) + ); + } + + #[test] + fn long_messages_are_cut_on_a_character_boundary() { + let error = ProtocolError::internal("\u{e9}".repeat(MAX_ERROR_MESSAGE_BYTES as usize)); + assert!(error.message.len() <= MAX_ERROR_MESSAGE_BYTES as usize); + assert!(error.message.len() > MAX_ERROR_MESSAGE_BYTES as usize - 2); + } } diff --git a/crates/environment-protocol/src/gateway.rs b/crates/environment-protocol/src/gateway.rs deleted file mode 100644 index 93445e36f..000000000 --- a/crates/environment-protocol/src/gateway.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Stable environment gateway route paths. - -pub const ROUTE_PATH_PREFIX: &str = "/environment-gateway/routes"; - -/// Appended to a provider controller base URL for Lightspeed-initiated data -/// connections. The remainder is -/// `/{universe}/{binding}/{environment}/{incarnation}/{target}`. -pub const PROVIDER_DATA_PATH_PREFIX: &str = "/routes"; diff --git a/crates/environment-protocol/src/handshake.rs b/crates/environment-protocol/src/handshake.rs new file mode 100644 index 000000000..ccc6ba291 --- /dev/null +++ b/crates/environment-protocol/src/handshake.rs @@ -0,0 +1,257 @@ +//! Values every handshake shares. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use thiserror::Error; + +#[cfg(feature = "schema")] +use crate::ids::NO_CONTROL_PATTERN; +use crate::limits::MAX_IMPLEMENTATION_FIELD_BYTES; + +/// The plane a connection serves. Each `initialize` response names its +/// plane, so a client that reached the wrong socket fails at the handshake. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum Plane { + /// envd's data plane, versioned by `ENVD_PROTOCOL_VERSION`. + Data, + /// A provider controller, versioned by `CONTROLLER_PROTOCOL_VERSION`. + Controller, + /// The registration channel between a daemon and the gateway, versioned + /// by `ENVD_PROTOCOL_VERSION`. + Registration, +} + +impl Plane { + pub fn as_str(self) -> &'static str { + match self { + Self::Data => "data", + Self::Controller => "controller", + Self::Registration => "registration", + } + } +} + +impl std::fmt::Display for Plane { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// What a peer says about its own build: `clientInfo`, `serverInfo` and a +/// daemon's registration `implementation`. Descriptive only; the protocol +/// version alone decides whether two peers can talk, and nothing here +/// selects, authenticates or gates. Every field is 1 to 256 bytes without control characters, +/// checked on decode and by every constructor, so a value that would be +/// refused can never be built or sent. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ImplementationInfo { + /// Product name, such as `lightspeed-envd`. + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 256), pattern(NO_CONTROL_PATTERN)) + )] + name: String, + /// Release version, such as `0.5.0`. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 256), pattern(NO_CONTROL_PATTERN)) + )] + version: Option, + /// Full or abbreviated git commit of the build. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 256), pattern(NO_CONTROL_PATTERN)) + )] + git_sha: Option, + /// Rust target triple of the build. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 256), pattern(NO_CONTROL_PATTERN)) + )] + target: Option, +} + +/// The decoded shape before the bounds are checked. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ImplementationInfoFields { + name: String, + #[serde(default)] + version: Option, + #[serde(default)] + git_sha: Option, + #[serde(default)] + target: Option, +} + +impl ImplementationInfoFields { + fn into_info(self) -> Result { + let info = ImplementationInfo { + name: self.name, + version: self.version, + git_sha: self.git_sha, + target: self.target, + }; + check_field("name", Some(&info.name))?; + check_field("version", info.version.as_deref())?; + check_field("gitSha", info.git_sha.as_deref())?; + check_field("target", info.target.as_deref())?; + Ok(info) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +#[error("{field} must be 1..={max} bytes without control characters", max = MAX_IMPLEMENTATION_FIELD_BYTES)] +pub struct ImplementationInfoError { + /// Wire name of the offending member. + pub field: &'static str, +} + +impl ImplementationInfo { + pub fn new(name: impl Into) -> Result { + let name = name.into(); + check_field("name", Some(&name))?; + Ok(Self { + name, + version: None, + git_sha: None, + target: None, + }) + } + + pub fn with_version( + mut self, + version: impl Into, + ) -> Result { + let version = version.into(); + check_field("version", Some(&version))?; + self.version = Some(version); + Ok(self) + } + + pub fn with_git_sha( + mut self, + git_sha: impl Into, + ) -> Result { + let git_sha = git_sha.into(); + check_field("gitSha", Some(&git_sha))?; + self.git_sha = Some(git_sha); + Ok(self) + } + + pub fn with_target( + mut self, + target: impl Into, + ) -> Result { + let target = target.into(); + check_field("target", Some(&target))?; + self.target = Some(target); + Ok(self) + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn version(&self) -> Option<&str> { + self.version.as_deref() + } + + pub fn git_sha(&self) -> Option<&str> { + self.git_sha.as_deref() + } + + pub fn target(&self) -> Option<&str> { + self.target.as_deref() + } + + /// For a value that has the shape of an `ImplementationInfo` but breaks + /// its bounds, the offending member. `None` when the value decodes, or + /// when it does not even have the shape (a strict decode reports that). + pub fn bounds_violation(value: &Value) -> Option { + ImplementationInfoFields::deserialize(value) + .ok()? + .into_info() + .err() + } +} + +impl<'de> Deserialize<'de> for ImplementationInfo { + fn deserialize>(deserializer: D) -> Result { + ImplementationInfoFields::deserialize(deserializer)? + .into_info() + .map_err(serde::de::Error::custom) + } +} + +fn check_field(field: &'static str, value: Option<&str>) -> Result<(), ImplementationInfoError> { + match value { + Some(value) + if value.is_empty() + || value.len() > MAX_IMPLEMENTATION_FIELD_BYTES as usize + || value.chars().any(char::is_control) => + { + Err(ImplementationInfoError { field }) + } + _ => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn implementation_info_is_bounded_on_decode() { + let ok = serde_json::json!({"name": "lightspeed-envd", "version": "0.5.0"}); + assert!(serde_json::from_value::(ok).is_ok()); + for bad in [ + serde_json::json!({"name": ""}), + serde_json::json!({"name": "x", "version": "a\nb"}), + serde_json::json!({"name": "x", "gitSha": "a".repeat(257)}), + serde_json::json!({"name": "x", "extra": 1}), + ] { + assert!(serde_json::from_value::(bad).is_err()); + } + } + + #[test] + fn constructors_enforce_the_decode_bounds() { + let info = ImplementationInfo::new("lightspeed-envd") + .and_then(|info| info.with_version("0.5.0")) + .unwrap(); + assert_eq!(info.version(), Some("0.5.0")); + assert_eq!( + ImplementationInfo::new(""), + Err(ImplementationInfoError { field: "name" }) + ); + assert_eq!( + info.with_git_sha("a\nb"), + Err(ImplementationInfoError { field: "gitSha" }) + ); + } + + #[test] + fn bounds_violations_are_told_apart_from_shape_errors() { + let long = serde_json::json!({"name": "x", "version": "v".repeat(257)}); + assert_eq!( + ImplementationInfo::bounds_violation(&long), + Some(ImplementationInfoError { field: "version" }) + ); + assert_eq!( + ImplementationInfo::bounds_violation(&serde_json::json!({"name": 1})), + None + ); + assert_eq!( + ImplementationInfo::bounds_violation(&serde_json::json!({"name": "ok"})), + None + ); + } +} diff --git a/crates/environment-protocol/src/idle.rs b/crates/environment-protocol/src/idle.rs new file mode 100644 index 000000000..ea4cf538a --- /dev/null +++ b/crates/environment-protocol/src/idle.rs @@ -0,0 +1,244 @@ +//! Idle policy: when a provider powers down or closes an idle target. +//! +//! The policy belongs to the provider. A template names a default +//! (`template/list` `idle`), `target/create` may override it (`idle`), and +//! the provider runs the clock: it reads envd `env/getIdle` over its own +//! guest channel and applies [`IdlePolicy::due_action`] to `idleForMs`. +//! Running processes, jobs and leftover process groups hold `idleForMs` at +//! zero, so they keep the target awake. A provider never powers a target +//! down while it relays a data connection to it. Lightspeed only validates +//! a policy and passes it through. + +use std::fmt; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::control::target::PowerState; + +/// Staged idle policy. Each set threshold is measured against envd +/// `env/getIdle` `idleForMs`, and the set thresholds are non-decreasing in +/// the order pause → suspend → stop → close. Every stage is optional; a +/// policy sets at least one. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct IdlePolicy { + /// Pause the target after this long idle. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pause_after_ms: Option, + /// Suspend the target after this long idle. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub suspend_after_ms: Option, + /// Stop the target after this long idle. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop_after_ms: Option, + /// Close the target after this long idle; the provider then reports it + /// `closed`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub close_after_ms: Option, +} + +/// One stage of an idle policy, in escalation order. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum IdleAction { + Pause, + Suspend, + Stop, + Close, +} + +impl IdleAction { + /// The power state this stage moves a target to; `None` for close. + pub fn power_state(self) -> Option { + match self { + Self::Pause => Some(PowerState::Paused), + Self::Suspend => Some(PowerState::Suspended), + Self::Stop => Some(PowerState::Stopped), + Self::Close => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Pause => "pause", + Self::Suspend => "suspend", + Self::Stop => "stop", + Self::Close => "close", + } + } +} + +impl fmt::Display for IdleAction { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Why an idle policy is invalid. +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum IdlePolicyError { + #[error("idle policy sets no stage")] + Empty, + + #[error("idle policy {0} threshold must be positive")] + ZeroThreshold(IdleAction), + + #[error("idle policy {later} threshold must not be below {earlier}")] + OutOfOrder { + earlier: IdleAction, + later: IdleAction, + }, +} + +impl IdlePolicy { + pub fn is_empty(&self) -> bool { + self.pause_after_ms.is_none() + && self.suspend_after_ms.is_none() + && self.stop_after_ms.is_none() + && self.close_after_ms.is_none() + } + + /// The set stages in escalation order, with their thresholds. + pub fn stages(&self) -> Vec<(IdleAction, u64)> { + [ + (IdleAction::Pause, self.pause_after_ms), + (IdleAction::Suspend, self.suspend_after_ms), + (IdleAction::Stop, self.stop_after_ms), + (IdleAction::Close, self.close_after_ms), + ] + .into_iter() + .filter_map(|(action, threshold)| threshold.map(|threshold| (action, threshold))) + .collect() + } + + /// At least one stage, every threshold positive, thresholds + /// non-decreasing in escalation order. + pub fn validate(&self) -> Result<(), IdlePolicyError> { + if self.is_empty() { + return Err(IdlePolicyError::Empty); + } + let mut previous: Option<(IdleAction, u64)> = None; + for (action, threshold) in self.stages() { + if threshold == 0 { + return Err(IdlePolicyError::ZeroThreshold(action)); + } + if let Some((earlier, earlier_threshold)) = previous + && threshold < earlier_threshold + { + return Err(IdlePolicyError::OutOfOrder { + earlier, + later: action, + }); + } + previous = Some((action, threshold)); + } + Ok(()) + } + + /// The most escalated stage whose threshold `idle_for_ms` has crossed + /// and whose power state is in `supported` (`close` always qualifies); + /// `None` while no stage applies. + pub fn due_action(&self, idle_for_ms: u64, supported: &[PowerState]) -> Option { + self.stages() + .into_iter() + .filter(|(_, threshold)| idle_for_ms >= *threshold) + .filter(|(action, _)| { + action + .power_state() + .is_none_or(|state| supported.contains(&state)) + }) + .map(|(action, _)| action) + .next_back() + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn full() -> IdlePolicy { + IdlePolicy { + pause_after_ms: Some(10), + suspend_after_ms: Some(20), + stop_after_ms: Some(30), + close_after_ms: Some(40), + } + } + + #[test] + fn validation_requires_a_positive_non_decreasing_stage() { + assert_eq!(full().validate(), Ok(())); + assert_eq!( + IdlePolicy::default().validate(), + Err(IdlePolicyError::Empty) + ); + assert_eq!( + IdlePolicy { + pause_after_ms: Some(0), + ..IdlePolicy::default() + } + .validate(), + Err(IdlePolicyError::ZeroThreshold(IdleAction::Pause)) + ); + assert_eq!( + IdlePolicy { + pause_after_ms: Some(30), + stop_after_ms: Some(10), + ..IdlePolicy::default() + } + .validate(), + Err(IdlePolicyError::OutOfOrder { + earlier: IdleAction::Pause, + later: IdleAction::Stop + }) + ); + assert_eq!( + IdlePolicy { + stop_after_ms: Some(10), + close_after_ms: Some(10), + ..IdlePolicy::default() + } + .validate(), + Ok(()), + "equal thresholds are allowed" + ); + } + + #[test] + fn the_most_escalated_supported_stage_is_due() { + let policy = full(); + let incus = [PowerState::Running, PowerState::Paused, PowerState::Stopped]; + assert_eq!(policy.due_action(5, &incus), None); + assert_eq!(policy.due_action(10, &incus), Some(IdleAction::Pause)); + // Suspend is due but unsupported: pause stays the most escalated + // applicable stage. + assert_eq!(policy.due_action(25, &incus), Some(IdleAction::Pause)); + assert_eq!(policy.due_action(30, &incus), Some(IdleAction::Stop)); + assert_eq!(policy.due_action(40, &incus), Some(IdleAction::Close)); + // Close needs no power support. + assert_eq!(policy.due_action(40, &[]), Some(IdleAction::Close)); + assert_eq!(policy.due_action(30, &[]), None); + } + + #[test] + fn the_wire_shape_is_camel_case_and_strict() { + let policy = IdlePolicy { + pause_after_ms: Some(600_000), + close_after_ms: Some(86_400_000), + ..IdlePolicy::default() + }; + let value = serde_json::to_value(&policy).unwrap(); + assert_eq!( + value, + json!({"pauseAfterMs": 600_000, "closeAfterMs": 86_400_000}) + ); + assert_eq!(serde_json::from_value::(value).unwrap(), policy); + assert!(serde_json::from_value::(json!({"pauseAfter": 1})).is_err()); + assert_eq!(IdleAction::Suspend.to_string(), "suspend"); + assert_eq!(IdleAction::Close.power_state(), None); + } +} diff --git a/crates/environment-protocol/src/ids.rs b/crates/environment-protocol/src/ids.rs new file mode 100644 index 000000000..f913540e4 --- /dev/null +++ b/crates/environment-protocol/src/ids.rs @@ -0,0 +1,318 @@ +//! Typed identifiers. +//! +//! Every id space has its own type with a private field and a validating +//! decoder. Two grammars exist: +//! +//! - **General** (Lightspeed-assigned ids): 1 to 128 bytes, an ASCII letter +//! or digit first, then ASCII letters, digits, `_`, `-`, `.` or `:`. +//! - **Opaque** (provider-assigned ids and daemon-minted tokens): 1 to 255 +//! bytes without control characters. Container, Kubernetes and cloud ids +//! such as `default/pod-abc` or an ARN are valid as they are. +//! +//! [`JobName`] is a third, caller-chosen name space with its own rule. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +pub const GENERAL_ID_MAX_BYTES: u32 = 128; +pub const OPAQUE_ID_MAX_BYTES: u32 = 255; +pub const JOB_NAME_MAX_BYTES: u32 = 128; + +/// JSON Schema pattern of the general grammar. +pub const GENERAL_ID_PATTERN: &str = "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$"; +/// JSON Schema pattern of a string without control characters (Unicode +/// category Cc), the character rule of opaque ids and job names. +pub const NO_CONTROL_PATTERN: &str = "^[^\\u0000-\\u001F\\u007F-\\u009F]+$"; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum IdError { + #[error("{kind} must not be empty")] + Empty { kind: &'static str }, + + #[error("{kind} is {actual} bytes; the maximum is {max}")] + TooLong { + kind: &'static str, + max: u32, + actual: usize, + }, + + #[error("{kind} must start with an ASCII letter or digit")] + InvalidStart { kind: &'static str }, + + #[error( + "{kind} contains {character:?} at byte {index}; allowed are ASCII letters, digits, '_', '-', '.', ':'" + )] + InvalidCharacter { + kind: &'static str, + index: usize, + character: char, + }, + + #[error("{kind} contains a control character at byte {index}")] + ControlCharacter { kind: &'static str, index: usize }, +} + +/// Check the general id grammar. +pub fn validate_general_id(kind: &'static str, value: &str) -> Result<(), IdError> { + check_length(kind, value, GENERAL_ID_MAX_BYTES)?; + if !value.as_bytes()[0].is_ascii_alphanumeric() { + return Err(IdError::InvalidStart { kind }); + } + if let Some((index, character)) = value + .char_indices() + .find(|(_, character)| !(character.is_ascii_alphanumeric() || "_-.:".contains(*character))) + { + return Err(IdError::InvalidCharacter { + kind, + index, + character, + }); + } + Ok(()) +} + +/// Check the opaque id grammar. +pub fn validate_opaque_id(kind: &'static str, value: &str) -> Result<(), IdError> { + check_length(kind, value, OPAQUE_ID_MAX_BYTES)?; + check_no_control(kind, value) +} + +/// Check the job name rule: 1 to [`JOB_NAME_MAX_BYTES`] bytes without +/// control characters. +pub fn validate_job_name(kind: &'static str, value: &str) -> Result<(), IdError> { + check_length(kind, value, JOB_NAME_MAX_BYTES)?; + check_no_control(kind, value) +} + +fn check_no_control(kind: &'static str, value: &str) -> Result<(), IdError> { + if let Some((index, _)) = value + .char_indices() + .find(|(_, character)| character.is_control()) + { + return Err(IdError::ControlCharacter { kind, index }); + } + Ok(()) +} + +fn check_length(kind: &'static str, value: &str, max: u32) -> Result<(), IdError> { + if value.is_empty() { + return Err(IdError::Empty { kind }); + } + if value.len() > max as usize { + return Err(IdError::TooLong { + kind, + max, + actual: value.len(), + }); + } + Ok(()) +} + +macro_rules! protocol_id { + ($(#[$doc:meta])* $name:ident, general) => { + protocol_id!( + $(#[$doc])* $name, + validate_general_id, + length(min = 1, max = 128), + pattern(GENERAL_ID_PATTERN) + ); + }; + ($(#[$doc:meta])* $name:ident, opaque) => { + protocol_id!( + $(#[$doc])* $name, + validate_opaque_id, + length(min = 1, max = 255), + pattern(NO_CONTROL_PATTERN) + ); + }; + ($(#[$doc:meta])* $name:ident, $validate:ident, $($schema:tt)*) => { + $(#[$doc])* + #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] + pub struct $name(#[cfg_attr(feature = "schema", schemars($($schema)*))] String); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + $validate(stringify!($name), &value)?; + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } + } + + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } + } + + impl FromStr for $name { + type Err = IdError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } + } + + impl TryFrom for $name { + type Error = IdError; + + fn try_from(value: String) -> Result { + Self::new(value) + } + } + + impl TryFrom<&str> for $name { + type Error = IdError; + + fn try_from(value: &str) -> Result { + Self::new(value) + } + } + + impl AsRef for $name { + fn as_ref(&self) -> &str { + &self.0 + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } + } + }; +} + +protocol_id!( + /// `processId`: caller-chosen handle of a process on one daemon. + ProcessId, + general +); +protocol_id!( + /// `jobId`: caller-chosen name of one job run on one daemon. + JobId, + general +); +protocol_id!( + /// `requestId` of a `job/start` call. Part of each job's identity: a + /// retried start with the same request id and spec replays the job; any + /// other reuse of the job id is refused. + JobRequestId, + general +); +protocol_id!( + /// `queueKey`: jobs sharing a key run one at a time, in start order. + QueueKey, + general +); +protocol_id!( + /// `operationId` of one transfer operation. + TransferOperationId, + general +); +protocol_id!( + /// `daemonInstanceId`: random per daemon start. A change between two + /// connections means the daemon restarted and every process handle from + /// the earlier instance is gone. + DaemonInstanceId, + general +); +protocol_id!( + /// `universeId` of a Lightspeed universe. + UniverseId, + general +); +protocol_id!( + /// `environmentId` of a Lightspeed environment. + EnvironmentId, + general +); +protocol_id!( + /// `daemonId` the gateway assigns to an admitted daemon identity. + DaemonId, + general +); +protocol_id!( + /// `controlConnectionId` the gateway assigns to one control connection. + ControlConnectionId, + general +); +protocol_id!( + /// `targetId`: the provider's own id for a target, opaque to Lightspeed. + ProviderTargetId, + opaque +); +protocol_id!( + /// `templateId`: the provider's own id for a template. + TemplateId, + opaque +); +protocol_id!( + /// `name` of a job and `dependsOn {kind: name}`: a caller-chosen name, + /// unique within one `job/start`, 1 to 128 bytes without control + /// characters. + JobName, + validate_job_name, + length(min = 1, max = 128), + pattern(NO_CONTROL_PATTERN) +); +protocol_id!( + /// `cursor` and `nextCursor` of `fs/readDirectory`: an opaque position + /// the daemon mints, echoed back unchanged. + DirectoryCursor, + opaque +); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn general_ids_validate_on_decode() { + assert!(serde_json::from_str::("\"build.1:a-b_c\"").is_ok()); + for bad in ["\"\"", "\"-lead\"", "\"has space\"", "\"é\""] { + assert!(serde_json::from_str::(bad).is_err(), "{bad}"); + } + let long = format!("\"{}\"", "a".repeat(GENERAL_ID_MAX_BYTES as usize + 1)); + assert!(serde_json::from_str::(&long).is_err()); + } + + #[test] + fn opaque_ids_accept_natural_provider_ids() { + for good in [ + "default/pod-abc", + "arn:aws:ec2:eu-west-1:123:instance/i-0abc", + "ls-3f2a", + ] { + assert!(ProviderTargetId::new(good).is_ok(), "{good}"); + } + assert!(ProviderTargetId::new("a\nb").is_err()); + assert!(ProviderTargetId::new("").is_err()); + assert!(ProviderTargetId::new("x".repeat(OPAQUE_ID_MAX_BYTES as usize + 1)).is_err()); + } + + #[test] + fn job_names_allow_spaces_but_not_control_characters() { + assert!(JobName::new("build & test").is_ok()); + assert!(JobName::new("a\tb").is_err()); + assert!(JobName::new("").is_err()); + assert!(JobName::new("x".repeat(JOB_NAME_MAX_BYTES as usize + 1)).is_err()); + } +} diff --git a/crates/environment-protocol/src/jsonrpc.rs b/crates/environment-protocol/src/jsonrpc.rs new file mode 100644 index 000000000..617d17906 --- /dev/null +++ b/crates/environment-protocol/src/jsonrpc.rs @@ -0,0 +1,601 @@ +//! The JSON-RPC 2.0 envelope every socket uses: the data plane, the +//! controller plane and the registration channel. +//! +//! Rules, applied by [`parse`] and the constructors here: +//! +//! - One message per WebSocket text frame, at most +//! [`MAX_MESSAGE_BYTES`]. Binary frames +//! are not part of any plane. +//! - `jsonrpc` must be `"2.0"`. +//! - Params are named: an object, or absent, which means `{}`. +//! - A request has a string or integer `id`; a notification has none and is +//! never answered, not even with an error. A malformed frame that has a +//! string `method` and no `id` is a malformed notification: [`parse`] +//! rejects it, and a server drops it silently +//! ([`Rejection::into_response`] returns `None`). +//! - Batches (JSON arrays) are not supported and are answered with one +//! `invalidRequest` error with a null id. +//! - A frame that is not JSON is answered with `parseError` and a null id; +//! the connection stays open. +//! - A server answers every other rejection, with the request's id when it +//! could be read and a null id otherwise; a client that rejects a frame +//! closes the connection. +//! - Requests may be pipelined. Responses arrive in any order and are +//! matched by id. +//! - [`CANCEL_REQUEST_METHOD`] asks the server to abort a pending request. + +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; +use thiserror::Error; + +use crate::{ + error::{ErrorObject, ProtocolError}, + limits::MAX_MESSAGE_BYTES, +}; + +pub const JSONRPC_VERSION: &str = "2.0"; + +/// Client-to-server notification asking the server to abort the request +/// with this id. Read-only waits stop and answer `aborted` with reason +/// `cancelled`; methods with side effects run to a consistent state and +/// answer normally. An unknown or finished id is ignored. +pub const CANCEL_REQUEST_METHOD: &str = "$/cancelRequest"; + +/// A request id: an integer or a string, echoed in the response. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(untagged)] +pub enum Id { + Number(i64), + String(String), +} + +impl From for Id { + fn from(value: i64) -> Self { + Self::Number(value) + } +} + +impl std::fmt::Display for Id { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Number(number) => number.fmt(formatter), + Self::String(text) => text.fmt(formatter), + } + } +} + +/// `$/cancelRequest` params. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CancelRequestParams { + /// Id of the request to abort. + pub id: Id, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Request { + pub id: Id, + pub method: String, + /// Always a JSON object. + pub params: Value, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Notification { + pub method: String, + /// Always a JSON object. + pub params: Value, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Response { + /// Absent (null on the wire) only when the request's id could not be + /// read. + pub id: Option, + pub result: Result, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Message { + Request(Request), + Notification(Notification), + Response(Response), +} + +/// A frame that cannot be accepted, with the error to answer it with. +/// +/// A server answers a rejection with [`into_response`](Self::into_response) +/// unless the frame was a malformed notification, which is dropped +/// silently. A client that rejects a frame closes the connection. +#[derive(Clone, Debug, PartialEq)] +pub struct Rejection { + /// The rejected request's id; absent when it could not be read. + pub id: Option, + pub error: ProtocolError, + /// The frame had a string `method` and no `id`. + notification: bool, +} + +impl Rejection { + fn of_frame(error: ProtocolError) -> Self { + Self { + id: None, + error, + notification: false, + } + } + + /// True when the frame was a malformed notification: it must not be + /// answered. + pub fn is_notification(&self) -> bool { + self.notification + } + + /// The error response a server sends for this rejection, or `None` for + /// a malformed notification, which is never answered. + pub fn into_response(self) -> Option { + (!self.notification).then_some(Response { + id: self.id, + result: Err(self.error), + }) + } +} + +/// Classify one text frame. +pub fn parse(text: &str) -> Result { + let value: Value = serde_json::from_str(text).map_err(|error| { + Rejection::of_frame(ProtocolError::parse_error(format!( + "frame is not JSON: {error}" + ))) + })?; + parse_value(value) +} + +/// Classify one decoded frame. +pub fn parse_value(value: Value) -> Result { + let mut object = match value { + Value::Object(object) => object, + Value::Array(_) => { + return Err(Rejection::of_frame(ProtocolError::invalid_request( + "batches are not supported", + ))); + } + _ => { + return Err(Rejection::of_frame(ProtocolError::invalid_request( + "a message must be a JSON object", + ))); + } + }; + let notification = + !object.contains_key("id") && matches!(object.get("method"), Some(Value::String(_))); + let id = match object.remove("id") { + None => None, + Some(value) => match serde_json::from_value::(value.clone()) { + Ok(id) => Some(id), + Err(_) if value.is_null() && object.contains_key("error") => None, + Err(_) => { + return Err(Rejection::of_frame(ProtocolError::invalid_request( + "id must be a string or an integer", + ))); + } + }, + }; + let reject = |error: ProtocolError| Rejection { + id: id.clone(), + error, + notification, + }; + match object.remove("jsonrpc") { + Some(Value::String(version)) if version == JSONRPC_VERSION => {} + _ => { + return Err(reject(ProtocolError::invalid_request( + "jsonrpc must be \"2.0\"", + ))); + } + } + + if let Some(method) = object.remove("method") { + let Value::String(method) = method else { + return Err(reject(ProtocolError::invalid_request( + "method must be a string", + ))); + }; + let params = match object.remove("params") { + None => Value::Object(Map::new()), + Some(params @ Value::Object(_)) => params, + Some(_) => { + return Err(reject(ProtocolError::invalid_params( + "params must be an object", + ))); + } + }; + if let Some(member) = object.keys().next() { + return Err(reject(ProtocolError::invalid_request(format!( + "unknown member {member:?}" + )))); + } + return Ok(match id { + Some(id) => Message::Request(Request { id, method, params }), + None => Message::Notification(Notification { method, params }), + }); + } + + let result = match (object.remove("result"), object.remove("error")) { + (Some(result), None) => Ok(result), + (None, Some(error)) => Err(ProtocolError::from_error_value(&error)), + _ => { + return Err(reject(ProtocolError::invalid_request( + "a response carries exactly one of result and error", + ))); + } + }; + if let Some(member) = object.keys().next() { + return Err(reject(ProtocolError::invalid_request(format!( + "unknown member {member:?}" + )))); + } + if id.is_none() && result.is_ok() { + return Err(reject(ProtocolError::invalid_request( + "a result needs an id", + ))); + } + Ok(Message::Response(Response { id, result })) +} + +impl Request { + pub fn new(id: Id, method: &str, params: &P) -> Result { + Ok(Self { + id, + method: method.to_owned(), + params: params_value(params)?, + }) + } + + /// Decode the params strictly; a failure is `invalidParams`. + pub fn params(&self) -> Result { + decode_params(&self.params) + } + + pub fn to_value(&self) -> Value { + serde_json::json!({ + "jsonrpc": JSONRPC_VERSION, + "id": self.id, + "method": self.method, + "params": self.params, + }) + } + + pub fn to_text(&self) -> String { + self.to_value().to_string() + } +} + +impl Notification { + pub fn new(method: &str, params: &P) -> Result { + Ok(Self { + method: method.to_owned(), + params: params_value(params)?, + }) + } + + /// Decode the params strictly; a failure is `invalidParams`. + pub fn params(&self) -> Result { + decode_params(&self.params) + } + + pub fn to_value(&self) -> Value { + serde_json::json!({ + "jsonrpc": JSONRPC_VERSION, + "method": self.method, + "params": self.params, + }) + } + + pub fn to_text(&self) -> String { + self.to_value().to_string() + } +} + +impl Response { + pub fn success(id: Id, result: &T) -> Self { + let result = serde_json::to_value(result) + .map_err(|error| ProtocolError::internal(format!("encode result: {error}"))); + Self { + id: Some(id), + result, + } + } + + pub fn failure(id: Option, error: ProtocolError) -> Self { + Self { + id, + result: Err(error), + } + } + + pub fn from_result(id: Id, result: Result) -> Self { + match result { + Ok(value) => Self::success(id, &value), + Err(error) => Self::failure(Some(id), error), + } + } + + /// Decode a successful result strictly. The two failures stay apart: the + /// peer answered with an error ([`ResponseError::Failed`]), or its result + /// breaks the contract ([`ResponseError::Undecodable`]). + pub fn decode(self) -> Result { + let value = self.result.map_err(ResponseError::Failed)?; + serde_path_to_error::deserialize(&value) + .map_err(|error| ResponseError::Undecodable(describe_path_error(&error))) + } + + pub fn to_value(&self) -> Value { + let id = self.id.as_ref().map_or(Value::Null, |id| { + serde_json::to_value(id).unwrap_or(Value::Null) + }); + match &self.result { + Ok(result) => { + serde_json::json!({"jsonrpc": JSONRPC_VERSION, "id": id, "result": result}) + } + Err(error) => serde_json::json!({ + "jsonrpc": JSONRPC_VERSION, + "id": id, + "error": ErrorObject::from(error.clone()), + }), + } + } + + /// Encode for sending. A response larger than + /// [`MAX_MESSAGE_BYTES`] is replaced by a `payloadTooLarge` error for the + /// same id, so a server never sends a frame its peer must drop. + pub fn to_text(&self) -> String { + let text = self.to_value().to_string(); + if (text.len() as u64) <= MAX_MESSAGE_BYTES { + return text; + } + Self::failure( + self.id.clone(), + ProtocolError::payload_too_large( + MAX_MESSAGE_BYTES, + format!("response of {} bytes exceeds the message limit", text.len()), + ), + ) + .to_value() + .to_string() + } +} + +/// Why a response yielded no typed result. +#[derive(Clone, Debug, PartialEq, Error)] +pub enum ResponseError { + /// The peer answered with this error. + #[error(transparent)] + Failed(ProtocolError), + + /// The result does not decode as the method's response type: the peer + /// broke the contract. + #[error("result does not decode: {0}")] + Undecodable(String), +} + +/// Check an encoded outgoing message against the message limit. +pub fn check_message_size(text: &str) -> Result<(), ProtocolError> { + let length = text.len() as u64; + if length > MAX_MESSAGE_BYTES { + return Err(ProtocolError::payload_too_large( + MAX_MESSAGE_BYTES, + format!("message of {length} bytes exceeds the {MAX_MESSAGE_BYTES}-byte limit"), + )); + } + Ok(()) +} + +fn params_value(params: &P) -> Result { + match serde_json::to_value(params) { + Ok(value @ Value::Object(_)) => Ok(value), + Ok(_) => Err(ProtocolError::internal( + "params must serialize as an object", + )), + Err(error) => Err(ProtocolError::internal(format!("encode params: {error}"))), + } +} + +/// Strict decode; a failure is `invalidParams` naming the offending field by +/// its wire path when the decoder knows it. +pub fn decode_params(params: &Value) -> Result { + serde_path_to_error::deserialize(params).map_err(|error| { + let field = path_of(&error); + let message = describe_path_error(&error); + match field { + Some(field) => ProtocolError::invalid_field(field, message), + None => ProtocolError::invalid_params(message), + } + }) +} + +fn path_of(error: &serde_path_to_error::Error) -> Option { + let path = error.path(); + path.iter().next().is_some().then(|| path.to_string()) +} + +fn describe_path_error(error: &serde_path_to_error::Error) -> String { + match path_of(error) { + Some(path) => format!("{path}: {}", error.inner()), + None => error.inner().to_string(), + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::error::{ErrorData, ProtocolErrorKind}; + + fn rejected(text: &str) -> Rejection { + parse(text).expect_err("rejected") + } + + #[test] + fn missing_params_mean_an_empty_object() { + let Message::Request(request) = + parse(r#"{"jsonrpc":"2.0","id":1,"method":"env/getIdle"}"#).expect("request") + else { + panic!("expected a request"); + }; + assert_eq!(request.params, json!({})); + assert_eq!(request.id, Id::Number(1)); + } + + #[test] + fn framing_errors_use_reserved_kinds_and_null_ids() { + let parse_error = rejected("not json"); + assert_eq!(parse_error.error.kind(), ProtocolErrorKind::ParseError); + assert_eq!(parse_error.id, None); + let batch = rejected(r#"[{"jsonrpc":"2.0","id":1,"method":"x"}]"#); + assert_eq!(batch.error.kind(), ProtocolErrorKind::InvalidRequest); + assert_eq!(batch.id, None); + let version = rejected(r#"{"jsonrpc":"1.0","id":7,"method":"x"}"#); + assert_eq!(version.error.kind(), ProtocolErrorKind::InvalidRequest); + assert_eq!(version.id, Some(Id::Number(7))); + let params = rejected(r#"{"jsonrpc":"2.0","id":"a","method":"x","params":[1]}"#); + assert_eq!(params.error.kind(), ProtocolErrorKind::InvalidParams); + let member = rejected(r#"{"jsonrpc":"2.0","id":1,"method":"x","extra":true}"#); + assert_eq!(member.error.kind(), ProtocolErrorKind::InvalidRequest); + } + + #[test] + fn notifications_carry_no_id() { + let message = parse(r#"{"jsonrpc":"2.0","method":"$/cancelRequest","params":{"id":3}}"#) + .expect("notification"); + let Message::Notification(notification) = message else { + panic!("expected a notification"); + }; + let params: CancelRequestParams = notification.params().expect("params"); + assert_eq!(params.id, Id::Number(3)); + } + + #[test] + fn error_responses_round_trip_with_null_ids() { + let response = Response::failure(None, ProtocolError::parse_error("bad")); + let text = response.to_text(); + let Message::Response(parsed) = parse(&text).expect("response") else { + panic!("expected a response"); + }; + assert_eq!(parsed, response); + assert_eq!( + serde_json::from_str::(&text).unwrap()["id"], + Value::Null + ); + } + + #[test] + fn oversized_responses_become_payload_too_large() { + let big = "x".repeat(MAX_MESSAGE_BYTES as usize + 1); + let text = Response::success(Id::Number(4), &json!({"data": big})).to_text(); + let Message::Response(parsed) = parse(&text).expect("response") else { + panic!("expected a response"); + }; + assert_eq!(parsed.id, Some(Id::Number(4))); + assert_eq!( + parsed.result.unwrap_err().kind(), + ProtocolErrorKind::PayloadTooLarge + ); + } + + #[test] + fn params_decode_failures_are_invalid_params_naming_the_field() { + let request = Request::new(Id::Number(1), "x", &json!({"id": true})).unwrap(); + let error = request.params::().unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::InvalidParams); + assert!(matches!( + error.data, + ErrorData::InvalidParams { field: Some(ref field), .. } if field == "id" + )); + } + + #[test] + fn malformed_notifications_are_never_answered() { + for text in [ + r#"{"jsonrpc":"2.0","method":"$/cancelRequest","params":[3]}"#, + r#"{"jsonrpc":"2.0","method":"$/cancelRequest","params":{"id":3},"extra":1}"#, + r#"{"jsonrpc":"1.0","method":"$/cancelRequest","params":{"id":3}}"#, + r#"{"method":"$/cancelRequest"}"#, + ] { + let rejection = rejected(text); + assert!(rejection.is_notification(), "{text}"); + assert_eq!(rejection.into_response(), None, "{text}"); + } + } + + #[test] + fn malformed_requests_and_frames_are_answered() { + for (text, id) in [ + ("not json", None), + ("[]", None), + ("7", None), + ( + r#"{"jsonrpc":"2.0","id":5,"method":"x","params":[1]}"#, + Some(Id::Number(5)), + ), + (r#"{"jsonrpc":"2.0","id":{},"method":"x"}"#, None), + (r#"{"jsonrpc":"2.0","method":7}"#, None), + ( + r#"{"jsonrpc":"2.0","id":"r","result":1,"error":{}}"#, + Some(Id::String("r".to_owned())), + ), + ] { + let rejection = rejected(text); + assert!(!rejection.is_notification(), "{text}"); + let response = rejection.into_response().expect("answered"); + assert_eq!(response.id, id, "{text}"); + assert!(response.result.is_err()); + } + } + + #[test] + fn responses_parse_strictly() { + let Message::Response(ok) = + parse(r#"{"jsonrpc":"2.0","id":1,"result":{"idleForMs":0}}"#).expect("response") + else { + panic!("expected a response"); + }; + assert_eq!(ok.id, Some(Id::Number(1))); + for bad in [ + r#"{"jsonrpc":"2.0","id":null,"result":{}}"#, + r#"{"jsonrpc":"2.0","result":{}}"#, + r#"{"id":1,"result":{}}"#, + r#"{"jsonrpc":"2.0","id":1}"#, + r#"{"jsonrpc":"2.0","id":1,"result":{},"extra":true}"#, + ] { + assert!(parse(bad).is_err(), "{bad}"); + } + let Message::Response(failed) = parse( + r#"{"jsonrpc":"2.0","id":null,"error":{"code":-32700,"message":"m","data":{"kind":"parseError"}}}"#, + ) + .expect("error response") else { + panic!("expected a response"); + }; + assert_eq!(failed.id, None); + assert_eq!( + failed.result.unwrap_err().kind(), + ProtocolErrorKind::ParseError + ); + } + + #[test] + fn decode_tells_a_failed_call_from_a_broken_result() { + let failed = Response::failure(Some(Id::Number(1)), ProtocolError::internal("x")); + assert!(matches!( + failed.decode::(), + Err(ResponseError::Failed(_)) + )); + let broken = Response::success(Id::Number(1), &json!({"id": [1]})); + assert!(matches!( + broken.decode::(), + Err(ResponseError::Undecodable(_)) + )); + } +} diff --git a/crates/environment-protocol/src/lib.rs b/crates/environment-protocol/src/lib.rs index 6882d6c69..35e589400 100644 --- a/crates/environment-protocol/src/lib.rs +++ b/crates/environment-protocol/src/lib.rs @@ -1,12 +1,103 @@ -//! Protocol types for environment execution targets. +//! Wire types of the environment protocol. //! -//! This crate is intentionally transport-free. It defines the stable -//! request/response records used by clients, provider controllers, and environment -//! implementations. +//! Three planes share one JSON-RPC 2.0 envelope ([`jsonrpc`]) and one error +//! vocabulary ([`error`]): +//! +//! - the **data plane** ([`data`]): the runtime drives envd's filesystem, +//! processes and jobs; versioned by [`ENVD_PROTOCOL_VERSION`]; +//! - the **controller plane** ([`control`]): the runtime drives a provider +//! controller's targets; versioned by [`CONTROLLER_PROTOCOL_VERSION`]; +//! - the **registration channel** ([`registration`]): a daemon registers +//! with the gateway and is asked for reverse-dialed streams; versioned by +//! [`ENVD_PROTOCOL_VERSION`]. +//! +//! [`manifest`] lists every method with its types; [`idle`] holds the idle +//! policy providers run against envd's `env/getIdle`, and +//! [`routes::DataRouteRefusal`] the body a data route answers instead of +//! upgrading. The crate is transport-free: it defines messages and the +//! rules both sides follow, not sockets. +//! +//! # Conventions +//! +//! These rules hold for every plane; the frozen bootstrap surface +//! ([`registration::ChallengeParams`], [`bootstrap`]) is the one exception +//! to rule 13. +//! +//! 1. **Framing.** JSON-RPC 2.0 over WebSocket text frames; `initialize` +//! first on the data and controller planes; requests may be pipelined +//! and are matched by id. +//! 2. **Errors.** Integer codes fixed by kind, `data.kind` authoritative, +//! typed fields instead of free-form details; [`error::ProtocolError`] +//! displays as `{kind}: {message}`. +//! 3. **Methods.** `domain/verb[Object]` in lowerCamel with singular +//! domains, batch-ness in params, no plane prefix; every method is a +//! manifest entry; types are `Params` and +//! `Response`. One exception: methods of a family that +//! all act on one thing and answer its state share that state as their +//! response type, and share one params type when they name only that +//! thing (`TransferStatus` and `TransferOperationParams` for +//! `transfer/*`, `TargetResponse` for `target/*` and `ingress/*`). +//! 4. **Fields.** camelCase; plural nouns for lists; `Count` +//! for counts, `Bytes` for sizes, `…Ms` for durations, +//! `…AtMs` for instants, `…Id`/`…Ids` for ids; payload bytes are always +//! `data`; one name per relation (`dependsOn`, `truncated`). Positions +//! name their unit: `nextOffset` (bytes of one file), `nextIndex` +//! (entries), `nextCursor` (opaque); an output page's `next` and `end` +//! are byte offsets into the handle's output stream. +//! 5. **Optionality.** No `null` except JSON-RPC's own `id`; absent options +//! and empty collections are skipped; flags are `bool`, default false, +//! skipped when false; no `Option`, no empty-string sentinels. +//! 6. **Numbers.** `u64` for bytes, offsets and durations; `u32` for counts, +//! indices and depths; `i64` for instants; no `usize` on the wire; every +//! value below 2^53. +//! 7. **Unions.** Data unions are tagged `kind`; variant fields are renamed +//! only through enum-level `rename_all_fields`; operations are methods, +//! never tagged actions. +//! 8. **Ids.** Validated newtypes ([`ids`]): the general grammar for +//! Lightspeed-assigned ids, the opaque rule for provider-assigned ids. +//! 9. **Bytes.** Padded standard base64 for payloads; canonical lowercase +//! hex for fixed-size crypto values; [`paths::ContentDigest`] as +//! `sha256:`; opaque fingerprints carry their own prefix. +//! 10. **Paths.** [`paths::EnvironmentPath`] (absolute) for endpoint paths, +//! [`paths::TreePath`] for relative tree entries. +//! 11. **Limits.** Caller quotas are required per request; endpoint ceilings +//! are advertised in `initialize`; the message size is a protocol +//! constant ([`limits`]); out-of-range values are rejected, never +//! clamped. Ceilings are chosen so every response stays below the +//! message limit: payload ceilings fit after base64, lists have length +//! ceilings, walks stop at [`limits::MAX_RESULT_BYTES`], descriptive +//! strings are bounded. [`jsonrpc::Response::to_text`]'s +//! `payloadTooLarge` replacement is a defect backstop, not paging. +//! 12. **Capabilities.** In each plane's handshake only, as instance facts +//! grouped by family; an absent family or fact is +//! `capabilityUnavailable`; an endpoint never silently ignores a +//! parameter. +//! 13. **Decoding.** Strict on every plane: unknown members are rejected, +//! and defaults exist only for documented optional knobs. +//! 14. **Docs and fixtures.** Doc comments use wire names; every params and +//! response type and every enum is documented; `fixtures/` holds one +//! params, response and error fixture per method. +//! 15. **Boundaries.** Wire types are not persistence, workflow or +//! model-facing types; each layer maps at its own boundary. +pub mod bootstrap; +pub mod bytes; +#[cfg(feature = "schema")] +pub mod contract; pub mod control; pub mod data; pub mod error; -pub mod gateway; +pub mod handshake; +pub mod idle; +pub mod ids; +pub mod jsonrpc; +pub mod limits; +pub mod manifest; +pub mod metadata; +pub mod paths; pub mod registration; -pub mod shared; +pub mod routes; +mod serde_util; +mod version; + +pub use version::{CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION}; diff --git a/crates/environment-protocol/src/limits.rs b/crates/environment-protocol/src/limits.rs new file mode 100644 index 000000000..8fcbffdf4 --- /dev/null +++ b/crates/environment-protocol/src/limits.rs @@ -0,0 +1,90 @@ +//! Protocol constants: the limits every hop enforces without reading a +//! handshake. +//! +//! Operational ceilings that a deployment configures (request concurrency, +//! retention, wait, scan and transfer maxima) are not constants: the data +//! plane advertises them in `initialize` as +//! [`DataLimits`](crate::data::handshake::DataLimits). Caller quotas are +//! required fields on each request, and a value above an advertised ceiling +//! is refused with `invalidParams` naming the field, never clamped. + +/// Largest WebSocket message, and therefore frame, that any hop sends or +/// accepts on any plane. Every reader and writer configures it explicitly, +/// with the frame limit equal to the message limit. A sender that would +/// exceed it fails with `payloadTooLarge` instead of sending. +pub const MAX_MESSAGE_BYTES: u64 = 16 * 1024 * 1024; + +/// A peer waiting for a response sends a WebSocket ping at least this often +/// while any request it sent is unanswered. +pub const KEEPALIVE_INTERVAL_MS: u64 = 20_000; + +/// Intermediaries (gateways, relays, proxies) forward pings and pongs, count +/// them as activity, and never close a connection for idleness sooner than +/// this. +pub const MIN_INTERMEDIARY_IDLE_MS: u64 = 60_000; + +/// Every data and controller server accepts at least this many requests in +/// flight on one connection before it answers `busy`. +pub const MIN_CONCURRENT_REQUESTS: u32 = 16; + +/// A registration-channel peer that receives no frame for this many +/// `heartbeatIntervalMs` treats the connection as dead. +pub const HEARTBEAT_MISSED_INTERVALS: u32 = 3; + +/// Longest [`EnvironmentPath`](crate::paths::EnvironmentPath) or +/// [`TreePath`](crate::paths::TreePath) in bytes. +pub const MAX_PATH_BYTES: u32 = 4096; + +/// Longest [`DescriptiveText`](crate::metadata::DescriptiveText) in bytes: +/// a target's `message`, a template's `description`, a +/// `registration/closed` message. +pub const MAX_DESCRIPTIVE_TEXT_BYTES: u32 = 1024; + +/// Longest URL a peer reports or is configured with (a data route base, +/// an ingress `endpoint`), in bytes. +pub const MAX_URL_BYTES: u32 = 2048; + +/// Longest error `message` in bytes. [`ProtocolError::new`] cuts longer +/// messages on a character boundary; the message is never parsed. +/// +/// [`ProtocolError::new`]: crate::error::ProtocolError::new +pub const MAX_ERROR_MESSAGE_BYTES: u32 = 4096; + +/// Longest a provider data route holds a dial before it upgrades or +/// answers a [`DataRouteRefusal`](crate::routes::DataRouteRefusal), in +/// milliseconds: long enough to resume a paused target, short enough that +/// the caller's own wait stays in charge. +pub const MAX_DATA_ROUTE_HOLD_MS: u64 = 5_000; + +/// Largest `Retry-After` a data route refusal names, in whole seconds. +/// Receivers clamp a larger value to it. +pub const RETRY_AFTER_MAX_SECS: u64 = 60; + +/// Most chunks plus gaps in one `process/read` page, and across all entries +/// of one `job/read` response. A page that reaches it ends early and loses +/// nothing: `next` says where to continue. +pub const MAX_PAGE_CHUNKS: u32 = 4096; + +/// Budget, in encoded bytes, for the variable part of a walk's result: the +/// matches of `fs/searchText` and `fs/globFiles`, and the entries, skips +/// and base64 content of all roots of one `fs/scan`. A walk stops (search +/// and glob: `stopped: responseLimit`) or a scan root fails +/// (`payloadTooLarge`) before its result would exceed it, so every response +/// stays below [`MAX_MESSAGE_BYTES`] with room for its fixed members. +pub const MAX_RESULT_BYTES: u64 = 12 * 1024 * 1024; + +/// Most skipped entries a scan root or transfer lists by path; the total is +/// always counted in `skippedEntryCount`. +pub const MAX_LISTED_SKIPS: u32 = 100; + +/// Most leftover processes a report lists; the rest are counted in +/// `omittedProcessCount`. +pub const MAX_LEFTOVER_PROCESSES: u32 = 64; + +/// Longest leftover `command` in bytes; longer command lines are cut on a +/// character boundary. +pub const MAX_LEFTOVER_COMMAND_BYTES: u32 = 1024; + +/// Longest field of an +/// [`ImplementationInfo`](crate::handshake::ImplementationInfo) in bytes. +pub const MAX_IMPLEMENTATION_FIELD_BYTES: u32 = 256; diff --git a/crates/environment-protocol/src/manifest.rs b/crates/environment-protocol/src/manifest.rs new file mode 100644 index 000000000..be27d664b --- /dev/null +++ b/crates/environment-protocol/src/manifest.rs @@ -0,0 +1,645 @@ +//! The method manifest: one table of every method on every plane, with its +//! params and response types and the facts servers and clients act on. +//! +//! envd dispatch and idle classification, the client's typed calls, the +//! contract export and the fixture tests all read this table, so a method +//! exists exactly when it has an entry here. Each entry also has a marker +//! type implementing [`RequestMethod`] or [`NotificationMethod`], which +//! binds the method name to its Rust types: +//! +//! ``` +//! use environment_protocol::manifest::{ProcessRead, RequestMethod}; +//! assert_eq!(ProcessRead::NAME, "process/read"); +//! ``` + +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; + +use crate::{ + control::{ + handshake::{ControllerInitializeParams, ControllerInitializeResponse}, + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + CloseTargetParams, CreateTargetParams, GetTargetParams, SetTargetPowerParams, + TargetResponse, + }, + template::{ListTemplatesParams, ListTemplatesResponse}, + }, + data::{ + fs::{ + CreateDirectoryParams, CreateDirectoryResponse, GetMetadataParams, GetMetadataResponse, + GlobFilesParams, GlobFilesResponse, ReadDirectoryParams, ReadDirectoryResponse, + ReadFileParams, ReadFileResponse, RemoveParams, RemoveResponse, RenameParams, + RenameResponse, SearchTextParams, SearchTextResponse, WriteFileParams, + WriteFileResponse, + }, + handshake::{InitializeParams, InitializeResponse}, + idle::{GetIdleParams, GetIdleResponse}, + job::{ + CancelJobsParams, CancelJobsResponse, ReadJobsParams, ReadJobsResponse, + StartJobsParams, StartJobsResponse, + }, + process::{ + ReadProcessParams, ReadProcessResponse, StartProcessParams, StartProcessResponse, + TerminateProcessParams, TerminateProcessResponse, WriteProcessParams, + WriteProcessResponse, + }, + scan::{ScanParams, ScanResponse}, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, ListTransferMissingParams, + ListTransferMissingResponse, ReadTransferContentParams, ReadTransferContentResponse, + ReadTransferInventoryParams, ReadTransferInventoryResponse, TransferOperationParams, + TransferStatus, WriteTransferContentParams, WriteTransferContentResponse, + }, + }, + error::Capability, + handshake::Plane, + jsonrpc::CancelRequestParams, + registration::{ + ChallengeParams, ClosedParams, GoodbyeParams, OpenStreamParams, OpenStreamResponse, + RegisterParams, RegisterResponse, + }, +}; + +/// A request method: its name, plane and the types of its params and +/// result. +pub trait RequestMethod { + const NAME: &'static str; + const PLANE: Plane; + type Params: Serialize + DeserializeOwned + Send + Sync + 'static; + type Response: Serialize + DeserializeOwned + Send + Sync + 'static; +} + +/// A notification method: never answered. +pub trait NotificationMethod { + const NAME: &'static str; + const PLANE: Plane; + type Params: Serialize + DeserializeOwned + Send + Sync + 'static; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum MessageKind { + Request, + Notification, +} + +impl MessageKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Request => "request", + Self::Notification => "notification", + } + } +} + +/// Who sends a method. The client is the peer that dialed: the runtime on +/// the data and controller planes, the daemon on the registration channel. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Direction { + ClientToServer, + ServerToClient, +} + +impl Direction { + pub fn as_str(self) -> &'static str { + match self { + Self::ClientToServer => "clientToServer", + Self::ServerToClient => "serverToClient", + } + } +} + +/// Method family: the method name's domain. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Family { + Handshake, + Envelope, + Env, + Filesystem, + Transfer, + Process, + Job, + Template, + Target, + Ingress, + Registration, +} + +impl Family { + pub fn as_str(self) -> &'static str { + match self { + Self::Handshake => "handshake", + Self::Envelope => "envelope", + Self::Env => "env", + Self::Filesystem => "filesystem", + Self::Transfer => "transfer", + Self::Process => "process", + Self::Job => "job", + Self::Template => "template", + Self::Target => "target", + Self::Ingress => "ingress", + Self::Registration => "registration", + } + } +} + +/// Whether a data-plane method counts toward the idle clock. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Activity { + /// Real work: resets `idleForMs`. + Use, + /// Looking, not using: handshakes, idle reports, discovery scans. + Observation, +} + +impl Activity { + pub fn as_str(self) -> &'static str { + match self { + Self::Use => "use", + Self::Observation => "observation", + } + } +} + +/// Schemas of one method, registered in a shared generator. +#[cfg(feature = "schema")] +pub struct MethodSchemas { + pub params: schemars::Schema, + pub response: Option, +} + +/// Re-encode a value through a wire type: decoding enforces the contract +/// strictly, encoding yields the canonical form. +pub type Normalize = fn(&Value) -> Result; + +/// One manifest entry. +pub struct MethodSpec { + pub name: &'static str, + pub plane: Plane, + pub kind: MessageKind, + pub direction: Direction, + pub family: Family, + /// Data-plane methods only. + pub activity: Option, + /// The capability a request needs; a server without it answers + /// `capabilityUnavailable` before doing anything. + pub capability: Option, + /// `$/cancelRequest` aborts it: a read-only wait or walk. Every other + /// method runs to a consistent state. + pub abortable: bool, + pub params_type: &'static str, + /// Absent for notifications. + pub response_type: Option<&'static str>, + pub summary: &'static str, + pub normalize_params: Normalize, + pub normalize_response: Option, + #[cfg(feature = "schema")] + pub schemas: fn(&mut schemars::SchemaGenerator) -> MethodSchemas, +} + +impl std::fmt::Debug for MethodSpec { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("MethodSpec") + .field("name", &self.name) + .field("plane", &self.plane) + .field("kind", &self.kind) + .finish_non_exhaustive() + } +} + +fn normalize(value: &Value) -> Result { + serde_json::to_value(T::deserialize(value)?) +} + +macro_rules! requests { + ($( + $marker:ident = $name:literal { + plane: $plane:ident, + direction: $direction:ident, + family: $family:ident, + activity: $activity:expr, + capability: $capability:expr, + abortable: $abortable:expr, + params: $params:ty, + response: $response:ty, + summary: $summary:literal $(,)? + } + )+) => { + $( + #[doc = concat!("`", $name, "`: ", $summary)] + #[derive(Clone, Copy, Debug)] + pub struct $marker; + + impl $marker { + pub const NAME: &'static str = $name; + } + + impl RequestMethod for $marker { + const NAME: &'static str = $name; + const PLANE: Plane = Plane::$plane; + type Params = $params; + type Response = $response; + } + )+ + + /// Every request method. + pub static REQUESTS: &[MethodSpec] = &[ + $( + MethodSpec { + name: $name, + plane: Plane::$plane, + kind: MessageKind::Request, + direction: Direction::$direction, + family: Family::$family, + activity: $activity, + capability: $capability, + abortable: $abortable, + params_type: stringify!($params), + response_type: Some(stringify!($response)), + summary: $summary, + normalize_params: normalize::<$params>, + normalize_response: Some(normalize::<$response>), + #[cfg(feature = "schema")] + schemas: |generator| MethodSchemas { + params: generator.subschema_for::<$params>(), + response: Some(generator.subschema_for::<$response>()), + }, + }, + )+ + ]; + }; +} + +macro_rules! notifications { + ($( + $marker:ident = $name:literal { + plane: $plane:ident, + direction: $direction:ident, + family: $family:ident, + activity: $activity:expr, + params: $params:ty, + summary: $summary:literal $(,)? + } + )+) => { + $( + #[doc = concat!("`", $name, "`: ", $summary)] + #[derive(Clone, Copy, Debug)] + pub struct $marker; + + impl $marker { + pub const NAME: &'static str = $name; + } + + impl NotificationMethod for $marker { + const NAME: &'static str = $name; + const PLANE: Plane = Plane::$plane; + type Params = $params; + } + )+ + + /// Every notification method. + pub static NOTIFICATIONS: &[MethodSpec] = &[ + $( + MethodSpec { + name: $name, + plane: Plane::$plane, + kind: MessageKind::Notification, + direction: Direction::$direction, + family: Family::$family, + activity: $activity, + capability: None, + abortable: false, + params_type: stringify!($params), + response_type: None, + summary: $summary, + normalize_params: normalize::<$params>, + normalize_response: None, + #[cfg(feature = "schema")] + schemas: |generator| MethodSchemas { + params: generator.subschema_for::<$params>(), + response: None, + }, + }, + )+ + ]; + }; +} + +const USE: Option = Some(Activity::Use); +const OBSERVATION: Option = Some(Activity::Observation); +const FILESYSTEM: Option = Some(Capability::Filesystem); +const WRITE: Option = Some(Capability::FilesystemWrite); +const PROCESS: Option = Some(Capability::Process); +const JOB: Option = Some(Capability::Job); + +requests! { + DataInitialize = "initialize" { + plane: Data, direction: ClientToServer, family: Handshake, activity: OBSERVATION, + capability: None, abortable: false, + params: InitializeParams, response: InitializeResponse, + summary: "Check the protocol version and read the daemon's instance id, capabilities and ceilings.", + } + EnvGetIdle = "env/getIdle" { + plane: Data, direction: ClientToServer, family: Env, activity: OBSERVATION, + capability: None, abortable: false, + params: GetIdleParams, response: GetIdleResponse, + summary: "Report how long the environment has been idle.", + } + FsReadFile = "fs/readFile" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ReadFileParams, response: ReadFileResponse, + summary: "Read a byte range of a regular file.", + } + FsWriteFile = "fs/writeFile" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: WRITE, abortable: false, + params: WriteFileParams, response: WriteFileResponse, + summary: "Write a file atomically, optionally conditioned on its version.", + } + FsCreateDirectory = "fs/createDirectory" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: WRITE, abortable: false, + params: CreateDirectoryParams, response: CreateDirectoryResponse, + summary: "Create a directory.", + } + FsGetMetadata = "fs/getMetadata" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: false, + params: GetMetadataParams, response: GetMetadataResponse, + summary: "Describe one entry.", + } + FsReadDirectory = "fs/readDirectory" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ReadDirectoryParams, response: ReadDirectoryResponse, + summary: "List one page of a directory's entries.", + } + FsRemove = "fs/remove" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: WRITE, abortable: false, + params: RemoveParams, response: RemoveResponse, + summary: "Remove a file, symlink or directory.", + } + FsRename = "fs/rename" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: WRITE, abortable: false, + params: RenameParams, response: RenameResponse, + summary: "Rename an entry atomically.", + } + FsSearchText = "fs/searchText" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: true, + params: SearchTextParams, response: SearchTextResponse, + summary: "Search file contents below a root with a regular expression.", + } + FsGlobFiles = "fs/globFiles" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: USE, + capability: FILESYSTEM, abortable: true, + params: GlobFilesParams, response: GlobFilesResponse, + summary: "List files below a root that match patterns.", + } + FsScan = "fs/scan" { + plane: Data, direction: ClientToServer, family: Filesystem, activity: OBSERVATION, + capability: FILESYSTEM, abortable: true, + params: ScanParams, response: ScanResponse, + summary: "Observe small trees below several roots, each reported on its own.", + } + TransferBegin = "transfer/begin" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: BeginTransferParams, response: TransferStatus, + summary: "Start or resume a transfer operation.", + } + TransferAdvance = "transfer/advance" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: TransferOperationParams, response: TransferStatus, + summary: "Do a bounded step of scanning or staging work.", + } + TransferGetStatus = "transfer/getStatus" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: TransferOperationParams, response: TransferStatus, + summary: "Read an operation's status or receipt.", + } + TransferReadInventory = "transfer/readInventory" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ReadTransferInventoryParams, response: ReadTransferInventoryResponse, + summary: "Read one page of a sealed inventory.", + } + TransferAppend = "transfer/append" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: AppendTransferInventoryParams, response: TransferStatus, + summary: "Append incoming inventory entries.", + } + TransferListMissing = "transfer/listMissing" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ListTransferMissingParams, response: ListTransferMissingResponse, + summary: "List digests whose content the daemon still needs.", + } + TransferRead = "transfer/read" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: ReadTransferContentParams, response: ReadTransferContentResponse, + summary: "Read captured content by digest.", + } + TransferWrite = "transfer/write" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: WriteTransferContentParams, response: WriteTransferContentResponse, + summary: "Upload content for a missing digest.", + } + TransferCommit = "transfer/commit" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: TransferOperationParams, response: TransferStatus, + summary: "Finish a ready operation: publish a staged tree, or release a capture.", + } + TransferAbort = "transfer/abort" { + plane: Data, direction: ClientToServer, family: Transfer, activity: USE, + capability: FILESYSTEM, abortable: false, + params: TransferOperationParams, response: TransferStatus, + summary: "Abandon an operation and discard its staging.", + } + ProcessStart = "process/start" { + plane: Data, direction: ClientToServer, family: Process, activity: USE, + capability: PROCESS, abortable: false, + params: StartProcessParams, response: StartProcessResponse, + summary: "Start a process under a caller-chosen handle.", + } + ProcessRead = "process/read" { + plane: Data, direction: ClientToServer, family: Process, activity: USE, + capability: PROCESS, abortable: true, + params: ReadProcessParams, response: ReadProcessResponse, + summary: "Wait for and read a page of a process's output and its state.", + } + ProcessWrite = "process/write" { + plane: Data, direction: ClientToServer, family: Process, activity: USE, + capability: PROCESS, abortable: false, + params: WriteProcessParams, response: WriteProcessResponse, + summary: "Queue bytes for a process's input pipe or PTY.", + } + ProcessTerminate = "process/terminate" { + plane: Data, direction: ClientToServer, family: Process, activity: USE, + capability: PROCESS, abortable: false, + params: TerminateProcessParams, response: TerminateProcessResponse, + summary: "Interrupt, terminate or kill a process's group.", + } + JobStart = "job/start" { + plane: Data, direction: ClientToServer, family: Job, activity: USE, + capability: JOB, abortable: false, + params: StartJobsParams, response: StartJobsResponse, + summary: "Start jobs, or replay an identical earlier start.", + } + JobRead = "job/read" { + plane: Data, direction: ClientToServer, family: Job, activity: OBSERVATION, + capability: JOB, abortable: true, + params: ReadJobsParams, response: ReadJobsResponse, + summary: "Wait for and read jobs' summaries and output pages.", + } + JobCancel = "job/cancel" { + plane: Data, direction: ClientToServer, family: Job, activity: USE, + capability: JOB, abortable: false, + params: CancelJobsParams, response: CancelJobsResponse, + summary: "Cancel jobs and optionally their dependents.", + } + ControllerInitialize = "initialize" { + plane: Controller, direction: ClientToServer, family: Handshake, activity: None, + capability: None, abortable: false, + params: ControllerInitializeParams, response: ControllerInitializeResponse, + summary: "Check the protocol version and read the controller's capabilities.", + } + TemplateList = "template/list" { + plane: Controller, direction: ClientToServer, family: Template, activity: None, + capability: None, abortable: false, + params: ListTemplatesParams, response: ListTemplatesResponse, + summary: "List the templates a universe may create targets from.", + } + TargetCreate = "target/create" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: None, abortable: false, + params: CreateTargetParams, response: TargetResponse, + summary: "Create the owner's target, or observe the one it has.", + } + TargetGet = "target/get" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: None, abortable: false, + params: GetTargetParams, response: TargetResponse, + summary: "Observe a target without changing it.", + } + TargetSetPower = "target/setPower" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: Some(Capability::Power), abortable: false, + params: SetTargetPowerParams, response: TargetResponse, + summary: "Move a target to a power state.", + } + TargetClose = "target/close" { + plane: Controller, direction: ClientToServer, family: Target, activity: None, + capability: None, abortable: false, + params: CloseTargetParams, response: TargetResponse, + summary: "Destroy the owner's target and everything it holds.", + } + IngressEnsure = "ingress/ensure" { + plane: Controller, direction: ClientToServer, family: Ingress, activity: None, + capability: Some(Capability::Ingress), abortable: false, + params: EnsureIngressParams, response: TargetResponse, + summary: "Route public ingress to a target.", + } + IngressRemove = "ingress/remove" { + plane: Controller, direction: ClientToServer, family: Ingress, activity: None, + capability: Some(Capability::Ingress), abortable: false, + params: RemoveIngressParams, response: TargetResponse, + summary: "Stop routing public ingress to a target that stays.", + } + RegistrationRegister = "registration/register" { + plane: Registration, direction: ClientToServer, family: Registration, activity: None, + capability: None, abortable: false, + params: RegisterParams, response: RegisterResponse, + summary: "Prove the daemon identity and be admitted to its environment.", + } + RegistrationOpenStream = "registration/openStream" { + plane: Registration, direction: ServerToClient, family: Registration, activity: None, + capability: None, abortable: false, + params: OpenStreamParams, response: OpenStreamResponse, + summary: "Ask the daemon to dial one reverse stream.", + } +} + +notifications! { + CancelRequest = "$/cancelRequest" { + plane: Data, direction: ClientToServer, family: Envelope, activity: OBSERVATION, + params: CancelRequestParams, + summary: "Abort a pending abortable request.", + } + RegistrationChallenge = "registration/challenge" { + plane: Registration, direction: ServerToClient, family: Registration, activity: None, + params: ChallengeParams, + summary: "The frozen first frame: the gateway's version and nonce.", + } + RegistrationGoodbye = "registration/goodbye" { + plane: Registration, direction: ClientToServer, family: Registration, activity: None, + params: GoodbyeParams, + summary: "The daemon is stopping deliberately.", + } + RegistrationClosed = "registration/closed" { + plane: Registration, direction: ServerToClient, family: Registration, activity: None, + params: ClosedParams, + summary: "The gateway ends the session for good.", + } +} + +/// Every method of every plane: requests, then notifications. +pub fn methods() -> impl Iterator { + REQUESTS.iter().chain(NOTIFICATIONS.iter()) +} + +/// Every method of one plane. +pub fn plane_methods(plane: Plane) -> impl Iterator { + methods().filter(move |spec| spec.plane == plane) +} + +/// The method of this name on this plane. +pub fn find(plane: Plane, name: &str) -> Option<&'static MethodSpec> { + plane_methods(plane).find(|spec| spec.name == name) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use super::*; + + #[test] + fn method_names_are_unique_per_plane() { + let mut seen = BTreeSet::new(); + for spec in methods() { + assert!( + seen.insert((spec.plane.as_str(), spec.name)), + "duplicate {} on {}", + spec.name, + spec.plane + ); + } + } + + #[test] + fn activity_is_declared_for_data_methods_only() { + for spec in methods() { + assert_eq!( + spec.activity.is_some(), + spec.plane == Plane::Data, + "{}", + spec.name + ); + } + assert_eq!( + find(Plane::Data, FsScan::NAME).unwrap().activity, + Some(Activity::Observation) + ); + } +} diff --git a/crates/environment-protocol/src/metadata.rs b/crates/environment-protocol/src/metadata.rs new file mode 100644 index 000000000..918e60336 --- /dev/null +++ b/crates/environment-protocol/src/metadata.rs @@ -0,0 +1,400 @@ +//! Bounds on descriptive values: display names, correlation metadata and +//! short human-readable texts. +//! +//! These values are descriptive: they never select, authenticate or gate. +//! The same bounds apply wherever Lightspeed accepts them (daemon +//! registration, provider observations and templates, environment and +//! session metadata, profile documents), so a session and the environment it +//! ran in accept the same keys. On the wire they are validated on decode by +//! [`DisplayName`], [`Metadata`] and [`DescriptiveText`]. + +use std::{collections::BTreeMap, fmt}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +use crate::limits::MAX_DESCRIPTIVE_TEXT_BYTES; + +pub const MAX_DISPLAY_NAME_BYTES: u32 = 128; +pub const MAX_METADATA_ENTRIES: u32 = 32; +pub const MAX_METADATA_KEY_BYTES: u32 = 64; +pub const MAX_METADATA_VALUE_BYTES: u32 = 256; +/// Keys under this prefix are written by Lightspeed only. +pub const RESERVED_METADATA_PREFIX: &str = "lightspeed."; + +/// JSON Schema pattern of [`DescriptiveText`]: no control characters except +/// line feed and tab. +pub const DESCRIPTIVE_TEXT_PATTERN: &str = "^[^\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F]+$"; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum MetadataError { + #[error("display name must be 1..={MAX_DISPLAY_NAME_BYTES} bytes without control characters")] + DisplayName, + + #[error("metadata has more than {MAX_METADATA_ENTRIES} entries")] + TooManyEntries, + + #[error("metadata key {key:?} must be 1..={MAX_METADATA_KEY_BYTES} bytes")] + KeyBytes { key: String }, + + #[error("metadata value for {key:?} must be 1..={MAX_METADATA_VALUE_BYTES} bytes")] + ValueBytes { key: String }, + + #[error("metadata entry {key:?} contains a control character")] + ControlCharacter { key: String }, + + #[error("metadata key {key:?} uses the reserved {RESERVED_METADATA_PREFIX} prefix")] + ReservedPrefix { key: String }, + + #[error( + "text must be 1..={MAX_DESCRIPTIVE_TEXT_BYTES} bytes without control characters other than line feed and tab" + )] + DescriptiveText, +} + +pub fn validate_display_name(name: &str) -> Result<(), MetadataError> { + if name.is_empty() + || name.len() > MAX_DISPLAY_NAME_BYTES as usize + || name.chars().any(char::is_control) + { + return Err(MetadataError::DisplayName); + } + Ok(()) +} + +/// A metadata map a caller supplies: at most [`MAX_METADATA_ENTRIES`] +/// bounded entries and no key under [`RESERVED_METADATA_PREFIX`]. +pub fn validate_caller_metadata(metadata: &BTreeMap) -> Result<(), MetadataError> { + if metadata.len() > MAX_METADATA_ENTRIES as usize { + return Err(MetadataError::TooManyEntries); + } + for (key, value) in metadata { + validate_metadata_entry(key, value)?; + if key.starts_with(RESERVED_METADATA_PREFIX) { + return Err(MetadataError::ReservedPrefix { key: key.clone() }); + } + } + Ok(()) +} + +/// A stored metadata map: caller keys obey [`validate_caller_metadata`]'s +/// count, and Lightspeed's own reserved keys are bounded per entry but not +/// counted. +pub fn validate_stored_metadata(metadata: &BTreeMap) -> Result<(), MetadataError> { + let caller_entries = metadata + .keys() + .filter(|key| !key.starts_with(RESERVED_METADATA_PREFIX)) + .count(); + if caller_entries > MAX_METADATA_ENTRIES as usize { + return Err(MetadataError::TooManyEntries); + } + for (key, value) in metadata { + validate_metadata_entry(key, value)?; + } + Ok(()) +} + +/// Key and value bytes and control characters of one entry. +pub fn validate_metadata_entry(key: &str, value: &str) -> Result<(), MetadataError> { + if key.is_empty() || key.len() > MAX_METADATA_KEY_BYTES as usize { + return Err(MetadataError::KeyBytes { + key: key.to_owned(), + }); + } + if value.is_empty() || value.len() > MAX_METADATA_VALUE_BYTES as usize { + return Err(MetadataError::ValueBytes { + key: key.to_owned(), + }); + } + if key.chars().any(char::is_control) || value.chars().any(char::is_control) { + return Err(MetadataError::ControlCharacter { + key: key.to_owned(), + }); + } + Ok(()) +} + +/// Check [`DescriptiveText`]'s rule. +pub fn validate_descriptive_text(text: &str) -> Result<(), MetadataError> { + if text.is_empty() + || text.len() > MAX_DESCRIPTIVE_TEXT_BYTES as usize + || text.chars().any(is_disallowed_text_control) + { + return Err(MetadataError::DescriptiveText); + } + Ok(()) +} + +fn is_disallowed_text_control(character: char) -> bool { + character.is_control() && character != '\n' && character != '\t' +} + +/// A display name: 1 to 128 bytes without control characters. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct DisplayName( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 128), pattern(crate::ids::NO_CONTROL_PATTERN)) + )] + String, +); + +impl DisplayName { + pub fn new(name: impl Into) -> Result { + let name = name.into(); + validate_display_name(&name)?; + Ok(Self(name)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +/// A short human-readable text: 1 to 1024 bytes without control characters +/// other than line feed and tab. Never parsed. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct DescriptiveText( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 1024), pattern(DESCRIPTIVE_TEXT_PATTERN)) + )] + String, +); + +impl DescriptiveText { + pub fn new(text: impl Into) -> Result { + let text = text.into(); + validate_descriptive_text(&text)?; + Ok(Self(text)) + } + + /// Build a valid text from any string, for senders: disallowed control + /// characters become spaces and the text is cut to the bound on a + /// character boundary. `None` when nothing is left. + pub fn lossy(text: &str) -> Option { + let mut cleaned: String = text + .chars() + .map(|character| { + if is_disallowed_text_control(character) { + ' ' + } else { + character + } + }) + .collect(); + let max = MAX_DESCRIPTIVE_TEXT_BYTES as usize; + if cleaned.len() > max { + let mut end = max; + while !cleaned.is_char_boundary(end) { + end -= 1; + } + cleaned.truncate(end); + } + (!cleaned.is_empty()).then_some(Self(cleaned)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +macro_rules! text_impls { + ($name:ident, $error:ty) => { + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } + } + + impl TryFrom for $name { + type Error = $error; + + fn try_from(value: String) -> Result { + Self::new(value) + } + } + + impl TryFrom<&str> for $name { + type Error = $error; + + fn try_from(value: &str) -> Result { + Self::new(value) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } + } + }; +} + +text_impls!(DisplayName, MetadataError); +text_impls!(DescriptiveText, MetadataError); + +/// Caller-supplied metadata on the wire: at most [`MAX_METADATA_ENTRIES`] +/// bounded entries and no key under [`RESERVED_METADATA_PREFIX`] +/// ([`validate_caller_metadata`]), checked on decode. Serialized as a JSON +/// object of strings. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Metadata(BTreeMap); + +impl Metadata { + pub fn new(entries: BTreeMap) -> Result { + validate_caller_metadata(&entries)?; + Ok(Self(entries)) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.0.get(key).map(String::as_str) + } + + pub fn iter(&self) -> impl Iterator { + self.0 + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + } + + pub fn as_map(&self) -> &BTreeMap { + &self.0 + } + + pub fn into_map(self) -> BTreeMap { + self.0 + } +} + +impl TryFrom> for Metadata { + type Error = MetadataError; + + fn try_from(entries: BTreeMap) -> Result { + Self::new(entries) + } +} + +impl Serialize for Metadata { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Metadata { + fn deserialize>(deserializer: D) -> Result { + let entries = BTreeMap::::deserialize(deserializer)?; + Self::new(entries).map_err(D::Error::custom) + } +} + +#[cfg(feature = "schema")] +impl schemars::JsonSchema for Metadata { + fn schema_name() -> std::borrow::Cow<'static, str> { + "Metadata".into() + } + + fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "description": "Descriptive key-value metadata: at most 32 entries, keys 1 to 64 bytes outside the reserved `lightspeed.` prefix, values 1 to 256 bytes, no control characters.", + "type": "object", + "maxProperties": MAX_METADATA_ENTRIES, + "propertyNames": { + "minLength": 1, + "maxLength": MAX_METADATA_KEY_BYTES, + "pattern": crate::ids::NO_CONTROL_PATTERN, + "not": {"pattern": "^lightspeed\\."}, + }, + "additionalProperties": { + "type": "string", + "minLength": 1, + "maxLength": MAX_METADATA_VALUE_BYTES, + "pattern": crate::ids::NO_CONTROL_PATTERN, + }, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn caller_metadata_is_bounded_and_keeps_out_of_the_reserved_prefix() { + let mut metadata = BTreeMap::new(); + metadata.insert("harbor.trialId".to_owned(), "trial-1".to_owned()); + assert_eq!(validate_caller_metadata(&metadata), Ok(())); + metadata.insert("lightspeed.x".to_owned(), "y".to_owned()); + assert_eq!( + validate_caller_metadata(&metadata), + Err(MetadataError::ReservedPrefix { + key: "lightspeed.x".to_owned() + }) + ); + assert_eq!(validate_stored_metadata(&metadata), Ok(())); + metadata.clear(); + metadata.insert( + "k".to_owned(), + "a".repeat(MAX_METADATA_VALUE_BYTES as usize + 1), + ); + assert!(matches!( + validate_caller_metadata(&metadata), + Err(MetadataError::ValueBytes { .. }) + )); + assert_eq!( + validate_display_name("bad\nname"), + Err(MetadataError::DisplayName) + ); + assert_eq!(validate_display_name(""), Err(MetadataError::DisplayName)); + } + + #[test] + fn stored_metadata_does_not_count_reserved_keys() { + let mut metadata: BTreeMap = (0..MAX_METADATA_ENTRIES as usize) + .map(|index| (format!("k{index}"), "v".to_owned())) + .collect(); + metadata.insert("lightspeed.envd.version".to_owned(), "0.5.0".to_owned()); + assert_eq!(validate_stored_metadata(&metadata), Ok(())); + assert!(validate_caller_metadata(&metadata).is_err()); + } + + #[test] + fn wire_values_validate_on_decode() { + assert!(serde_json::from_str::("\"runner 7\"").is_ok()); + assert!(serde_json::from_str::("\"\"").is_err()); + assert!(serde_json::from_str::(r#"{"lightspeed.x": "y"}"#).is_err()); + assert!(serde_json::from_str::(r#"{"k": "v"}"#).is_ok()); + assert!(serde_json::from_str::("\"line one\\nline two\"").is_ok()); + assert!(serde_json::from_str::("\"bell\\u0007\"").is_err()); + let long = "x".repeat(MAX_DESCRIPTIVE_TEXT_BYTES as usize + 1); + assert!(DescriptiveText::new(long.clone()).is_err()); + let cut = DescriptiveText::lossy(&format!("a\u{7}{long}")).unwrap(); + assert_eq!(cut.as_str().len(), MAX_DESCRIPTIVE_TEXT_BYTES as usize); + assert!(cut.as_str().starts_with("a ")); + assert!(DescriptiveText::lossy("").is_none()); + } +} diff --git a/crates/environment-protocol/src/paths.rs b/crates/environment-protocol/src/paths.rs new file mode 100644 index 000000000..27242259d --- /dev/null +++ b/crates/environment-protocol/src/paths.rs @@ -0,0 +1,475 @@ +//! Path and digest values. +//! +//! **Path model.** A wire path is an absolute path in the environment's own +//! filesystem namespace, the one its processes see, not a path relative to +//! the daemon's filesystem root: with the root narrowed to `/workspace`, +//! `/workspace/src` names that directory and `/etc` is outside the root. +//! The root is not a chroot. +//! +//! **Confinement.** One rule for every data-plane method, a command's `cwd` +//! included. A path that is not the filesystem root or below it is refused +//! with `permissionDenied` (reason `outsideRoot`). Otherwise the daemon +//! strips the root prefix and resolves the remaining segments one by one +//! beneath the root's directory descriptor. A symlink met on the way is +//! followed only while its target stays inside the root; a link that leaves +//! it is refused the same way. No method addresses the daemon's state +//! directory: a path that names it, lies inside it or (for removes, renames, +//! writes and captures) contains it is refused with `permissionDenied` +//! (reason `daemonState`), and walks prune it. +//! +//! **Names.** Paths are UTF-8 and POSIX: `/` is the only separator, and +//! every other character except NUL, the backslash included, may appear in +//! a name. An entry whose name is not UTF-8 is skipped and counted wherever +//! a method enumerates entries, never returned lossily. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +use crate::{bytes::decode_hex_array, limits::MAX_PATH_BYTES}; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum PathError { + #[error("path is empty")] + Empty, + + #[error("path is {actual} bytes; the maximum is {max}")] + TooLong { actual: usize, max: u32 }, + + #[error("path must be absolute: {path}")] + NotAbsolute { path: String }, + + #[error("path must be relative: {path}")] + NotRelative { path: String }, + + #[error("path contains a NUL byte")] + NulByte, + + #[error("path has an empty, '.' or '..' segment or a trailing '/': {path}")] + NotNormalized { path: String }, + + #[error("entry name must be one segment other than '.' and '..': {name}")] + NotAName { name: String }, +} + +/// JSON Schema pattern of [`EnvironmentPath`]. +pub const ENVIRONMENT_PATH_PATTERN: &str = "^(/|(/(?!\\.\\.?(/|$))[^/\\u0000]+)+)$"; +/// JSON Schema pattern of [`TreePath`]. +pub const TREE_PATH_PATTERN: &str = + "^(\\.|(?!\\.\\.?(/|$))[^/\\u0000]+(/(?!\\.\\.?(/|$))[^/\\u0000]+)*)$"; +/// JSON Schema pattern of [`EntryName`]. +pub const ENTRY_NAME_PATTERN: &str = "^(?!\\.\\.?$)[^/\\u0000]+$"; +/// JSON Schema pattern of [`ContentDigest`]. +pub const CONTENT_DIGEST_PATTERN: &str = "^sha256:[0-9a-f]{64}$"; + +/// An absolute, normalized POSIX path in the environment: `/`, or `/` and +/// one or more segments joined by `/`. Empty, `.` and `..` segments and a +/// trailing `/` are rejected, never rewritten. At most 4096 bytes. +/// `defaultCwd`, `homeDirectory`, every `cwd` and every path parameter use +/// this type; see the module's path model and confinement rule. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct EnvironmentPath( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 4096), pattern(ENVIRONMENT_PATH_PATTERN)) + )] + String, +); + +impl EnvironmentPath { + pub fn new(path: impl Into) -> Result { + let path = path.into(); + check_common(&path)?; + if !path.starts_with('/') { + return Err(PathError::NotAbsolute { path }); + } + if path != "/" && !segments_normalized(&path[1..]) { + return Err(PathError::NotNormalized { path }); + } + Ok(Self(path)) + } + + pub fn root() -> Self { + Self("/".to_owned()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn is_root(&self) -> bool { + self.0 == "/" + } + + /// Path segments below the root. + pub fn segments(&self) -> impl Iterator { + self.0.split('/').filter(|segment| !segment.is_empty()) + } + + pub fn file_name(&self) -> Option<&str> { + self.segments().last() + } + + pub fn parent(&self) -> Option { + if self.is_root() { + return None; + } + let index = self.0.rfind('/').unwrap_or(0); + Some(if index == 0 { + Self::root() + } else { + Self(self.0[..index].to_owned()) + }) + } + + /// Append a tree path. An empty tree path returns `self`. + pub fn join(&self, tree: &TreePath) -> Result { + if tree.is_root() { + return Ok(self.clone()); + } + let joined = if self.is_root() { + format!("/{}", tree.as_str()) + } else { + format!("{}/{}", self.0, tree.as_str()) + }; + Self::new(joined) + } + + /// True when `self` is `base` or lies below it. + pub fn starts_with(&self, base: &EnvironmentPath) -> bool { + base.is_root() + || self.0 == base.0 + || (self.0.starts_with(&base.0) && self.0.as_bytes().get(base.0.len()) == Some(&b'/')) + } + + /// The tree path of `self` below `base`, or `None` when it is outside. + pub fn relative_to(&self, base: &EnvironmentPath) -> Option { + if !self.starts_with(base) { + return None; + } + let rest = if base.is_root() { + &self.0[1..] + } else { + self.0[base.0.len()..].trim_start_matches('/') + }; + Some(if rest.is_empty() { + TreePath::root() + } else { + TreePath(rest.to_owned()) + }) + } +} + +/// A relative, normalized path below a selected root: segments joined by +/// `/`, without empty, `.` or `..` segments. The single segment `.` names +/// the root itself. At most 4096 bytes. Used for scan, inventory and +/// transfer entries. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct TreePath( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 4096), pattern(TREE_PATH_PATTERN)) + )] + String, +); + +impl Default for TreePath { + fn default() -> Self { + Self::root() + } +} + +impl TreePath { + /// Wire spelling of the root. + pub const ROOT: &'static str = "."; + + pub fn new(path: impl Into) -> Result { + let path = path.into(); + if path == Self::ROOT { + return Ok(Self(path)); + } + check_common(&path)?; + if path.starts_with('/') { + return Err(PathError::NotRelative { path }); + } + if !segments_normalized(&path) { + return Err(PathError::NotNormalized { path }); + } + Ok(Self(path)) + } + + pub fn root() -> Self { + Self(Self::ROOT.to_owned()) + } + + /// The wire spelling: `.` for the root. + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn is_root(&self) -> bool { + self.0 == Self::ROOT + } + + /// Segments below the root; none for the root itself. + pub fn segments(&self) -> impl Iterator { + let path = if self.is_root() { "" } else { self.0.as_str() }; + path.split('/').filter(|segment| !segment.is_empty()) + } + + /// Number of segments: the root has depth 0, its direct children 1. + pub fn depth(&self) -> u32 { + self.segments().count() as u32 + } + + pub fn join(&self, segment: &str) -> Result { + if self.is_root() { + Self::new(segment) + } else { + Self::new(format!("{}/{segment}", self.0)) + } + } +} + +fn check_common(path: &str) -> Result<(), PathError> { + if path.is_empty() { + return Err(PathError::Empty); + } + if path.len() > MAX_PATH_BYTES as usize { + return Err(PathError::TooLong { + actual: path.len(), + max: MAX_PATH_BYTES, + }); + } + if path.contains('\0') { + return Err(PathError::NulByte); + } + Ok(()) +} + +fn segments_normalized(path: &str) -> bool { + path.split('/') + .all(|segment| !matches!(segment, "" | "." | "..")) +} + +macro_rules! path_string_impls { + ($name:ident) => { + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(formatter) + } + } + + impl FromStr for $name { + type Err = PathError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } + } + + impl TryFrom<&str> for $name { + type Error = PathError; + + fn try_from(value: &str) -> Result { + Self::new(value) + } + } + + impl TryFrom for $name { + type Error = PathError; + + fn try_from(value: String) -> Result { + Self::new(value) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } + } + }; +} + +/// One entry name: a single path segment, neither `.` nor `..`, without +/// `/` or NUL, at most 4096 bytes. `fs/readDirectory` entries use it. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct EntryName( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 4096), pattern(ENTRY_NAME_PATTERN)) + )] + String, +); + +impl EntryName { + pub fn new(name: impl Into) -> Result { + let name = name.into(); + check_common(&name)?; + if name.contains('/') || name == "." || name == ".." { + return Err(PathError::NotAName { name }); + } + Ok(Self(name)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +path_string_impls!(EnvironmentPath); +path_string_impls!(TreePath); +path_string_impls!(EntryName); + +/// `sha256:` followed by 64 lowercase hex digits: the digest of a file's raw +/// bytes, independent of its path and mode. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct ContentDigest( + #[cfg_attr( + feature = "schema", + schemars(with = "String", pattern(CONTENT_DIGEST_PATTERN)) + )] + [u8; 32], +); + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +#[error("content digest must be sha256: followed by 64 lowercase hex digits")] +pub struct DigestError; + +impl ContentDigest { + pub const PREFIX: &'static str = "sha256:"; + + pub fn from_sha256(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + pub fn sha256(&self) -> &[u8; 32] { + &self.0 + } + + /// The 64 lowercase hex digits without the prefix. + pub fn hex(&self) -> String { + crate::bytes::encode_hex(&self.0) + } + + pub fn parse(value: &str) -> Result { + let hex = value.strip_prefix(Self::PREFIX).ok_or(DigestError)?; + decode_hex_array::<32>(hex) + .map(Self) + .map_err(|_| DigestError) + } +} + +impl fmt::Display for ContentDigest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}{}", Self::PREFIX, self.hex()) + } +} + +impl FromStr for ContentDigest { + type Err = DigestError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } +} + +impl Serialize for ContentDigest { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for ContentDigest { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::parse(&value).map_err(D::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn environment_paths_are_rejected_rather_than_rewritten() { + for good in ["/", "/workspace", "/a/b.c/d"] { + assert_eq!(EnvironmentPath::new(good).unwrap().as_str(), good); + } + for bad in ["", "relative", "/../etc", "/a/./b", "/a//b", "/a/", "/a\0b"] { + assert!(EnvironmentPath::new(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn backslashes_are_ordinary_name_bytes() { + assert_eq!( + EnvironmentPath::new("/a\\b").unwrap().file_name(), + Some("a\\b") + ); + assert!(EntryName::new("a\\b").is_ok()); + assert!(TreePath::new("dir/a\\b").is_ok()); + } + + #[test] + fn tree_paths_are_relative_and_normalized() { + let root = TreePath::new(".").unwrap(); + assert!(root.is_root()); + assert_eq!(root.depth(), 0); + assert_eq!(serde_json::to_value(TreePath::root()).unwrap(), "."); + assert_eq!(TreePath::root().join("a").unwrap().as_str(), "a"); + assert_eq!(TreePath::new("a/b").unwrap().depth(), 2); + for bad in ["", "/a", "a/../b", "a//b", "./a", "a/", "a/."] { + assert!(TreePath::new(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn entry_names_are_one_segment() { + assert!(EntryName::new("notes.md").is_ok()); + for bad in ["", ".", "..", "a/b"] { + assert!(EntryName::new(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn paths_join_and_relativize() { + let base = EnvironmentPath::new("/workspace").unwrap(); + let tree = TreePath::new("src/lib.rs").unwrap(); + let joined = base.join(&tree).unwrap(); + assert_eq!(joined.as_str(), "/workspace/src/lib.rs"); + assert_eq!(joined.relative_to(&base), Some(tree)); + assert_eq!(base.relative_to(&base), Some(TreePath::root())); + assert_eq!(base.join(&TreePath::root()).unwrap(), base); + assert!( + EnvironmentPath::new("/workspace2") + .unwrap() + .relative_to(&base) + .is_none() + ); + assert_eq!(joined.parent().unwrap().as_str(), "/workspace/src"); + assert_eq!(base.parent().unwrap(), EnvironmentPath::root()); + } + + #[test] + fn digests_are_canonical_lowercase() { + let text = format!("sha256:{}", "ab".repeat(32)); + let digest: ContentDigest = serde_json::from_value(serde_json::json!(text)).unwrap(); + assert_eq!(digest.to_string(), text); + assert!(ContentDigest::parse(&format!("sha256:{}", "AB".repeat(32))).is_err()); + assert!(ContentDigest::parse(&"ab".repeat(32)).is_err()); + } +} diff --git a/crates/environment-protocol/src/registration.rs b/crates/environment-protocol/src/registration.rs index 8ee167db9..36695ab1b 100644 --- a/crates/environment-protocol/src/registration.rs +++ b/crates/environment-protocol/src/registration.rs @@ -1,306 +1,470 @@ -//! Key-based outbound registration: the control-channel messages an `envd` -//! exchanges with the environment gateway after dialing out. +//! The registration channel: the control connection an envd dials to the +//! gateway at [`CONNECT_PATH`](crate::routes::CONNECT_PATH) to register and +//! to be asked for reverse-dialed streams. It runs the shared JSON-RPC +//! envelope; the daemon is the client (it dialed), the gateway the server. +//! Both directions carry requests and notifications. It never carries +//! data-plane messages: every stream is a separate socket the daemon dials +//! on request. //! -//! The control connection carries registration, liveness, and requests to -//! open data connections. It never carries environment-protocol frames: -//! every worker route is served by a separate, reverse-dialed data socket -//! that runs the ordinary data protocol. +//! Sequence: +//! +//! 1. Gateway → daemon, notification [`registration/challenge`]: always the +//! first frame (see [`ChallengeParams`]). +//! 2. Daemon → gateway, request `registration/register`: signs the nonce +//! bound to the gateway origin. The response carries the receipt and +//! `heartbeatIntervalMs`; an error refuses admission and the gateway +//! closes the socket. +//! 3. Gateway → daemon, request `registration/openStream`, any number of +//! times: the daemon dials +//! [`gateway_stream_url`](crate::routes::gateway_stream_url) with the +//! token and answers once the stream socket is upgraded, or with an +//! error. +//! 4. Daemon → gateway, notification `registration/goodbye` before a +//! deliberate shutdown; gateway → daemon, notification +//! `registration/closed` before the gateway ends the session for good. +//! +//! **Frame rules.** +//! +//! - Text frames only. The gateway answers a binary frame as the envelope +//! says (`invalidRequest`, null id); the daemon closes the connection on +//! one. +//! - An unknown request is answered with `methodNotFound`; an unknown +//! notification is ignored and logged. A response to an id the receiver +//! never sent is ignored and logged. +//! - Before admission the gateway accepts only `registration/register`: +//! any other request is answered with `notInitialized` and the gateway +//! closes the connection; any notification closes it without an answer. +//! A `registration/register` after admission is `invalidRequest`, and +//! the connection stays open. +//! - Before it has the register response, the daemon accepts only the +//! challenge (first) and that response. Any request or other +//! notification from the gateway in that window is a protocol violation: +//! the daemon closes the connection and reconnects with backoff. The +//! gateway sends `registration/openStream` and `registration/closed` only +//! after it sent the register response. +//! +//! **Liveness.** The gateway pings every `heartbeatIntervalMs`; the daemon +//! answers. A peer that receives no frame for +//! [`HEARTBEAT_MISSED_INTERVALS`](crate::limits::HEARTBEAT_MISSED_INTERVALS) +//! intervals treats the connection as dead and closes it; the daemon then +//! reconnects with backoff. +//! +//! [`registration/challenge`]: CHALLENGE_METHOD use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest as _, Sha256}; +use thiserror::Error; -use crate::shared::{CURRENT_PROTOCOL_VERSION, ImplementationInfo, SecretString}; - -/// Public route an `envd` dials to register and hold its control connection. -pub const CONNECT_PATH: &str = "/environment-gateway/connect"; - -/// Public route an `envd` dials, presenting a one-time token as a bearer -/// header, to serve one worker route. -pub const DATA_PATH: &str = "/environment-gateway/data"; +use crate::{ + bytes::{SecretString, hex_value}, + error::ProtocolError, + handshake::ImplementationInfo, + ids::{ControlConnectionId, DaemonId, EnvironmentId}, + jsonrpc::decode_params, + metadata::{ + DescriptiveText, DisplayName, Metadata, validate_caller_metadata, validate_display_name, + }, + routes::CanonicalOrigin, +}; -/// Domain separator the daemon signs together with the gateway's nonce, so a -/// signature over a nonce can never be confused with any other Ed25519 use -/// of the same key. -pub const REGISTRATION_SIGNATURE_DOMAIN: &[u8] = b"lightspeed-envd-registration/v1"; +/// Method name of the frozen first frame. +pub const CHALLENGE_METHOD: &str = "registration/challenge"; -/// Bytes of the gateway's per-connection challenge nonce. -pub const REGISTRATION_NONCE_BYTES: usize = 32; +/// Domain separator of the signed registration message. Its suffix numbers +/// the signed-message layout, not the protocol. +pub const REGISTRATION_SIGNATURE_DOMAIN: &[u8] = b"lightspeed-envd-registration/v2"; -pub const MAX_DISPLAY_NAME_BYTES: usize = 128; -pub const MAX_METADATA_ENTRIES: usize = 32; -pub const MAX_METADATA_KEY_BYTES: usize = 64; -pub const MAX_METADATA_VALUE_BYTES: usize = 256; -/// Metadata keys under this prefix are written by Lightspeed only. -pub const RESERVED_METADATA_PREFIX: &str = "lightspeed."; +hex_value!( + /// `nonce`: the gateway's 32 random bytes for one control connection. + ChallengeNonce, + 32, + "^[0-9a-f]{64}$" +); +hex_value!( + /// `daemonPublicKey`: a raw Ed25519 public key. + DaemonPublicKey, + 32, + "^[0-9a-f]{64}$" +); +hex_value!( + /// `signature`: an Ed25519 signature over + /// [`signed_registration_message`]. + Ed25519Signature, + 64, + "^[0-9a-f]{128}$" +); -/// Messages the gateway sends on the control connection. +/// `registration/challenge` params. +/// +/// **Frozen.** This is the first frame of every control connection and the +/// only frame a daemon of any version must parse to learn it is stale: it +/// only ever gains fields, is decoded tolerantly (unknown members are +/// ignored), and a receiver reads `protocolVersion` with +/// [`challenge_protocol_version`] before decoding anything else. The whole +/// frame is +/// `{"jsonrpc":"2.0","method":"registration/challenge","params":{"protocolVersion":N,"nonce":"<64 hex>"}}`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "type", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum GatewayControlMessage { - /// First frame after the upgrade. `nonce` is hex; the daemon signs - /// [`signed_registration_message`] of its raw bytes. - Challenge { - protocol_version: u32, - nonce: String, - }, - /// The daemon is admitted; the receipt identifies the environment it - /// now serves. `heartbeatIntervalMs` tells the daemon how often the - /// gateway pings. - Accepted { - #[serde(flatten)] - receipt: RegistrationReceipt, - heartbeat_interval_ms: u64, - }, - /// Registration failed. The gateway closes the socket after this frame. - Rejected { - code: RegistrationRejectionCode, - message: String, - }, - /// A worker route is waiting: dial `dataUrl` with `token` as a bearer - /// header and run the data protocol on that socket. - OpenData { - token: SecretString, - data_url: String, - }, +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct ChallengeParams { + /// The gateway's `ENVD_PROTOCOL_VERSION`. + pub protocol_version: u32, + pub nonce: ChallengeNonce, } -/// Messages the daemon sends on the control connection. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde( - tag = "type", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -pub enum DaemonControlMessage { - Register(RegisterParams), +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum BootstrapError { + #[error("first frame is not JSON")] + NotJson, + #[error("first frame is not a registration/challenge notification")] + NotChallenge, + #[error("challenge carries no integer protocolVersion")] + NoVersion, +} + +/// Read the gateway's protocol version from a first frame without +/// decoding anything else, so a daemon of any version can tell it is stale +/// before it fails on fields it does not know. +pub fn challenge_protocol_version(frame: &str) -> Result { + let value: Value = serde_json::from_str(frame).map_err(|_| BootstrapError::NotJson)?; + if value.get("method").and_then(Value::as_str) != Some(CHALLENGE_METHOD) { + return Err(BootstrapError::NotChallenge); + } + value + .get("params") + .and_then(|params| params.get("protocolVersion")) + .and_then(Value::as_u64) + .and_then(|version| u32::try_from(version).ok()) + .ok_or(BootstrapError::NoVersion) } +/// `registration/register` params. `registrationKey`, `displayName` and +/// `metadata` are read only at a daemon identity's first admission; a +/// daemon sends the key until it is admitted once and never afterwards. +/// +/// The gateway decodes them with [`RegisterParams::decode`]: build facts, +/// a display name or metadata out of bounds refuse admission with +/// `invalidRequest`; anything else that does not decode is +/// `invalidParams`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RegisterParams { - #[serde(default = "default_protocol_version")] + /// The daemon's `ENVD_PROTOCOL_VERSION`. pub protocol_version: u32, - /// Required for a first-seen daemon public key; ignored for a known one. #[serde(default, skip_serializing_if = "Option::is_none")] pub registration_key: Option, - /// Lowercase hex of the raw 32-byte Ed25519 public key. - pub daemon_public_key: String, - /// Lowercase hex of the 64-byte Ed25519 signature over - /// [`signed_registration_message`] of the challenge nonce. - pub signature: String, + pub daemon_public_key: DaemonPublicKey, + /// Over [`signed_registration_message`] of the challenge nonce and the + /// canonical origin of the URL the daemon dialed. + pub signature: Ed25519Signature, #[serde(default, skip_serializing_if = "Option::is_none")] - pub display_name: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub metadata: BTreeMap, + pub display_name: Option, + #[serde(default, skip_serializing_if = "Metadata::is_empty")] + pub metadata: Metadata, pub implementation: ImplementationInfo, } +impl RegisterParams { + /// Decode `registration/register` params the way the gateway answers + /// them. Descriptive values that have the right shape but break their + /// bounds (`implementation`, `displayName`, `metadata`) are + /// `invalidRequest`, naming the member in the message: admission is + /// refused, never granted with the value dropped. Every other decode + /// failure is `invalidParams` naming the field. + pub fn decode(params: &Value) -> Result { + if let Some(implementation) = params.get("implementation") + && let Some(error) = ImplementationInfo::bounds_violation(implementation) + { + return Err(ProtocolError::invalid_request(format!( + "implementation.{error}" + ))); + } + if let Some(Value::String(name)) = params.get("displayName") + && let Err(error) = validate_display_name(name) + { + return Err(ProtocolError::invalid_request(format!( + "displayName: {error}" + ))); + } + if let Some(metadata) = params.get("metadata") + && let Ok(entries) = BTreeMap::::deserialize(metadata) + && let Err(error) = validate_caller_metadata(&entries) + { + return Err(ProtocolError::invalid_request(format!("metadata: {error}"))); + } + decode_params(params) + } +} + +/// `registration/register` response: the daemon is admitted. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RegisterResponse { + pub receipt: RegistrationReceipt, + /// How often the gateway pings; see the liveness rule. + pub heartbeat_interval_ms: u64, +} + /// What the daemon learns about the environment it serves. Written to the -/// receipt file and logged once; contains no secret material. +/// receipt file as is; contains no secret. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RegistrationReceipt { - pub environment_id: String, - pub incarnation_id: String, - pub daemon_id: String, - pub connection_id: String, - pub identity_mode: String, + pub environment_id: EnvironmentId, + pub daemon_id: DaemonId, + /// The gateway's id for this control connection. + pub control_connection_id: ControlConnectionId, + pub identity_mode: RegistrationIdentityMode, + pub admission: Admission, } -/// Why a registration was refused. Terminal codes mean the daemon must not -/// retry with the same identity and configuration; the others are transient. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// Registration-key policy for the daemon's environment. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[serde(rename_all = "camelCase")] -pub enum RegistrationRejectionCode { - UnsupportedProtocol, - InvalidRequest, - InvalidSignature, - InvalidRegistrationKey, - RegistrationKeyRevoked, - RegistrationKeyExpired, - /// A daemon identity the gateway has never seen presented no key. - UnknownDaemon, - /// The identity's environment was closed; the identity is spent. - EnvironmentClosed, - /// The identity is already bound in another universe. - IdentityInUse, - CapacityExhausted, - RateLimited, - Unavailable, +pub enum RegistrationIdentityMode { + /// The environment outlives disconnects. + Persistent, + /// The environment closes after it stays disconnected past its grace. + Ephemeral, } -impl RegistrationRejectionCode { - pub fn is_terminal(self) -> bool { - !matches!( - self, - Self::CapacityExhausted | Self::RateLimited | Self::Unavailable - ) - } +/// Whether this admission created the environment or rejoined it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum Admission { + /// First admission of this daemon identity: a new environment. + Created, + /// A known identity reconnected to its environment. + Reconnected, } -/// The bytes a daemon signs for one challenge. -pub fn signed_registration_message(nonce: &[u8]) -> Vec { - let mut message = Vec::with_capacity(REGISTRATION_SIGNATURE_DOMAIN.len() + nonce.len()); - message.extend_from_slice(REGISTRATION_SIGNATURE_DOMAIN); - message.extend_from_slice(nonce); - message +/// `registration/openStream` params: the gateway asks the daemon to dial +/// one stream. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OpenStreamParams { + /// One-time bearer token for the stream dial; short-lived. + pub token: SecretString, + pub stream: StreamSpec, } -/// Check the bounds on caller-supplied correlation metadata and display -/// name. Metadata is descriptive only; it never selects or authenticates. -/// Caller-supplied maps may not use the reserved prefix; stored records -/// may, because Lightspeed annotates them itself (see -/// [`validate_metadata_bounds`]). Sessions share this validator so a -/// session and the environment it ran in accept the same keys. -pub fn validate_registration_metadata( - display_name: Option<&str>, - metadata: &BTreeMap, -) -> Result<(), String> { - if let Some(name) = display_name - && (name.is_empty() - || name.len() > MAX_DISPLAY_NAME_BYTES - || name.chars().any(char::is_control)) - { - return Err(format!( - "display name must be 1..={MAX_DISPLAY_NAME_BYTES} bytes without control characters" - )); - } - validate_metadata_bounds(metadata)?; - if let Some(key) = metadata - .keys() - .find(|key| key.starts_with(RESERVED_METADATA_PREFIX)) - { - return Err(format!( - "metadata key {key:?} uses the reserved {RESERVED_METADATA_PREFIX} prefix" - )); - } - Ok(()) +/// What the dialed stream carries. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum StreamSpec { + /// The data plane, served by the daemon as on any data connection. + DataProtocol {}, } -/// Entry count, key and value bytes, and control characters: the bounds -/// every stored metadata map obeys, including the keys under the reserved -/// prefix that Lightspeed writes itself. -pub fn validate_metadata_bounds(metadata: &BTreeMap) -> Result<(), String> { - if metadata.len() > MAX_METADATA_ENTRIES { - return Err(format!( - "metadata has more than {MAX_METADATA_ENTRIES} entries" - )); - } - for (key, value) in metadata { - validate_metadata_entry(key, value)?; - } - Ok(()) +/// `registration/openStream` response, sent once the stream socket is +/// upgraded and served. A failed dial answers `unavailable`; the gateway +/// fails the waiting route at once. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OpenStreamResponse {} + +/// `registration/goodbye` params: the daemon is stopping deliberately. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct GoodbyeParams { + pub reason: GoodbyeReason, } -/// Key and value bytes and control characters for one entry, independent of -/// how many entries a map may hold. -pub fn validate_metadata_entry(key: &str, value: &str) -> Result<(), String> { - if key.is_empty() || key.len() > MAX_METADATA_KEY_BYTES || key.chars().any(char::is_control) { - return Err(format!( - "metadata key {key:?} must be 1..={MAX_METADATA_KEY_BYTES} bytes without control characters" - )); - } - if value.is_empty() - || value.len() > MAX_METADATA_VALUE_BYTES - || value.chars().any(char::is_control) - { - return Err(format!( - "metadata value for {key:?} must be 1..={MAX_METADATA_VALUE_BYTES} bytes without control characters" - )); - } - Ok(()) +/// Why the daemon is leaving. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum GoodbyeReason { + /// `SIGTERM` or `SIGINT`. + Shutdown, } -pub fn encode_hex(bytes: &[u8]) -> String { - use std::fmt::Write as _; - let mut hex = String::with_capacity(bytes.len() * 2); - for byte in bytes { - let _ = write!(hex, "{byte:02x}"); - } - hex +/// `registration/closed` params: the gateway ends the session for good and +/// closes the socket after this frame. The daemon exits or parks, as +/// configured, and does not reconnect. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ClosedParams { + pub reason: ClosedReason, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, } -pub fn decode_hex(value: &str) -> Option> { - if !value.len().is_multiple_of(2) { - return None; - } - (0..value.len()) - .step_by(2) - .map(|index| u8::from_str_radix(&value[index..index + 2], 16).ok()) - .collect() +/// Why the gateway ended the session for good. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub enum ClosedReason { + /// Another process with this daemon identity connected; the newest + /// connection wins and this daemon exits with its own status naming the + /// shared identity. + Superseded, + /// The identity's environment was closed; the identity is spent. + EnvironmentClosed, } -fn default_protocol_version() -> u32 { - CURRENT_PROTOCOL_VERSION +/// The bytes a daemon signs for one challenge: +/// [`REGISTRATION_SIGNATURE_DOMAIN`] ‖ nonce ‖ sha256(origin), where +/// `origin` is the [`canonical_origin`](crate::routes::canonical_origin) of +/// the gateway URL the daemon dialed; the gateway verifies against the +/// canonical origin of each public URL it is configured with. Binding the +/// origin stops a gateway the daemon was pointed at from relaying another +/// gateway's nonce. +pub fn signed_registration_message(nonce: &ChallengeNonce, origin: &CanonicalOrigin) -> Vec { + let origin_hash = Sha256::digest(origin.as_str().as_bytes()); + let mut message = Vec::with_capacity(REGISTRATION_SIGNATURE_DOMAIN.len() + 32 + 32); + message.extend_from_slice(REGISTRATION_SIGNATURE_DOMAIN); + message.extend_from_slice(nonce.as_bytes()); + message.extend_from_slice(&origin_hash); + message } #[cfg(test)] mod tests { + use serde_json::json; + use super::*; #[test] - fn control_messages_are_tagged_and_flatten_the_receipt() { - let accepted = GatewayControlMessage::Accepted { - receipt: RegistrationReceipt { - environment_id: "environment_a".to_owned(), - incarnation_id: "incarnation_a".to_owned(), - daemon_id: "daemon_a".to_owned(), - connection_id: "connection_a".to_owned(), - identity_mode: "ephemeral".to_owned(), - }, - heartbeat_interval_ms: 30_000, + fn the_challenge_version_is_read_before_anything_else() { + let future = json!({ + "jsonrpc": "2.0", + "method": CHALLENGE_METHOD, + "params": {"protocolVersion": 9, "nonce": 7, "newField": {"x": 1}}, + }); + assert_eq!(challenge_protocol_version(&future.to_string()), Ok(9)); + assert_eq!( + challenge_protocol_version("{\"method\":\"other\"}"), + Err(BootstrapError::NotChallenge) + ); + assert_eq!( + challenge_protocol_version("garbage"), + Err(BootstrapError::NotJson) + ); + } + + #[test] + fn the_challenge_decodes_tolerantly() { + let params: ChallengeParams = serde_json::from_value(json!({ + "protocolVersion": 4, + "nonce": "00".repeat(32), + "addedLater": true, + })) + .unwrap(); + assert_eq!(params.protocol_version, 4); + } + + #[test] + fn crypto_values_are_canonical_hex() { + let key = format!("\"{}\"", "ab".repeat(32)); + assert!(serde_json::from_str::(&key).is_ok()); + for bad in [ + format!("\"{}\"", "AB".repeat(32)), + format!("\"{}\"", "ab".repeat(31)), + format!("\"{}\u{e9}\"", "a".repeat(62)), + ] { + assert!( + serde_json::from_str::(&bad).is_err(), + "{bad}" + ); + } + } + + #[test] + fn the_signed_message_binds_the_origin() { + let nonce = ChallengeNonce::from_bytes([9; 32]); + let origin = |url| crate::routes::canonical_origin(url).unwrap(); + let first = signed_registration_message(&nonce, &origin("wss://a.example/x")); + let second = signed_registration_message(&nonce, &origin("wss://b.example")); + assert_eq!( + first, + signed_registration_message(&nonce, &origin("WSS://A.example:443/y")) + ); + assert!(first.starts_with(REGISTRATION_SIGNATURE_DOMAIN)); + assert_ne!(first, second); + assert_eq!(first.len(), REGISTRATION_SIGNATURE_DOMAIN.len() + 64); + } + + #[test] + fn stream_tokens_are_redacted_in_debug_output() { + let params = OpenStreamParams { + token: SecretString::new("secret-token"), + stream: StreamSpec::DataProtocol {}, }; - let json = serde_json::to_value(&accepted).expect("serialize"); - assert_eq!(json["type"], "accepted"); - assert_eq!(json["environmentId"], "environment_a"); - assert_eq!(json["heartbeatIntervalMs"], 30_000); - let parsed: GatewayControlMessage = serde_json::from_value(json).expect("parse"); - assert_eq!(parsed, accepted); - - let open = serde_json::json!({"type": "openData", "token": "t", "dataUrl": "wss://g/d"}); - let parsed: GatewayControlMessage = serde_json::from_value(open).expect("parse"); - assert!(matches!(parsed, GatewayControlMessage::OpenData { .. })); - assert!(!format!("{parsed:?}").contains("\"t\"")); + assert!(!format!("{params:?}").contains("secret-token")); + assert_eq!( + serde_json::to_value(¶ms).unwrap(), + json!({"token": "secret-token", "stream": {"kind": "dataProtocol"}}) + ); + } + + fn register_params() -> Value { + json!({ + "protocolVersion": 4, + "daemonPublicKey": "ab".repeat(32), + "signature": "cd".repeat(64), + "implementation": {"name": "lightspeed-envd", "version": "0.5.0"}, + }) } #[test] - fn rejection_codes_split_terminal_from_retryable() { - assert!(RegistrationRejectionCode::EnvironmentClosed.is_terminal()); - assert!(RegistrationRejectionCode::InvalidSignature.is_terminal()); - assert!(RegistrationRejectionCode::RegistrationKeyRevoked.is_terminal()); - assert!(!RegistrationRejectionCode::CapacityExhausted.is_terminal()); - assert!(!RegistrationRejectionCode::RateLimited.is_terminal()); - assert!(!RegistrationRejectionCode::Unavailable.is_terminal()); + fn out_of_bounds_descriptive_values_refuse_admission() { + assert!(RegisterParams::decode(®ister_params()).is_ok()); + let mut implementation = register_params(); + implementation["implementation"]["version"] = json!("v\u{7}"); + let mut name = register_params(); + name["displayName"] = json!("x".repeat(200)); + let mut metadata = register_params(); + metadata["metadata"] = json!({"lightspeed.daemon": "spoofed"}); + for params in [implementation, name, metadata] { + assert_eq!( + RegisterParams::decode(¶ms).unwrap_err().kind(), + crate::error::ProtocolErrorKind::InvalidRequest, + "{params}" + ); + } } #[test] - fn metadata_bounds_and_reserved_prefix_are_enforced() { - let mut metadata = BTreeMap::new(); - metadata.insert("harbor.trialId".to_owned(), "trial-1".to_owned()); - assert!(validate_registration_metadata(Some("worker"), &metadata).is_ok()); - metadata.insert("lightspeed.x".to_owned(), "y".to_owned()); - assert!(validate_registration_metadata(None, &metadata).is_err()); - metadata.clear(); - metadata.insert("k".to_owned(), "a".repeat(MAX_METADATA_VALUE_BYTES + 1)); - assert!(validate_registration_metadata(None, &metadata).is_err()); - assert!(validate_registration_metadata(Some("bad\nname"), &BTreeMap::new()).is_err()); - assert!(validate_registration_metadata(Some(""), &BTreeMap::new()).is_err()); + fn other_register_decode_failures_are_invalid_params() { + let mut key = register_params(); + key["daemonPublicKey"] = json!("AB".repeat(32)); + let mut extra = register_params(); + extra["dataUrl"] = json!("wss://elsewhere"); + let mut shape = register_params(); + shape["implementation"] = json!({"name": 7}); + for params in [key, extra, shape] { + assert_eq!( + RegisterParams::decode(¶ms).unwrap_err().kind(), + crate::error::ProtocolErrorKind::InvalidParams, + "{params}" + ); + } } #[test] - fn hex_round_trips_and_signed_message_is_domain_separated() { - let bytes = [0u8, 1, 0xab, 0xff]; - assert_eq!(encode_hex(&bytes), "0001abff"); - assert_eq!(decode_hex("0001abff").as_deref(), Some(&bytes[..])); - assert_eq!(decode_hex("abc"), None); - assert_eq!(decode_hex("zz"), None); - let message = signed_registration_message(&[9u8; 4]); - assert!(message.starts_with(REGISTRATION_SIGNATURE_DOMAIN)); - assert!(message.ends_with(&[9u8; 4])); + fn fieldless_stream_kinds_reject_unknown_members() { + assert!(serde_json::from_value::(json!({"kind": "dataProtocol"})).is_ok()); + assert!( + serde_json::from_value::(json!({"kind": "dataProtocol", "x": 1})).is_err() + ); } } diff --git a/crates/environment-protocol/src/routes.rs b/crates/environment-protocol/src/routes.rs new file mode 100644 index 000000000..ae09a789e --- /dev/null +++ b/crates/environment-protocol/src/routes.rs @@ -0,0 +1,712 @@ +//! Public route paths and the URL rules both peers must compute the same +//! way. + +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _}; +use thiserror::Error; + +use crate::{ + control::target::TargetOwner, + ids::ProviderTargetId, + limits::{MAX_URL_BYTES, RETRY_AFTER_MAX_SECS}, + metadata::DescriptiveText, +}; + +/// Gateway route an envd dials to register and hold its control connection. +/// A gateway may be served under a path prefix (`wss://host/ls` plus this +/// path); the daemon is configured with the full connect URL. +pub const CONNECT_PATH: &str = "/environment-gateway/connect"; + +/// Gateway route an envd dials, with a stream token as a bearer header, to +/// serve one reverse-dialed stream. Always derived from the connect URL by +/// replacing its [`CONNECT_PATH`] suffix ([`gateway_stream_url`]), so it +/// shares the connect URL's origin and path prefix. +pub const DATA_PATH: &str = "/environment-gateway/data"; + +/// Gateway route serving the envd discovery document. +pub const DISCOVERY_PATH: &str = "/.well-known/lightspeed-envd"; + +/// Path of the data routes under a controller's origin when the provider +/// is registered without a data route base +/// ([`DataRouteBase::derived_from_controller`]). +pub const DEFAULT_DATA_ROUTE_PATH: &str = "/routes"; + +/// JSON Schema pattern of [`DataRouteBase`]. +pub const DATA_ROUTE_BASE_PATTERN: &str = "^wss?://[^/?#@\\s]+(/[^?#\\s]*[^/?#\\s])?$"; + +/// JSON Schema pattern of [`PublicUrl`]. +pub const PUBLIC_URL_PATTERN: &str = "^https?://[^/?#@\\s]+([/?][^#\\s]*)?$"; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum UrlError { + #[error("URL has an unsupported scheme: {0}")] + Scheme(String), + + #[error("URL has no host: {0}")] + NoHost(String), + + #[error("URL must not carry user information: {0}")] + UserInfo(String), + + #[error("URL has an invalid port: {0}")] + Port(String), + + #[error("URL must not carry a query or fragment: {0}")] + QueryOrFragment(String), + + #[error("URL must not end with '/': {0}")] + TrailingSlash(String), + + #[error("URL is longer than {MAX_URL_BYTES} bytes")] + TooLong, + + #[error("URL contains whitespace or a control character: {0}")] + Character(String), + + #[error("gateway connect URL path must end with {CONNECT_PATH}: {0}")] + NotConnectPath(String), + + #[error("a controller reached over wss needs a wss data route base: {0}")] + Downgrade(String), +} + +/// The parts of an absolute URL the protocol's rules look at. +struct UrlParts<'a> { + /// As written. + scheme: &'a str, + host: &'a str, + port: Option, + path: &'a str, + query: Option<&'a str>, + fragment: Option<&'a str>, +} + +fn split_url(url: &str) -> Result, UrlError> { + if url.len() > MAX_URL_BYTES as usize { + return Err(UrlError::TooLong); + } + if url + .chars() + .any(|character| character.is_whitespace() || character.is_control()) + { + return Err(UrlError::Character(url.to_owned())); + } + let (scheme, rest) = url + .split_once("://") + .ok_or_else(|| UrlError::Scheme(url.to_owned()))?; + let (rest, fragment) = match rest.split_once('#') { + Some((rest, fragment)) => (rest, Some(fragment)), + None => (rest, None), + }; + let (rest, query) = match rest.split_once('?') { + Some((rest, query)) => (rest, Some(query)), + None => (rest, None), + }; + let (authority, path) = match rest.find('/') { + Some(index) => rest.split_at(index), + None => (rest, ""), + }; + if authority.contains('@') { + return Err(UrlError::UserInfo(url.to_owned())); + } + let (host, port) = split_host_port(authority).ok_or_else(|| UrlError::Port(url.to_owned()))?; + if host.is_empty() || host == "[]" { + return Err(UrlError::NoHost(url.to_owned())); + } + Ok(UrlParts { + scheme, + host, + port, + path, + query, + fragment, + }) +} + +/// The canonical WebSocket origin of a URL, the value the registration +/// signature binds. Only [`canonical_origin`] builds one, so the daemon +/// (from the URL it dialed) and the gateway (from its configured public +/// URLs) cannot pass a raw URL where an origin belongs. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CanonicalOrigin(String); + +impl CanonicalOrigin { + pub fn as_str(&self) -> &str { + &self.0 + } + + /// True for a `wss` origin. + pub fn is_secure(&self) -> bool { + self.0.starts_with("wss://") + } +} + +impl fmt::Display for CanonicalOrigin { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +/// The canonical WebSocket origin of a URL: `ws` or `wss` (`http` maps to +/// `ws`, `https` to `wss`, any case), `://`, the lowercase host, and +/// `:port` only when it is not the scheme's default. Path, query and +/// fragment are dropped. +pub fn canonical_origin(url: &str) -> Result { + let parts = split_url(url)?; + let (scheme, default_port) = match parts.scheme.to_ascii_lowercase().as_str() { + "ws" | "http" => ("ws", 80), + "wss" | "https" => ("wss", 443), + _ => return Err(UrlError::Scheme(url.to_owned())), + }; + let host = parts.host.to_ascii_lowercase(); + Ok(CanonicalOrigin(match parts.port { + Some(port) if port != default_port => format!("{scheme}://{host}:{port}"), + _ => format!("{scheme}://{host}"), + })) +} + +/// The URL a daemon dials for a reverse-dialed stream: the canonical origin +/// of its configured gateway connect URL, the connect URL's path with its +/// [`CONNECT_PATH`] suffix replaced by [`DATA_PATH`]. A path prefix in front +/// of the gateway routes is kept. The gateway never names this URL, so a +/// stream can never be pointed at another host or a weaker scheme than the +/// control connection. A connect URL whose path does not end with +/// [`CONNECT_PATH`], or that carries a query or fragment, is refused. +pub fn gateway_stream_url(gateway_connect_url: &str) -> Result { + let parts = split_url(gateway_connect_url)?; + if parts.query.is_some() || parts.fragment.is_some() { + return Err(UrlError::QueryOrFragment(gateway_connect_url.to_owned())); + } + let prefix = parts + .path + .strip_suffix(CONNECT_PATH) + .ok_or_else(|| UrlError::NotConnectPath(gateway_connect_url.to_owned()))?; + Ok(format!( + "{}{prefix}{DATA_PATH}", + canonical_origin(gateway_connect_url)? + )) +} + +/// The absolute `ws://` or `wss://` base URL of a provider's data routes, +/// validated on decode and never rewritten: lowercase scheme, a host and +/// optional port, an optional path that does not end with `/`, no user +/// information, query or fragment, at most 2048 bytes. The operator may +/// configure one when registering the provider; without one the runtime +/// derives it from the controller URL +/// ([`DataRouteBase::derived_from_controller`]). A controller reached over +/// `wss` needs a `wss` base, so a provider credential never travels in +/// cleartext; the runtime checks a configured base with +/// [`DataRouteBase::check_for_controller`]. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct DataRouteBase( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 2048), pattern(DATA_ROUTE_BASE_PATTERN)) + )] + String, +); + +impl DataRouteBase { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let parts = split_url(&value)?; + if !matches!(parts.scheme, "ws" | "wss") { + return Err(UrlError::Scheme(value)); + } + if parts.query.is_some() || parts.fragment.is_some() { + return Err(UrlError::QueryOrFragment(value)); + } + if parts.path.ends_with('/') { + return Err(UrlError::TrailingSlash(value)); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn is_secure(&self) -> bool { + self.0.starts_with("wss://") + } + + /// The base a provider registered without one uses: the controller + /// URL's scheme, host and port as written, followed by + /// [`DEFAULT_DATA_ROUTE_PATH`]; the controller URL's path, query and + /// fragment are dropped. Only a `ws` or `wss` controller URL has one; + /// any other scheme (such as `unix`) is [`UrlError::Scheme`]. + pub fn derived_from_controller(controller_url: &str) -> Result { + // The scheme decides first: a `unix://` controller endpoint has no + // network origin to derive from, so it needs an explicit base. + let scheme = controller_url.split_once("://").map(|(scheme, _)| scheme); + if !matches!(scheme, Some("ws" | "wss")) { + return Err(UrlError::Scheme(controller_url.to_owned())); + } + let parts = split_url(controller_url)?; + let authority = match parts.port { + Some(port) => format!("{}:{port}", parts.host), + None => parts.host.to_owned(), + }; + Self::new(format!( + "{}://{authority}{DEFAULT_DATA_ROUTE_PATH}", + parts.scheme + )) + } + + /// The downgrade rule: a controller whose URL is `wss` (or `https`) may + /// only use a `wss` base. The runtime checks a configured base when the + /// provider is registered. + pub fn check_for_controller(&self, controller_url: &str) -> Result<(), UrlError> { + if canonical_origin(controller_url)?.is_secure() && !self.is_secure() { + return Err(UrlError::Downgrade(self.0.clone())); + } + Ok(()) + } +} + +/// A public `http://` or `https://` URL a provider reports, such as a +/// target's ingress `endpoint`: lowercase scheme, a host and optional port, +/// optional path and query, no user information or fragment, at most 2048 +/// bytes. Validated on decode. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +pub struct PublicUrl( + #[cfg_attr( + feature = "schema", + schemars(length(min = 1, max = 2048), pattern(PUBLIC_URL_PATTERN)) + )] + String, +); + +impl PublicUrl { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let parts = split_url(&value)?; + if !matches!(parts.scheme, "http" | "https") { + return Err(UrlError::Scheme(value)); + } + if parts.fragment.is_some() { + return Err(UrlError::QueryOrFragment(value)); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +macro_rules! url_impls { + ($name:ident) => { + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } + } + + impl std::str::FromStr for $name { + type Err = UrlError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::new(value).map_err(D::Error::custom) + } + } + }; +} + +url_impls!(DataRouteBase); +url_impls!(PublicUrl); + +/// A target's data route under a provider's data route base: the base +/// followed by `/{universeId}/{environmentId}/{targetId}`, each segment +/// percent-encoded. The runtime dials it with the provider credential, when +/// one is configured. Before the upgrade the provider may hold the dial up +/// to [`MAX_DATA_ROUTE_HOLD_MS`](crate::limits::MAX_DATA_ROUTE_HOLD_MS) +/// while it wakes the target, then answers a [`DataRouteRefusal`]. +pub fn provider_data_route( + base: &DataRouteBase, + owner: &TargetOwner, + target_id: &ProviderTargetId, +) -> String { + format!( + "{}/{}/{}/{}", + base.as_str(), + encode_path_segment(owner.universe_id.as_str()), + encode_path_segment(owner.environment_id.as_str()), + encode_path_segment(target_id.as_str()), + ) +} + +/// Body of the HTTP refusal a data route answers instead of upgrading, with +/// `Content-Type: application/json`. A provider data route uses `waking`, +/// `booting`, `gone` and `failed`; the gateway's worker route adds +/// `provisioning`, `disconnected` and `away` and forwards a provider's +/// refusal unchanged. Every 503 carries `Retry-After` in whole seconds, +/// 1 to [`RETRY_AFTER_MAX_SECS`](crate::limits::RETRY_AFTER_MAX_SECS). +/// The body is an HTTP refusal, not a JSON-RPC message, and is tagged by +/// `state`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde( + tag = "state", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub enum DataRouteRefusal { + /// 503: resuming from a powered-down state. + Waking {}, + /// 503: running, envd not answering yet. + Booting {}, + /// 503 (gateway): the environment is still provisioning. + Provisioning {}, + /// 503 (gateway): the registered daemon is not connected. + Disconnected {}, + /// 410 (gateway): the registered daemon has been away past its + /// reconnect bound. + Away {}, + /// 410: the target no longer exists. + Gone {}, + /// 409: broken beyond the provider's recovery. + Failed { message: DescriptiveText }, +} + +impl DataRouteRefusal { + /// The HTTP status the refusal is answered with. + pub fn status(&self) -> u16 { + match self { + Self::Waking {} | Self::Booting {} | Self::Provisioning {} | Self::Disconnected {} => { + 503 + } + Self::Away {} | Self::Gone {} => 410, + Self::Failed { .. } => 409, + } + } + + /// True for the 503 refusals: the same dial may succeed after + /// `Retry-After`. + pub fn is_retryable(&self) -> bool { + self.status() == 503 + } + + /// The refusal a refused upgrade names: its body when that decodes, + /// else what its status alone names. The upgrade response carries only + /// the bytes read along with its headers, so a body that arrives in a + /// later TCP segment or TLS record is missing; the refusal then must not + /// depend on how the transport split the bytes. 410 is `gone`, 409 is + /// `failed`, and a 503 with `Retry-After` is `booting`, a wait. Any other + /// status, such as a 503 without `Retry-After` from a proxy, names no + /// refusal. + pub fn from_refused_upgrade( + status: u16, + body: Option<&[u8]>, + retry_after_ms: Option, + ) -> Option { + if let Some(refusal) = body.and_then(|body| serde_json::from_slice::(body).ok()) { + return Some(refusal); + } + match status { + 410 => Some(Self::Gone {}), + 409 => Some(Self::Failed { + message: DescriptiveText::new("the data route reports the target failed") + .expect("a fixed failure message is valid text"), + }), + 503 if retry_after_ms.is_some() => Some(Self::Booting {}), + _ => None, + } + } + + /// The `state` tag. + pub fn state(&self) -> &'static str { + match self { + Self::Waking {} => "waking", + Self::Booting {} => "booting", + Self::Provisioning {} => "provisioning", + Self::Disconnected {} => "disconnected", + Self::Away {} => "away", + Self::Gone {} => "gone", + Self::Failed { .. } => "failed", + } + } +} + +impl fmt::Display for DataRouteRefusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Failed { message } => write!(formatter, "failed: {message}"), + refusal => formatter.write_str(refusal.state()), + } + } +} + +/// A `Retry-After` header value in whole seconds, clamped to +/// [`RETRY_AFTER_MAX_SECS`], as milliseconds. The HTTP-date form is not +/// used by the protocol and names no delay. +pub fn parse_retry_after_ms(value: &str) -> Option { + let seconds: u64 = value.trim().parse().ok()?; + Some(seconds.min(RETRY_AFTER_MAX_SECS) * 1_000) +} + +/// Percent-encode one URL path segment: everything except RFC 3986 +/// unreserved characters. +pub fn encode_path_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(byte as char); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +fn split_host_port(authority: &str) -> Option<(&str, Option)> { + if let Some(rest) = authority.strip_prefix('[') { + let (host, after) = rest.split_once(']')?; + let host = &authority[..host.len() + 2]; + return match after { + "" => Some((host, None)), + port => Some((host, Some(port.strip_prefix(':')?.parse().ok()?))), + }; + } + match authority.rsplit_once(':') { + Some((host, port)) => Some((host, Some(port.parse().ok()?))), + None => Some((authority, None)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn origins_are_canonical() { + for (url, origin) in [ + ( + "wss://GW.Example.com/environment-gateway/connect", + "wss://gw.example.com", + ), + ("wss://gw.example.com:443/x?y#z", "wss://gw.example.com"), + ("https://gw.example.com:8443", "wss://gw.example.com:8443"), + ( + "ws://127.0.0.1:18080/environment-gateway/connect", + "ws://127.0.0.1:18080", + ), + ("http://[::1]:80/", "ws://[::1]"), + ] { + assert_eq!(canonical_origin(url).unwrap().as_str(), origin, "{url}"); + } + for bad in [ + "ftp://x", + "wss://", + "wss://user@host", + "wss://host:port", + "host", + "wss://host/a b", + ] { + assert!(canonical_origin(bad).is_err(), "{bad}"); + } + } + + #[test] + fn stream_urls_derive_from_the_connect_url() { + assert_eq!( + gateway_stream_url("wss://gw.example.com/environment-gateway/connect").unwrap(), + "wss://gw.example.com/environment-gateway/data" + ); + assert_eq!( + gateway_stream_url("wss://GW.example.com:443/ls/environment-gateway/connect").unwrap(), + "wss://gw.example.com/ls/environment-gateway/data", + "a path prefix is kept" + ); + assert!(matches!( + gateway_stream_url("wss://gw.example.com/connect"), + Err(UrlError::NotConnectPath(_)) + )); + assert!(matches!( + gateway_stream_url("wss://gw.example.com/environment-gateway/connect?x=1"), + Err(UrlError::QueryOrFragment(_)) + )); + } + + #[test] + fn data_route_bases_are_validated_not_rewritten() { + for good in [ + "wss://p.example/routes", + "ws://127.0.0.1:9000", + "wss://p.example", + ] { + assert_eq!(DataRouteBase::new(good).unwrap().as_str(), good); + } + for bad in [ + "wss://p.example/routes/", + "wss://p.example/routes?x=1", + "WSS://p.example", + "https://p.example", + "wss://u@p.example", + "wss:///routes", + ] { + assert!(DataRouteBase::new(bad).is_err(), "{bad}"); + } + assert!(serde_json::from_str::("\"wss://p.example/\"").is_err()); + } + + #[test] + fn a_wss_controller_cannot_use_a_ws_data_route_base() { + let plain = DataRouteBase::new("ws://p.example/routes").unwrap(); + assert!(matches!( + plain.check_for_controller("wss://p.example/control"), + Err(UrlError::Downgrade(_)) + )); + assert!(plain.check_for_controller("ws://127.0.0.1/control").is_ok()); + let secure = DataRouteBase::new("wss://p.example/routes").unwrap(); + assert!( + secure + .check_for_controller("wss://p.example/control") + .is_ok() + ); + } + + #[test] + fn public_urls_allow_paths_and_queries() { + assert!(PublicUrl::new("https://app.example/x?y=1").is_ok()); + for bad in ["ftp://x", "https://x/#f", "https://u@x", "wss://x"] { + assert!(PublicUrl::new(bad).is_err(), "{bad}"); + } + } + + #[test] + fn data_routes_encode_opaque_target_ids() { + let owner = TargetOwner { + universe_id: "u1".parse().unwrap(), + environment_id: "env_1".parse().unwrap(), + }; + let target = ProviderTargetId::new("default/pod abc").unwrap(); + let base = DataRouteBase::new("wss://p.example/routes").unwrap(); + assert_eq!( + provider_data_route(&base, &owner, &target), + "wss://p.example/routes/u1/env_1/default%2Fpod%20abc" + ); + } + + #[test] + fn data_route_bases_derive_from_the_controller_origin() { + for (controller, base) in [ + ("wss://p.example/control", "wss://p.example/routes"), + ( + "wss://p.example:19090/x/y?z=1", + "wss://p.example:19090/routes", + ), + ("ws://127.0.0.1:9000", "ws://127.0.0.1:9000/routes"), + ("ws://[::1]:9000/control", "ws://[::1]:9000/routes"), + ] { + assert_eq!( + DataRouteBase::derived_from_controller(controller) + .unwrap() + .as_str(), + base, + "{controller}" + ); + } + for bad in ["unix:///run/provider.sock", "https://p.example", "wss://"] { + assert!( + DataRouteBase::derived_from_controller(bad).is_err(), + "{bad}" + ); + } + assert!(matches!( + DataRouteBase::derived_from_controller("unix:///run/provider.sock"), + Err(UrlError::Scheme(_)) + )); + } + + #[test] + fn refusals_are_state_tagged_with_fixed_statuses() { + use serde_json::json; + + let cases = [ + (json!({"state": "waking"}), 503), + (json!({"state": "booting"}), 503), + (json!({"state": "provisioning"}), 503), + (json!({"state": "disconnected"}), 503), + (json!({"state": "away"}), 410), + (json!({"state": "gone"}), 410), + (json!({"state": "failed", "message": "disk lost"}), 409), + ]; + for (body, status) in cases { + let refusal: DataRouteRefusal = serde_json::from_value(body.clone()).unwrap(); + assert_eq!(refusal.status(), status, "{body}"); + assert_eq!(refusal.is_retryable(), status == 503, "{body}"); + assert_eq!(body["state"], refusal.state()); + assert_eq!(serde_json::to_value(&refusal).unwrap(), body); + } + for bad in [ + json!({"state": "waking", "retryAfterMs": 1}), + json!({"state": "failed"}), + json!({"state": "sleeping"}), + json!({}), + ] { + assert!( + serde_json::from_value::(bad.clone()).is_err(), + "{bad}" + ); + } + assert_eq!( + DataRouteRefusal::Failed { + message: DescriptiveText::lossy("disk lost").unwrap() + } + .to_string(), + "failed: disk lost" + ); + } + + #[test] + fn refused_upgrades_decode_their_body_or_fall_back_on_their_status() { + let refused = DataRouteRefusal::from_refused_upgrade; + assert_eq!( + refused(503, Some(br#"{"state":"waking"}"#), Some(2_000)), + Some(DataRouteRefusal::Waking {}) + ); + assert_eq!(refused(410, None, None), Some(DataRouteRefusal::Gone {})); + assert_eq!( + refused(410, Some(br#"{"state":"aw"#), None), + Some(DataRouteRefusal::Gone {}) + ); + assert!(matches!( + refused(409, Some(b""), None), + Some(DataRouteRefusal::Failed { .. }) + )); + assert_eq!( + refused(503, None, Some(4_000)), + Some(DataRouteRefusal::Booting {}) + ); + assert_eq!(refused(503, None, None), None); + assert_eq!(refused(502, None, Some(4_000)), None); + } + + #[test] + fn retry_after_is_whole_seconds_clamped_to_the_maximum() { + assert_eq!(parse_retry_after_ms(" 3 "), Some(3_000)); + assert_eq!( + parse_retry_after_ms("3600"), + Some(RETRY_AFTER_MAX_SECS * 1_000) + ); + assert_eq!(parse_retry_after_ms("Wed, 21 Oct 2026 07:28:00 GMT"), None); + } +} diff --git a/crates/environment-protocol/src/serde_util.rs b/crates/environment-protocol/src/serde_util.rs new file mode 100644 index 000000000..659ebbc82 --- /dev/null +++ b/crates/environment-protocol/src/serde_util.rs @@ -0,0 +1,13 @@ +//! Serialization predicates shared by the wire types. + +pub(crate) fn is_false(value: &bool) -> bool { + !*value +} + +pub(crate) fn is_zero_u32(value: &u32) -> bool { + *value == 0 +} + +pub(crate) fn is_zero_u64(value: &u64) -> bool { + *value == 0 +} diff --git a/crates/environment-protocol/src/shared.rs b/crates/environment-protocol/src/shared.rs deleted file mode 100644 index 3c6f1924d..000000000 --- a/crates/environment-protocol/src/shared.rs +++ /dev/null @@ -1,461 +0,0 @@ -//! Shared protocol primitives. - -use std::{fmt, str::FromStr}; - -use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as SerdeError}; -use thiserror::Error; - -pub const CURRENT_PROTOCOL_VERSION: u32 = 2; - -macro_rules! string_id { - ($name:ident) => { - #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] - #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] - #[serde(transparent)] - pub struct $name(pub String); - - impl $name { - pub fn new(value: impl Into) -> Self { - Self(value.into()) - } - - pub fn as_str(&self) -> &str { - &self.0 - } - } - - impl fmt::Display for $name { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(formatter) - } - } - - impl From<&str> for $name { - fn from(value: &str) -> Self { - Self::new(value) - } - } - - impl From for $name { - fn from(value: String) -> Self { - Self::new(value) - } - } - }; -} - -string_id!(ProviderTargetId); -string_id!(EnvironmentConnectionId); -string_id!(ProcessId); -string_id!(JobId); - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum EnvironmentTransport { - WebSocket, - Http, - Stdio, - Ssh, - Provider { - #[serde(rename = "providerType")] - provider_type: String, - }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentDataConnection { - pub target_id: ProviderTargetId, - pub endpoint: String, - pub transport: EnvironmentTransport, - pub scope: EnvironmentScope, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_cwd: Option, - pub capabilities: EnvironmentCapabilities, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum EnvironmentScope { - Default, - #[serde(rename_all = "camelCase")] - Session { - session_id: String, - }, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct EnvironmentCapabilities { - #[serde(default)] - pub network: bool, - #[serde(default)] - pub filesystem_read: bool, - #[serde(default)] - pub filesystem_write: bool, - /// The environment endpoint implements the bounded `fs/searchText` operation natively, so - /// callers can avoid per-file transfer for recursive text search. - #[serde(default)] - pub filesystem_search: bool, - /// The environment endpoint implements the bounded `fs/globFiles` operation natively, so - /// callers can avoid per-directory transfer for recursive enumeration. - #[serde(default)] - pub filesystem_glob: bool, - /// The environment endpoint honors `offset`/`maxBytes` on `fs/readFile` and reports the - /// true file size, so oversized files are truncated at the source instead - /// of transferred in full. - #[serde(default)] - pub filesystem_ranged_read: bool, - /// Bounded inline capture; absent on older endpoints. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub filesystem_capture: bool, - /// Paged transfer sessions with whole-file digests and bounded byte exchanges. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub filesystem_transfer: bool, - /// Bounded filesystem inventory observations with conditional fingerprints. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub filesystem_scan: bool, - /// Bounded staged materialization; false for read-only endpoints. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub filesystem_materialize: bool, - #[serde(default)] - pub process_start: bool, - #[serde(default)] - pub process_stdin: bool, - #[serde(default)] - pub process_terminate: bool, - #[serde(default)] - pub process_output_polling: bool, - #[serde(default)] - pub process_output_notifications: bool, - #[serde(default)] - pub process_pty: bool, - #[serde(default)] - pub job_start: bool, - #[serde(default)] - pub job_list: bool, - #[serde(default)] - pub job_read: bool, - #[serde(default)] - pub job_cancel: bool, - #[serde(default)] - pub job_wait_hint: bool, - #[serde(default)] - pub job_dependencies: bool, - #[serde(default)] - pub job_queue_keys: bool, -} - -impl EnvironmentCapabilities { - pub fn filesystem(read: bool, write: bool) -> Self { - Self { - filesystem_read: read, - filesystem_write: write, - ..Self::default() - } - } - - pub fn with_filesystem_search(mut self) -> Self { - self.filesystem_search = true; - self - } - - pub fn with_process(mut self) -> Self { - self.process_start = true; - self.process_stdin = true; - self.process_terminate = true; - self.process_output_polling = true; - self.process_output_notifications = true; - self - } - - pub fn with_jobs(mut self) -> Self { - self.job_start = true; - self.job_list = true; - self.job_read = true; - self.job_cancel = true; - self.job_wait_hint = false; - self.job_dependencies = true; - self.job_queue_keys = true; - self - } -} - -/// What an endpoint says about its own build. Descriptive only: the protocol -/// version decides compatibility, and nothing here selects, authenticates, -/// or gates. The gateway records these facts on the environment row so a -/// deployment can see which build serves each environment. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ImplementationInfo { - pub name: String, - /// Release version of the build, for example `0.1.0`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub version: Option, - /// Full git commit the build was made from. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub git_sha: Option, - /// Rust target triple the build runs on. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub target: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ByteChunk(pub Vec); - -impl ByteChunk { - pub fn new(bytes: impl Into>) -> Self { - Self(bytes.into()) - } - - pub fn into_inner(self) -> Vec { - self.0 - } - - pub fn as_slice(&self) -> &[u8] { - &self.0 - } -} - -impl From> for ByteChunk { - fn from(value: Vec) -> Self { - Self(value) - } -} - -impl From<&[u8]> for ByteChunk { - fn from(value: &[u8]) -> Self { - Self(value.to_vec()) - } -} - -impl Serialize for ByteChunk { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&BASE64_STANDARD.encode(&self.0)) - } -} - -impl<'de> Deserialize<'de> for ByteChunk { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let encoded = String::deserialize(deserializer)?; - BASE64_STANDARD - .decode(encoded) - .map(Self) - .map_err(SerdeError::custom) - } -} - -/// Secret string carried over the environment data protocol. -/// -/// It serializes as a string because it is a wire value, but `Debug` is -/// redacted so ordinary structured logging does not print plaintext. -#[derive(Clone, PartialEq, Eq)] -pub struct SecretString(String); - -impl SecretString { - pub fn new(value: impl Into) -> Self { - Self(value.into()) - } - - pub fn expose(&self) -> &str { - &self.0 - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -impl fmt::Debug for SecretString { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("SecretString()") - } -} - -impl From for SecretString { - fn from(value: String) -> Self { - Self(value) - } -} - -impl From<&str> for SecretString { - fn from(value: &str) -> Self { - Self(value.to_owned()) - } -} - -impl Serialize for SecretString { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.0) - } -} - -impl<'de> Deserialize<'de> for SecretString { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - String::deserialize(deserializer).map(Self) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[cfg_attr(feature = "schema", schemars(with = "String"))] -pub struct EnvironmentPath { - normalized: String, -} - -impl EnvironmentPath { - pub fn new(path: impl AsRef) -> Result { - normalize_path(path.as_ref()).map(|normalized| Self { normalized }) - } - - pub fn root() -> Self { - Self { - normalized: "/".to_owned(), - } - } - - pub fn current_dir() -> Self { - Self { - normalized: ".".to_owned(), - } - } - - pub fn as_str(&self) -> &str { - &self.normalized - } - - pub fn is_absolute(&self) -> bool { - self.normalized.starts_with('/') - } -} - -impl fmt::Display for EnvironmentPath { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - self.normalized.fmt(formatter) - } -} - -impl FromStr for EnvironmentPath { - type Err = EnvironmentPathError; - - fn from_str(value: &str) -> Result { - Self::new(value) - } -} - -impl TryFrom<&str> for EnvironmentPath { - type Error = EnvironmentPathError; - - fn try_from(value: &str) -> Result { - Self::new(value) - } -} - -impl Serialize for EnvironmentPath { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(self.as_str()) - } -} - -impl<'de> Deserialize<'de> for EnvironmentPath { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::new(value).map_err(SerdeError::custom) - } -} - -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum EnvironmentPathError { - #[error("environment path is empty")] - Empty, - - #[error("environment path contains a NUL byte")] - NulByte, - - #[error("environment path must use '/' separators: {path}")] - NonUnixSeparator { path: String }, -} - -fn normalize_path(path: &str) -> Result { - if path.is_empty() { - return Err(EnvironmentPathError::Empty); - } - if path.contains('\0') { - return Err(EnvironmentPathError::NulByte); - } - if path.contains('\\') { - return Err(EnvironmentPathError::NonUnixSeparator { - path: path.to_owned(), - }); - } - - let absolute = path.starts_with('/'); - let mut segments = Vec::new(); - for segment in path.split('/') { - match segment { - "" | "." => {} - ".." => { - if let Some(last) = segments.last() - && *last != ".." - { - segments.pop(); - continue; - } - if !absolute { - segments.push(".."); - } - } - segment => segments.push(segment), - } - } - - if absolute { - if segments.is_empty() { - Ok("/".to_owned()) - } else { - Ok(format!("/{}", segments.join("/"))) - } - } else if segments.is_empty() { - Ok(".".to_owned()) - } else { - Ok(segments.join("/")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn byte_chunk_serializes_as_base64_string() { - let value = serde_json::to_value(ByteChunk::from(b"hello".as_slice())).expect("serialize"); - - assert_eq!(value, serde_json::json!("aGVsbG8=")); - } - - #[test] - fn environment_path_normalizes_like_agent_fs_path() { - let path = EnvironmentPath::new("src//./lib/../main.rs").expect("path"); - - assert_eq!(path.as_str(), "src/main.rs"); - } -} diff --git a/crates/environment-protocol/src/version.rs b/crates/environment-protocol/src/version.rs new file mode 100644 index 000000000..8ab3c45a6 --- /dev/null +++ b/crates/environment-protocol/src/version.rs @@ -0,0 +1,17 @@ +//! Protocol version numbers. +//! +//! Each number is matched exactly: a peer that states any other number is +//! refused with `versionMismatch`, and no range or negotiation exists. A +//! change to a plane's wire shape or to the meaning of an existing field or +//! method needs a new number for that plane. `release/metadata.env` carries +//! the same numbers and `scripts/release/verify-metadata.sh` checks them. + +/// Version of the data plane and the registration channel. +/// +/// Both share one number so a data-plane change also changes the number in +/// the gateway's `registration/challenge`, which is what triggers a +/// registered daemon's automatic upgrade. +pub const ENVD_PROTOCOL_VERSION: u32 = 4; + +/// Version of the provider controller plane. +pub const CONTROLLER_PROTOCOL_VERSION: u32 = 4; diff --git a/crates/environment-protocol/tests/contract.rs b/crates/environment-protocol/tests/contract.rs new file mode 100644 index 000000000..e772e0f1b --- /dev/null +++ b/crates/environment-protocol/tests/contract.rs @@ -0,0 +1,47 @@ +//! The committed `contract/` documents match the current wire types. When +//! this fails, regenerate with +//! `cargo run -p environment-protocol --features schema --bin export-contract` +//! and commit the result with the type change. + +use std::{fs, path::PathBuf}; + +use serde_json::Value; + +#[test] +fn committed_contract_is_current() { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("contract"); + for plane in environment_protocol::contract::export() { + for (name, current) in [ + (plane.schema_file_name(), &plane.schema_bundle), + (plane.openrpc_file_name(), &plane.openrpc), + ] { + let path = dir.join(&name); + let text = fs::read_to_string(&path).unwrap_or_else(|error| { + panic!( + "missing {}: {error}; run the export-contract binary", + path.display() + ) + }); + let committed: Value = serde_json::from_str(&text).expect("contract parses"); + assert_eq!( + &committed, current, + "{name} is stale; run the export-contract binary" + ); + } + } +} + +#[test] +fn every_method_appears_in_its_plane_document() { + for plane in environment_protocol::contract::export() { + let names: Vec<&str> = plane.openrpc["methods"] + .as_array() + .expect("methods") + .iter() + .filter_map(|method| method["name"].as_str()) + .collect(); + for spec in environment_protocol::manifest::plane_methods(plane.plane) { + assert!(names.contains(&spec.name), "{} missing", spec.name); + } + } +} diff --git a/crates/environment-protocol/tests/fixtures.rs b/crates/environment-protocol/tests/fixtures.rs new file mode 100644 index 000000000..60d3d7c8e --- /dev/null +++ b/crates/environment-protocol/tests/fixtures.rs @@ -0,0 +1,203 @@ +//! One fixture per method: params, response and an error, round-tripped +//! through the manifest's wire types under the strict decoding policy. + +use std::{collections::BTreeSet, fs, path::PathBuf}; + +use environment_protocol::{ + bootstrap::{BuildFacts, DiscoveryDocument}, + error::ProtocolError, + handshake::Plane, + manifest::{MessageKind, MethodSpec, RegistrationChallenge, methods}, +}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Value, json}; + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures") +} + +fn fixture_file(spec: &MethodSpec) -> PathBuf { + let name = match spec.name.strip_prefix("$/") { + Some(rest) => format!("rpc/{rest}"), + None => spec.name.to_owned(), + }; + fixtures_dir() + .join(spec.plane.as_str()) + .join(format!("{}.json", name.replace('/', "."))) +} + +fn load(path: &PathBuf) -> Value { + let text = fs::read_to_string(path) + .unwrap_or_else(|error| panic!("missing fixture {}: {error}", path.display())); + serde_json::from_str(&text).unwrap_or_else(|error| panic!("{}: {error}", path.display())) +} + +fn round_trip(value: &Value) -> Value { + serde_json::to_value(serde_json::from_value::(value.clone()).expect("decode")) + .expect("encode") +} + +#[test] +fn every_method_has_fixtures_that_round_trip() { + for spec in methods() { + let path = fixture_file(spec); + let fixture = load(&path); + let params = &fixture["params"]; + assert_eq!( + (spec.normalize_params)(params) + .unwrap_or_else(|error| panic!("{} params: {error}", spec.name)), + *params, + "{} params are not canonical", + spec.name + ); + match spec.kind { + MessageKind::Request => { + let response = &fixture["response"]; + let normalize = spec.normalize_response.expect("requests have responses"); + assert_eq!( + normalize(response) + .unwrap_or_else(|error| panic!("{} response: {error}", spec.name)), + *response, + "{} response is not canonical", + spec.name + ); + let error = &fixture["error"]; + let decoded: ProtocolError = serde_json::from_value(error.clone()) + .unwrap_or_else(|decode| panic!("{} error: {decode}", spec.name)); + assert_eq!( + error["code"], + json!(decoded.code()), + "{} error code does not match its kind", + spec.name + ); + assert_eq!(serde_json::to_value(&decoded).unwrap(), *error); + } + MessageKind::Notification => { + assert!( + fixture.get("response").is_none() && fixture.get("error").is_none(), + "notification {} is never answered", + spec.name + ); + } + } + } +} + +#[test] +fn every_fixture_belongs_to_a_method() { + let expected: BTreeSet = methods().map(fixture_file).collect(); + for plane in [Plane::Data, Plane::Controller, Plane::Registration] { + for entry in fs::read_dir(fixtures_dir().join(plane.as_str())).expect("fixture dir") { + let path = entry.expect("entry").path(); + assert!( + expected.contains(&path), + "orphan fixture {}", + path.display() + ); + } + } +} + +/// Members whose values are free-form string maps rather than wire +/// structs. +const FREE_FORM_MAPS: &[&str] = &["metadata", "env", "secretEnv"]; + +/// The path of every JSON object inside `value` that is a wire struct, +/// `value` itself included. +fn object_paths(value: &Value, path: &mut Vec, paths: &mut Vec>) { + match value { + Value::Object(map) => { + paths.push(path.clone()); + for (key, child) in map { + if FREE_FORM_MAPS.contains(&key.as_str()) { + continue; + } + path.push(key.clone()); + object_paths(child, path, paths); + path.pop(); + } + } + Value::Array(items) => { + for (index, child) in items.iter().enumerate() { + path.push(index.to_string()); + object_paths(child, path, paths); + path.pop(); + } + } + _ => {} + } +} + +/// `value` with an unknown member added to the object at `path`. +fn with_unknown_member(value: &Value, path: &[String]) -> Value { + let mut value = value.clone(); + let mut target = &mut value; + for segment in path { + target = match target { + Value::Array(items) => &mut items[segment.parse::().expect("index")], + other => &mut other[segment.as_str()], + }; + } + target["unexpectedMember"] = json!(true); + value +} + +/// Every nested object of `value` refuses an unknown member under `decode`. +fn assert_strict(name: &str, part: &str, value: &Value, decode: impl Fn(&Value) -> bool) { + let mut paths = Vec::new(); + object_paths(value, &mut Vec::new(), &mut paths); + assert!(!paths.is_empty(), "{name} {part} has no object"); + for path in paths { + assert!( + !decode(&with_unknown_member(value, &path)), + "{name} {part} accepts an unknown member at /{}", + path.join("/") + ); + } +} + +#[test] +fn versioned_messages_reject_unknown_members_at_every_depth() { + for spec in methods() { + if spec.name == RegistrationChallenge::NAME { + continue; + } + let fixture = load(&fixture_file(spec)); + assert_strict(spec.name, "params", &fixture["params"], |value| { + (spec.normalize_params)(value).is_ok() + }); + if let Some(normalize) = spec.normalize_response { + assert_strict(spec.name, "response", &fixture["response"], |value| { + normalize(value).is_ok() + }); + assert_strict(spec.name, "error", &fixture["error"], |value| { + serde_json::from_value::(value.clone()).is_ok() + }); + } + } +} + +#[test] +fn the_frozen_challenge_tolerates_additions() { + let spec = methods() + .find(|spec| spec.name == RegistrationChallenge::NAME) + .expect("challenge"); + let mut params = load(&fixture_file(spec))["params"].clone(); + params["addedInALaterVersion"] = json!({"any": "shape"}); + assert!((spec.normalize_params)(¶ms).is_ok()); +} + +#[test] +fn bootstrap_documents_round_trip_and_tolerate_additions() { + let dir = fixtures_dir().join("bootstrap"); + let discovery = load(&dir.join("discovery-document.json")); + assert_eq!(round_trip::(&discovery), discovery); + let build = load(&dir.join("build-facts.json")); + assert_eq!(round_trip::(&build), build); + + let mut extended = discovery.clone(); + extended["addedLater"] = json!(1); + extended["artifacts"]["x86_64-unknown-linux-musl"]["url"] = Value::Null; + let decoded: DiscoveryDocument = serde_json::from_value(extended).expect("tolerant"); + assert!(decoded.artifacts["x86_64-unknown-linux-musl"].url.is_none()); +} diff --git a/crates/environment-protocol/tests/serde.rs b/crates/environment-protocol/tests/serde.rs index 2223b5652..7476c770f 100644 --- a/crates/environment-protocol/tests/serde.rs +++ b/crates/environment-protocol/tests/serde.rs @@ -1,416 +1,221 @@ -use std::collections::BTreeMap; +//! Wire-shape rules that the per-method fixtures do not spell out. use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION, control::{ - handshake::ControllerInitializeParams, - methods::{ - CREATE_TARGET_METHOD, INITIALIZE_METHOD as CONTROL_INITIALIZE_METHOD, - LIST_TARGETS_METHOD, SET_TARGET_POWER_METHOD, - }, - targets::{ - CreateTargetParams, CreateTargetResponse, ListTargetsResponse, PowerState, - ProviderBindingContext, ProviderTargetStatus, ProviderTargetSummary, - SetTargetPowerParams, - }, + handshake::{ControllerInitializeParams, ControllerInitializeResponse}, + target::{CreateTargetParams, PowerState, TargetObservation, TargetPhase, TargetResponse}, + template::ListTemplatesResponse, }, data::{ - fs::ReadFileResponse, - handshake::InitializeParams, - idle::IdleResponse, - jobs::{ - JobArtifact, JobDependency, JobDependencyPolicy, JobOutputChunk, JobOutputStream, - JobReadResult, JobStartSpec, JobStatus, JobSummary, ListJobsParams, ReadJobsResponse, - StartJobsParams, - }, - methods::{ - ENV_IDLE_METHOD, FS_READ_FILE_METHOD, INITIALIZE_METHOD, JOB_CANCEL_METHOD, - JOB_LIST_METHOD, JOB_READ_METHOD, JOB_START_METHOD, PROCESS_OUTPUT_METHOD, - PROCESS_START_METHOD, - }, - process::{ - LeftoverProcess, ProcessOutputChunk, ProcessOutputStream, ReadProcessResponse, - StartProcessParams, - }, - }, - shared::{ - ByteChunk, CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentConnectionId, - EnvironmentScope, JobId, ProcessId, ProviderTargetId, + exec::{ExecInput, ExitStatus, GapReason, OutputPage}, + handshake::{DataCapabilities, InitializeParams, InitializeResponse}, + job::{JobDependency, JobReadEntry, JobStatus, ReadJobsResponse, StartJobsParams}, + process::{ReadProcessParams, ReadProcessResponse}, + scan::{ScanResponse, ScanRootResult}, }, + error::{NotFoundReason, ProtocolError, ProtocolErrorKind}, + handshake::Plane, + manifest::{FsScan, ProcessRead, RequestMethod}, + registration::{RegisterParams, RegisterResponse}, }; -use serde::{Serialize, de::DeserializeOwned}; +use serde::de::DeserializeOwned; use serde_json::{Value, json}; -fn fixture(name: &str) -> Value { - serde_json::from_str(match name { - "data_initialize_params" => include_str!("../fixtures/data_initialize_params.json"), - "fs_read_file_response" => include_str!("../fixtures/fs_read_file_response.json"), - "process_start_params" => include_str!("../fixtures/process_start_params.json"), - "process_read_response" => include_str!("../fixtures/process_read_response.json"), - "job_start_params" => include_str!("../fixtures/job_start_params.json"), - "job_read_response" => include_str!("../fixtures/job_read_response.json"), - "controller_initialize_params" => { - include_str!("../fixtures/controller_initialize_params.json") - } - "controller_create_target_params" => { - include_str!("../fixtures/controller_create_target_params.json") - } - "controller_create_target_response" => { - include_str!("../fixtures/controller_create_target_response.json") - } - "controller_list_targets_response" => { - include_str!("../fixtures/controller_list_targets_response.json") - } - other => panic!("unknown fixture {other}"), - }) - .expect("fixture JSON") +fn fixture(path: &str) -> Value { + let full = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join(path); + serde_json::from_str(&std::fs::read_to_string(&full).expect("fixture")).expect("JSON") } -fn assert_round_trip(value: T, expected: Value) -where - T: Serialize + DeserializeOwned + PartialEq + std::fmt::Debug, -{ - let encoded = serde_json::to_value(&value).expect("serialize"); - assert_eq!(encoded, expected); - let decoded: T = serde_json::from_value(encoded).expect("deserialize"); - assert_eq!(decoded, value); -} - -#[test] -fn single_file_transfer_has_distinct_wire_semantics() { - use environment_protocol::{ - data::transfer_session::TransferSelection, shared::EnvironmentPath, - }; - assert_round_trip( - TransferSelection::WriteFile { - destination: EnvironmentPath::new("/workspace/output.bin").unwrap(), - }, - json!({"direction":"writeFile","destination":"/workspace/output.bin"}), - ); - let existing: TransferSelection = - serde_json::from_value(json!({"direction":"materialize","destination":"/workspace/tree"})) - .unwrap(); - assert_eq!( - serde_json::to_value(existing).unwrap(), - json!({"direction":"materialize","destination":"/workspace/tree","onExisting":"replace"}) - ); +fn decode(value: &Value) -> T { + serde_json::from_value(value.clone()).expect("decode") } #[test] -fn method_names_match_data_plane_contract() { - assert_eq!(INITIALIZE_METHOD, "initialize"); - assert_eq!(FS_READ_FILE_METHOD, "fs/readFile"); - assert_eq!(PROCESS_START_METHOD, "process/start"); - assert_eq!(JOB_START_METHOD, "job/start"); - assert_eq!(JOB_LIST_METHOD, "job/list"); - assert_eq!(JOB_READ_METHOD, "job/read"); - assert_eq!(JOB_CANCEL_METHOD, "job/cancel"); - assert_eq!(PROCESS_OUTPUT_METHOD, "process/output"); +fn both_numbers_are_four() { + assert_eq!(ENVD_PROTOCOL_VERSION, 4); + assert_eq!(CONTROLLER_PROTOCOL_VERSION, 4); } #[test] -fn method_names_match_controller_plane_contract() { - assert_eq!(CONTROL_INITIALIZE_METHOD, "controller/initialize"); - assert_eq!(LIST_TARGETS_METHOD, "controller/listTargets"); - assert_eq!(CREATE_TARGET_METHOD, "controller/createTarget"); -} - -#[test] -fn initialize_params_match_fixture() { - assert_round_trip( - InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-test".to_owned(), - scope: EnvironmentScope::Session { - session_id: "sandbox-123".to_owned(), - }, - resume_connection_id: Some(EnvironmentConnectionId::new("conn-prev")), - }, - fixture("data_initialize_params"), +fn handshakes_require_a_stated_protocol_version() { + let client_info = json!({"name": "x"}); + assert!( + serde_json::from_value::(json!({"clientInfo": client_info})).is_err() ); -} - -#[test] -fn read_file_response_uses_base64_byte_chunk() { - assert_round_trip( - ReadFileResponse { - data: ByteChunk::from(b"hello\n".as_slice()), - file_size: None, - truncated: false, - }, - fixture("fs_read_file_response"), + assert!( + serde_json::from_value::(json!({"clientInfo": client_info})) + .is_err() ); + let mut register = fixture("registration/registration.register.json")["params"].clone(); + register.as_object_mut().unwrap().remove("protocolVersion"); + assert!(serde_json::from_value::(register).is_err()); } #[test] -fn process_start_params_match_fixture() { - assert_round_trip( - StartProcessParams { - process_id: ProcessId::new("proc-1"), - argv: vec!["sh".to_owned(), "-lc".to_owned(), "cat".to_owned()], - cwd: Some("/workspace".try_into().expect("cwd")), - env: BTreeMap::from([("RUST_LOG".to_owned(), "debug".to_owned())]), - secret_env: BTreeMap::new(), - stdin: Some(ByteChunk::from(b"input\n".as_slice())), - timeout_ms: Some(60_000), - tty: false, - }, - fixture("process_start_params"), - ); +fn initialize_names_the_plane_and_the_daemon_instance() { + let response: InitializeResponse = decode(&fixture("data/initialize.json")["response"]); + assert_eq!(response.plane, Plane::Data); + assert_eq!(response.daemon_instance_id.as_str(), "envd_01jb8x7k2m"); + assert!(response.capabilities.filesystem.unwrap().write); + assert!(response.limits.max_concurrent_requests >= 16); + let absent: DataCapabilities = decode(&json!({})); + assert!(absent.process.is_none(), "an absent family is unavailable"); } #[test] -fn process_read_response_preserves_ordered_output_chunks() { - assert_round_trip( - ReadProcessResponse { - chunks: vec![ - ProcessOutputChunk { - seq: 1, - stream: ProcessOutputStream::Stdout, - chunk: ByteChunk::from(b"ok\n".as_slice()), - }, - ProcessOutputChunk { - seq: 2, - stream: ProcessOutputStream::Stderr, - chunk: ByteChunk::from(b"warn\n".as_slice()), - }, - ], - next_seq: 3, - exited: true, - exit_code: Some(0), - closed: true, - failure: None, - termination: None, - pid: Some(90), - omitted_bytes: 0, - leftover_processes: vec![LeftoverProcess { - pid: 91, - command: "python -m http.server 8080".to_owned(), - }], - }, - fixture("process_read_response"), - ); -} +fn process_reads_require_bounded_waits_and_report_byte_gaps() { + let mut params = fixture("data/process.read.json")["params"].clone(); + params.as_object_mut().unwrap().remove("waitMs"); + assert!(serde_json::from_value::(params).is_err()); -#[test] -fn job_start_params_match_fixture() { - assert_round_trip( - StartJobsParams { - namespace: "session_1".to_owned(), - request_id: "request-1".to_owned(), - jobs: vec![JobStartSpec { - job_id: JobId::new("checkout"), - name: Some("checkout".to_owned()), - argv: vec!["git".to_owned(), "status".to_owned()], - cwd: Some("/workspace".try_into().expect("cwd")), - env: BTreeMap::from([("RUST_LOG".to_owned(), "debug".to_owned())]), - secret_env: BTreeMap::new(), - stdin: Some(ByteChunk::from(b"hello\n".as_slice())), - timeout_ms: Some(60_000), - depends_on: vec![JobDependency::name("setup")], - dependency_policy: JobDependencyPolicy::AllSucceeded, - queue_key: Some("repo".to_owned()), - }], - }, - fixture("job_start_params"), - ); -} - -#[test] -fn job_read_response_matches_fixture() { - assert_round_trip( - ReadJobsResponse { - jobs: vec![JobReadResult { - summary: JobSummary { - namespace: "session_1".to_owned(), - job_id: JobId::new("tests"), - name: Some("tests".to_owned()), - status: JobStatus::Succeeded, - dependencies: vec![JobId::new("checkout")], - created_at_ms: 1, - queued_at_ms: Some(2), - started_at_ms: Some(3), - finished_at_ms: Some(4), - exit_code: Some(0), - orphaned_descendants: true, - failure: None, - queue_key: Some("repo".to_owned()), - }, - output_chunks: vec![JobOutputChunk { - seq: 1, - stream: JobOutputStream::Stdout, - chunk: ByteChunk::from(b"ok\n".as_slice()), - }], - output_next_seq: 2, - artifacts: vec![JobArtifact { - path: "/workspace/result.txt".try_into().expect("artifact path"), - kind: Some("file".to_owned()), - metadata: BTreeMap::from([("role".to_owned(), "report".to_owned())]), - }], - }], - }, - fixture("job_read_response"), - ); + let response: ReadProcessResponse = decode(&fixture("data/process.read.json")["response"]); + let OutputPage { + chunks, + gaps, + next, + end, + } = response.output; + let gap = gaps[0]; + assert_eq!(gap.reason, GapReason::Overflow); + assert_eq!(chunks[0].offset + chunks[0].data.len() as u64, gap.offset); + assert_eq!(gap.offset + gap.omitted_bytes, chunks[1].offset); + assert_eq!(next, end); + assert!(matches!( + response.exit, + Some(ExitStatus::Signal { + core_dumped: true, + .. + }) + )); } #[test] -fn job_list_params_use_session_namespace_and_limit() { - assert_round_trip( - ListJobsParams { - namespace: "session_1".to_owned(), - limit: Some(25), - }, - json!({ - "namespace": "session_1", - "limit": 25 - }), +fn job_entries_report_unknown_ids_as_errors() { + let response: ReadJobsResponse = decode(&fixture("data/job.read.json")["response"]); + let JobReadEntry::Error { job_id, error } = &response.jobs[2] else { + panic!("expected an error entry"); + }; + assert_eq!(job_id.as_str(), "gone"); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + let JobReadEntry::Job { summary, .. } = &response.jobs[0] else { + panic!("expected a job entry"); + }; + assert_eq!(summary.status, JobStatus::Failed); + assert!(summary.status.is_terminal()); + assert!(!JobStatus::Queued.is_terminal()); +} + +#[test] +fn job_dependencies_are_kind_tagged() { + let params: StartJobsParams = decode(&fixture("data/job.start.json")["params"]); + assert!(matches!( + ¶ms.jobs[1].depends_on[0], + JobDependency::Name { name } if name.as_str() == "setup" + )); + assert!( + serde_json::from_value::( + json!({"kind": "jobId", "jobId": "a", "name": "b"}) + ) + .is_err() + ); + let mut job = fixture("data/job.start.json")["params"].clone(); + job["jobs"][0]["namespace"] = json!("shared"); + assert!(serde_json::from_value::(job).is_err()); + let pipe = json!({"argv": ["cat"], "input": {"kind": "pipe"}}); + let spec: environment_protocol::data::exec::CommandSpec = decode(&pipe); + assert_eq!(spec.input, ExecInput::Pipe {}); +} + +#[test] +fn scan_roots_are_reported_independently() { + let response: ScanResponse = decode(&fixture("data/fs.scan.json")["response"]); + assert_eq!(response.roots.len(), 3); + assert!(matches!( + response.roots[1], + ScanRootResult::Unchanged { .. } + )); + let ScanRootResult::Complete { skipped, .. } = &response.roots[2] else { + panic!("expected a complete root"); + }; + assert!( + skipped[1].path.is_none(), + "non-UTF-8 names are counted, not returned" ); + assert_eq!(FsScan::PLANE, Plane::Data); } #[test] -fn byte_chunk_rejects_invalid_base64() { - assert!(serde_json::from_value::(json!("not base64 !!!")).is_err()); +fn close_by_owner_answers_closed_without_a_target_id() { + let response: TargetResponse = decode(&fixture("controller/target.close.json")["response"]); + let TargetObservation { + target_id, + phase, + power, + .. + } = response.target; + assert!(target_id.is_none()); + assert_eq!(phase, TargetPhase::Closed); + assert!(power.is_none()); } #[test] -fn controller_initialize_params_match_fixture() { - assert_round_trip( - ControllerInitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-test".to_owned(), - }, - fixture("controller_initialize_params"), +fn owners_name_only_the_universe_and_environment() { + let mut params = fixture("controller/target.get.json")["params"].clone(); + params["target"]["owner"]["bindingId"] = json!("primary"); + assert!( + serde_json::from_value::(params) + .is_err() ); + let mut observation = fixture("controller/target.get.json")["response"].clone(); + observation["target"]["daemonReady"] = json!(true); + assert!(serde_json::from_value::(observation).is_err()); } #[test] -fn create_target_params_match_provider_fixture() { - assert_round_trip( - CreateTargetParams { - request_id: "request-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-1".to_owned(), - binding_id: "primary".to_owned(), - }, - template_id: "lightspeed-dev-v1".to_owned(), - }, - fixture("controller_create_target_params"), - ); +fn templates_carry_power_states_and_a_default_idle_policy() { + let response: ListTemplatesResponse = + decode(&fixture("controller/template.list.json")["response"]); + let template = &response.templates[0]; + assert!(template.power_states.contains(&PowerState::Paused)); + let idle = template.idle.as_ref().expect("default idle policy"); + assert_eq!(idle.validate(), Ok(())); + let create: CreateTargetParams = decode(&fixture("controller/target.create.json")["params"]); + assert_eq!(create.idle.expect("override").pause_after_ms, Some(600_000)); } #[test] -fn create_target_response_carries_only_provider_lifecycle_state() { - assert_round_trip( - CreateTargetResponse { - target: ready_target_summary(), - }, - fixture("controller_create_target_response"), - ); +fn controller_capabilities_name_power_and_ingress_only() { + let response: ControllerInitializeResponse = + decode(&fixture("controller/initialize.json")["response"]); + assert!(response.capabilities.power && response.capabilities.ingress); + let mut routed = fixture("controller/initialize.json")["response"].clone(); + routed["routes"] = json!({"data": "wss://p.example/routes"}); + assert!(serde_json::from_value::(routed).is_err()); } #[test] -fn list_targets_response_matches_fixture() { - assert_round_trip( - ListTargetsResponse { - targets: vec![ready_target_summary()], - }, - fixture("controller_list_targets_response"), - ); -} - -fn ready_target_summary() -> ProviderTargetSummary { - ProviderTargetSummary { - target_id: ProviderTargetId::new("sandbox-123"), - display_name: Some("lightspeed dev".to_owned()), - status: ProviderTargetStatus::Ready, - scope: EnvironmentScope::Session { - session_id: "sandbox-123".to_owned(), - }, - capabilities: remote_environment_capabilities(), - default_cwd: Some("/workspace".try_into().expect("cwd")), - power_states: Vec::new(), - metadata: BTreeMap::from([("provider".to_owned(), "smolvm".to_owned())]), - } +fn the_receipt_is_nested_and_typed() { + let response: RegisterResponse = + decode(&fixture("registration/registration.register.json")["response"]); + assert_eq!(response.receipt.daemon_id.as_str(), "daemon_3b3b"); + let mut flattened = fixture("registration/registration.register.json")["response"].clone(); + flattened["environmentId"] = json!("env_1"); + assert!(serde_json::from_value::(flattened).is_err()); } #[test] -fn power_vocabulary_round_trips_and_maps_steady_states() { - assert_eq!(SET_TARGET_POWER_METHOD, "controller/setTargetPower"); - assert_eq!(ENV_IDLE_METHOD, "env/idle"); - let params = SetTargetPowerParams { - request_id: "power-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-1".to_owned(), - binding_id: "binding-1".to_owned(), - }, - target_id: ProviderTargetId::new("sandbox-123"), - power: PowerState::Suspended, - }; - assert_round_trip( - params, - json!({ - "requestId": "power-1", - "environmentId": "environment-1", - "incarnationId": "incarnation-1", - "binding": {"universeId": "universe-1", "bindingId": "binding-1"}, - "targetId": "sandbox-123", - "power": "suspended" - }), - ); - let mut summary = ready_target_summary(); - summary.status = ProviderTargetStatus::Paused; - summary.power_states = vec![PowerState::Running, PowerState::Paused]; - let encoded = serde_json::to_value(&summary).expect("serialize"); - assert_eq!(encoded["status"], json!("paused")); - assert_eq!(encoded["powerStates"], json!(["running", "paused"])); - // A summary from a provider that predates power control decodes with an - // empty state list. - let legacy: ProviderTargetSummary = - serde_json::from_value(serde_json::to_value(ready_target_summary()).unwrap()).unwrap(); - assert!(legacy.power_states.is_empty()); - assert_eq!( - ProviderTargetStatus::Suspended.power_state(), - Some(PowerState::Suspended) - ); - assert_eq!(ProviderTargetStatus::Starting.power_state(), None); - assert_round_trip( - IdleResponse { - idle_for_ms: 1_500, - running_processes: 0, - running_jobs: 2, - leftover_process_groups: 0, - }, - json!({"idleForMs": 1500, "runningProcesses": 0, "runningJobs": 2}), - ); -} - -fn remote_environment_capabilities() -> EnvironmentCapabilities { - EnvironmentCapabilities { - filesystem_read: true, - filesystem_write: true, - filesystem_search: true, - filesystem_glob: true, - filesystem_ranged_read: true, - filesystem_capture: false, - filesystem_transfer: false, - filesystem_scan: false, - filesystem_materialize: false, - process_start: true, - process_stdin: true, - process_terminate: true, - process_output_polling: true, - process_output_notifications: true, - process_pty: false, - job_start: true, - job_list: true, - job_read: true, - job_cancel: true, - job_wait_hint: false, - job_dependencies: true, - job_queue_keys: true, - network: true, - } +fn errors_carry_typed_data() { + let error: ProtocolError = decode(&fixture("data/process.read.json")["error"]); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + assert!(matches!( + error.data, + environment_protocol::error::ErrorData::NotFound { + reason: Some(NotFoundReason::Expired) + } + )); + assert!(!error.is_retryable()); + assert_eq!(ProcessRead::NAME, "process/read"); } diff --git a/crates/environment-provider-incus/CLUSTER.md b/crates/environment-provider-incus/CLUSTER.md index 99e2d752d..81bd0ec7b 100644 --- a/crates/environment-provider-incus/CLUSTER.md +++ b/crates/environment-provider-incus/CLUSTER.md @@ -49,7 +49,10 @@ incus cluster unset member-a scheduler.instance ## Verification -The non-destructive live topology/reconciliation test is opt-in: +The live tests are opt-in. The topology and inventory test is +non-destructive; every other test in `tests/incus_live.rs` creates and +deletes one VM in the universe `lightspeed-conformance`, covering controller +and data conformance, waking on dial, and the idle loop: ```bash LIGHTSPEED_INCUS_PROVIDER_CONFIG=/absolute/path/provider.json \ @@ -57,7 +60,7 @@ LIGHTSPEED_INCUS_PROVIDER_CONFIG=/absolute/path/provider.json \ ``` Before production rollout, also provision an expendable environment, verify -its `incusMember` target metadata, disable the serving API endpoint, exercise -filesystem/process/PTY/job and public-ingress traffic through another API -endpoint, and close the environment. Repeat with a `clusterGroup`-constrained -template and a cordoned member. +its placement with `incus list` (the `LOCATION` column), disable the serving +API endpoint, exercise filesystem/process/PTY/job and public-ingress traffic +through another API endpoint, and close the environment. Repeat with a +`clusterGroup`-constrained template and a cordoned member. diff --git a/crates/environment-provider-incus/Cargo.toml b/crates/environment-provider-incus/Cargo.toml index 6ac582bff..db0ce284f 100644 --- a/crates/environment-provider-incus/Cargo.toml +++ b/crates/environment-provider-incus/Cargo.toml @@ -19,10 +19,15 @@ sha2 = "0.11" thiserror = "2" tokio = { version = "1", features = ["fs", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "handshake", "rustls-tls-webpki-roots"] } -uuid = { version = "1", features = ["v4"] } [dev-dependencies] +# The controller conformance suite runs against this server in unit tests +# (with an in-memory backend) and against live Incus. +environment-client = { path = "../environment-client", features = ["conformance"] } tempfile = "3" +# The data conformance suite runs against a guest envd on live Incus. +tools = { path = "../tools" } +uuid = { version = "1", features = ["v4"] } [[bin]] name = "lightspeed-provider-incus" diff --git a/crates/environment-provider-incus/README.md b/crates/environment-provider-incus/README.md index 1f48afeea..3a357f8a9 100644 --- a/crates/environment-provider-incus/README.md +++ b/crates/environment-provider-incus/README.md @@ -1,16 +1,25 @@ # Incus environment provider -`lightspeed-provider-incus` is a stateless controller, passive data endpoint, -optional application edge, and single-node/cluster Incus adapter. It depends -on the public host -protocol crate, not Lightspeed's database, API implementation, harness, or -Temporal runtime. - -Start from `config.example.json`. Application-level authentication for both -the provider endpoint and private envd endpoint is intentionally deferred. -Protect those listeners with the deployment network/transport boundary for -now. The Incus client certificate and key remain deployment files mounted into -the provider process. +`lightspeed-provider-incus` is a stateless controller, data-route relay that +wakes powered-down targets, idle power manager, optional application edge, +and single-node/cluster Incus adapter. It serves the controller plane of the +environment protocol and depends on the public protocol crate, not +Lightspeed's database, API implementation, harness, or Temporal runtime. + +Start from `config.example.json`. The listener serves the controller on +`/control` and the data routes under +`/routes/{universeId}/{environmentId}/{targetId}`. With `credentialFile` +set, both require the provider credential: the file holds a secret of at +least 32 bytes without whitespace (a trailing newline is ignored), which +Lightspeed presents as `Authorization: Bearer` and the provider compares in +constant time. A missing or wrong credential is refused with HTTP 401 before +the WebSocket upgrade. Register the same credential with the provider in +Lightspeed. Without `credentialFile` the listener checks no credential and +Lightspeed sends none; only run it that way on a network where nothing but +Lightspeed can reach the listener. `GET /health` needs no credential. The +guest envd listener has no authentication of its own; keep guest networks +private. The Incus client certificate and key remain deployment files +mounted into the provider process. ## Incus topology @@ -31,7 +40,7 @@ The provider does not form the cluster, join or remove members, initialize member-local storage/uplinks, evacuate instances, or recover quorum. Perform those operations with deployment tooling and the Incus CLI. -Cluster mode uses one provider-managed OVN network per binding over the +Cluster mode uses one provider-managed OVN network per universe over the existing `incus.clusterNetworkUplink`. The provider and optional public edge must be able to route to addresses on those networks. Standalone mode retains provider-managed bridge networks. @@ -74,21 +83,30 @@ cargo run -p environment-provider-incus -- \ --config crates/environment-provider-incus/config.example.json ``` -Register its controller as `ws://127.0.0.1:19090/control`. Lightspeed derives -the sibling `/routes/...` endpoint and connects only while an environment is -being used; the provider then dials private envd and reaps idle connections. -Registering the provider and creating an enabled binding in Lightspeed is -sufficient to use it. Provider configuration contains no universe or binding -allowlist. Every request carries a binding context, and the provider lazily -creates a deterministic restricted Incus project, managed network, ACL, and -profile for that ownership namespace. Incus chooses and persists an unused -subnet, so adding a universe does not require editing or restarting the -provider. +Register its controller as `ws://127.0.0.1:19090/control`, with the credential +when one is configured. Lightspeed derives the data route base from the +controller URL (`ws://127.0.0.1:19090/routes`), which is where this listener +serves its data routes. When the environment gateway reaches the listener +under another address, set the data route base on the provider record in +Lightspeed instead; it must use `wss://` when Lightspeed reaches the +controller over TLS. Lightspeed connects to a data route only while an +environment is being used; the provider then dials private envd and relays +frames unchanged. The relay forwards WebSocket pings, counts every frame as +activity, and closes a connection only after `relayIdleSeconds` without +frames, which cannot be set below 60. + +Registering the provider and enabling it for a universe in Lightspeed is +sufficient to use it. Provider configuration contains no universe allowlist. +Every request carries its universe, and the provider lazily creates a +deterministic restricted Incus project, managed network, ACL, and profile +for that universe; every environment of the universe lives in it. Incus +chooses and persists an unused subnet, so adding a universe does not +require editing or restarting the provider. The provider configuration owns only provider-wide offerings and physical policy: templates, blocked egress destinations, Incus topology, storage, and optional ingress. A blocked CIDR must not overlap a subnet Incus assigns to a -binding network; the provider rejects that conflict during lazy reconciliation. +universe network; the provider rejects that conflict during lazy reconciliation. The provider also derives sibling-network ACL rules from live Incus inventory. Lightspeed owns whether a universe may route to this provider. Per-universe commercial quotas or specialized offerings would need a future provider @@ -108,3 +126,110 @@ bodies, and WebSockets to the approved private guest port. The provider records the realized hostname and port in `user.lightspeed.*` Incus target metadata so route operations remain ownership-fenced and reconstructible without a provider database or configured universe list. + +## Controller behavior + +Every controller method answers within about five seconds with the target's +current observation. Slow Incus work (an image pull, a boot, a graceful stop) +runs as one background job per target, serialized per target; later calls +report `provisioning` or `closing` while it runs, and a failed create reports +`failed` with the Incus error as the target's `message`. Job state is in +memory; Incus itself, read through the deterministic owner name, is the record +of every target, so a provider restart loses nothing but the idle clock of +powered-down targets (see below). + +- A target belongs to `{universeId, environmentId}`, and its instance name, + which is its `targetId`, is derived from that owner. `target/create` looks + up the owner's instance before applying any creation policy, so + deprecating a template never affects existing targets. +- `target/get` only reads. A missing instance is `notFound`. Whether envd + answers is not part of an observation: Lightspeed proves it by dialing the + data route. +- `target/close` resolves the instance by owner when no target id is given, + stops it gracefully for up to 30 seconds, then forces the stop and deletes + the instance, its disk, and its ingress metadata. +- `target/setPower` moves a target to `running`, `paused` (an Incus freeze) + or `stopped`; Incus targets cannot be `suspended`. Lightspeed calls it only + for a manual power change, and the provider stays the owner of power + afterwards. +- Templates report typed `resources`, their `powerStates`, and their default + `idle` policy. `memory` and `disk` must be whole Incus sizes such as + `4GiB`; the provider converts them to bytes at startup. +- Cloud-init sets `LIGHTSPEED_ENVD_STATE_DIR=/var/lib/lightspeed-envd`, so the + daemon key and job records live outside the agent's `/workspace`. +- Incus API failures answer `unavailable`, which the runtime retries. +- The provider advertises `power`, and `ingress` when an `ingress` block is + configured. + +## Waking on dial + +A dial of a paused or stopped target's data route wakes it: the provider +unfreezes or starts the instance and then tries envd until it answers. The +dial is held for at most five seconds, and a dial that arrives while a pause +or stop is still running waits for that power-down and the wake after it +before it tries envd. When envd answers in time the +connection upgrades and relays as usual; otherwise the provider refuses with +a typed JSON body and lets Lightspeed retry: + +| Situation | Answer | +| --- | --- | +| The target was powered down and is waking | 503 `{"state":"waking"}`, `Retry-After: 2` | +| The target runs but envd does not answer yet | 503 `{"state":"booting"}`, `Retry-After: 2` | +| No instance exists for the owner, or it is being closed | 410 `{"state":"gone"}` | +| Incus reports the instance in its `Error` state | 409 `{"state":"failed","message":…}` | + +An Incus API failure during the dial answers 502 without a body. + +## Idle policy + +The provider powers idle targets down on its own clock. A template may set a +default `idle` policy, and Lightspeed may pass another one on +`target/create`; the target keeps the policy it was created with, recorded +as the `user.lightspeed.idle` instance key. A policy names thresholds in +milliseconds, non-decreasing in the order `pauseAfterMs`, `stopAfterMs`, +`closeAfterMs`; `suspendAfterMs` is not accepted in a template here and +never fires, because Incus targets cannot be suspended. + +Every `idleCheckSeconds` (default 60) the provider visits each target with a +policy: + +- A running target's idle time is envd's `env/getIdle` report, read over + the guest network. Running processes, jobs and processes a command left + running hold it at zero, so they keep the target awake. +- A paused or stopped target has no envd to ask. The provider continues + from the idle time it measured when it powered the target down, kept in + memory; after a provider restart, or for a target someone else powered + down, it starts from zero. +- A target with a data connection in progress, or with a dial waking it, is + never powered down or closed. +- A target a dial woke is left alone for a grace of `idleCheckSeconds`, at + least 30 seconds, after that dial's connection ends. Unpausing does not + reset envd's idle time, so the grace keeps the next pass from pausing the + target again before the caller's first call arrives. + +The most escalated threshold the idle time has reached is applied when it +goes beyond the target's current power: pause, then stop, then close. An +idle close deletes the instance like `target/close`; Lightspeed then sees the +target gone and records the environment as ended by the provider. The next +data route dial wakes a paused or stopped target. + +## Conformance + +The ignored live tests in `tests/incus_live.rs` run against a live Incus +server or cluster, each creating and deleting one VM from the first +non-deprecated template in the fixed universe `lightspeed-conformance`: + +- `the_controller_conforms_on_live_incus` runs the environment client's + controller conformance suite; +- `the_guest_envd_conforms_on_live_incus` runs the data conformance suite + against the guest's envd through the provider's data route; +- `a_paused_target_wakes_on_dial_on_live_incus` pauses a target and dials it + awake; +- `the_idle_loop_pauses_an_idle_target_on_live_incus` creates a target with + a short idle override and waits for the idle loop to pause it. + +```bash +LIGHTSPEED_INCUS_PROVIDER_CONFIG=/absolute/path/provider.json \ + cargo test -p environment-provider-incus --test incus_live \ + the_controller_conforms_on_live_incus -- --ignored +``` diff --git a/crates/environment-provider-incus/config.cluster.example.json b/crates/environment-provider-incus/config.cluster.example.json index e0755c1ce..a22eb87d1 100644 --- a/crates/environment-provider-incus/config.cluster.example.json +++ b/crates/environment-provider-incus/config.cluster.example.json @@ -1,5 +1,6 @@ { "controllerListen": "127.0.0.1:19090", + "credentialFile": "/run/secrets/lightspeed-provider-credential", "incus": { "mode": "cluster", "endpoints": [ @@ -16,6 +17,7 @@ }, "relayIdleSeconds": 60, "dialTimeoutSeconds": 10, + "idleCheckSeconds": 60, "envdPort": 19091, "ingress": { "publicBaseUrl": "https://env.example.com", @@ -35,7 +37,11 @@ "disk": "40GiB", "clusterGroup": "x86-production", "publicIngress": true, - "ingressPort": 8080 + "ingressPort": 8080, + "idle": { + "pauseAfterMs": 1800000, + "stopAfterMs": 14400000 + } } ] } diff --git a/crates/environment-provider-incus/config.example.json b/crates/environment-provider-incus/config.example.json index 5fd78e2d0..e451f619a 100644 --- a/crates/environment-provider-incus/config.example.json +++ b/crates/environment-provider-incus/config.example.json @@ -1,5 +1,6 @@ { "controllerListen": "127.0.0.1:19090", + "credentialFile": "/run/secrets/lightspeed-provider-credential", "incus": { "mode": "single", "endpoints": ["https://hz02.example.ts.net:8443"], @@ -11,6 +12,7 @@ }, "relayIdleSeconds": 60, "dialTimeoutSeconds": 10, + "idleCheckSeconds": 60, "envdPort": 19091, "ingress": { "publicBaseUrl": "https://env.example.com", @@ -29,7 +31,11 @@ "memory": "4GiB", "disk": "40GiB", "publicIngress": true, - "ingressPort": 8080 + "ingressPort": 8080, + "idle": { + "pauseAfterMs": 1800000, + "stopAfterMs": 14400000 + } } ] } diff --git a/crates/environment-provider-incus/image/lightspeed-envd.service b/crates/environment-provider-incus/image/lightspeed-envd.service index 2527417f4..f7449e9c9 100644 --- a/crates/environment-provider-incus/image/lightspeed-envd.service +++ b/crates/environment-provider-incus/image/lightspeed-envd.service @@ -11,6 +11,9 @@ EnvironmentFile=/etc/lightspeed-envd/provider.env ExecStart=/usr/local/bin/lightspeed-envd Restart=always RestartSec=1 +# envd exits 3 (registration refused), 4 (environment closed) or 5 +# (superseded) when a restart cannot help. +RestartPreventExitStatus=3 4 5 NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict diff --git a/crates/environment-provider-incus/src/config.rs b/crates/environment-provider-incus/src/config.rs index b07e40ef2..f338b2a4d 100644 --- a/crates/environment-provider-incus/src/config.rs +++ b/crates/environment-provider-incus/src/config.rs @@ -2,7 +2,24 @@ use std::{collections::BTreeMap, net::SocketAddr, path::PathBuf, sync::Arc}; use anyhow::{Context as _, bail}; use clap::Parser; +use environment_protocol::{ + bytes::SecretString, + control::template::TemplateResources, + idle::IdlePolicy, + ids::TemplateId, + limits::MIN_INTERMEDIARY_IDLE_MS, + metadata::{DescriptiveText, DisplayName}, +}; use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; + +use crate::incus::unsupported_idle_stage; + +/// Shortest provider credential accepted, in bytes. +const MIN_CREDENTIAL_BYTES: usize = 32; + +/// Default pause between two passes of the idle loop, in seconds. +const DEFAULT_IDLE_CHECK_SECONDS: u64 = 60; #[derive(Parser, Debug)] #[command(name = "lightspeed-provider-incus", version = release_info::LONG_VERSION)] @@ -15,27 +32,36 @@ pub struct ProviderArgs { #[serde(rename_all = "camelCase", deny_unknown_fields)] struct RawConfig { controller_listen: SocketAddr, + /// File holding the provider credential Lightspeed sends as + /// `Authorization: Bearer` on `/control` and every data route. Without + /// one the listener checks no credential. + credential_file: Option, incus: IncusConfig, templates: Vec, #[serde(default)] network: NetworkPolicy, relay_idle_seconds: Option, dial_timeout_seconds: Option, + idle_check_seconds: Option, envd_port: Option, ingress: Option, } #[derive(Clone, Debug)] -pub struct Config(Arc); +pub struct Config(pub(crate) Arc); #[derive(Debug)] pub struct ConfigInner { pub controller_listen: SocketAddr, + /// `None`: requests are not authenticated. + pub credential: Option, pub incus: IncusConfig, - pub templates: BTreeMap, + pub templates: BTreeMap, pub network: NetworkPolicy, pub relay_idle_seconds: u64, pub dial_timeout_seconds: u64, + /// Pause between two passes of the idle loop. + pub idle_check_seconds: u64, pub envd_port: u16, pub ingress: Option, } @@ -50,7 +76,7 @@ pub struct IncusConfig { pub server_ca_pem: PathBuf, pub storage_pool: String, /// Existing physical/bridge uplink used when the provider creates its - /// per-binding OVN networks in cluster mode. + /// per-universe OVN networks in cluster mode. pub cluster_network_uplink: Option, /// Trusted Incus image server used to lazily cache missing immutable /// fingerprints on the target node. @@ -74,12 +100,14 @@ pub struct NetworkPolicy { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct TemplatePolicy { - pub template_id: String, - pub display_name: String, - pub description: Option, + pub template_id: TemplateId, + pub display_name: DisplayName, + pub description: Option, pub image_fingerprint: String, pub cpu: u32, + /// Incus size string, such as `4GiB`. pub memory: String, + /// Incus size string, such as `40GiB`. pub disk: String, /// Optional native Incus cluster group used as the hard placement set. #[serde(default)] @@ -90,6 +118,13 @@ pub struct TemplatePolicy { pub ingress_port: Option, #[serde(default)] pub deprecated: bool, + /// Idle policy of targets whose `target/create` names none. Only stages + /// this provider can carry out (pause, stop, close) are accepted. + #[serde(default)] + pub idle: Option, + /// `cpu`, `memory` and `disk` in provider-neutral units, derived at load. + #[serde(skip)] + pub resources: TemplateResources, } #[derive(Clone, Debug, Deserialize)] @@ -114,8 +149,26 @@ impl ProviderArgs { if raw.templates.is_empty() { bail!("provider must configure at least one template") } + let envd_port = raw.envd_port.unwrap_or(19091); + let relay_idle_seconds = raw + .relay_idle_seconds + .unwrap_or(MIN_INTERMEDIARY_IDLE_MS / 1000); + if relay_idle_seconds * 1000 < MIN_INTERMEDIARY_IDLE_MS { + bail!( + "relayIdleSeconds must be at least {}: relays never idle-close a data connection sooner", + MIN_INTERMEDIARY_IDLE_MS / 1000 + ) + } + let idle_check_seconds = raw.idle_check_seconds.unwrap_or(DEFAULT_IDLE_CHECK_SECONDS); + if idle_check_seconds == 0 { + bail!("idleCheckSeconds must be positive") + } + let credential = match &raw.credential_file { + Some(path) => Some(ProviderCredential::load(path).await?), + None => None, + }; let mut templates = BTreeMap::new(); - for template in raw.templates { + for mut template in raw.templates { if template.cluster_group.is_some() && raw.incus.mode != IncusMode::Cluster { bail!( "template {} selects a clusterGroup in single mode", @@ -132,11 +185,12 @@ impl ProviderArgs { template.template_id ) } + if let Some(idle) = &template.idle { + validate_template_idle(idle) + .with_context(|| format!("template {} idle", template.template_id))?; + } if let Some(port) = template.ingress_port { - if port == 0 - || port == raw.envd_port.unwrap_or(19091) - || matches!(port, 22 | 2375 | 2376) - { + if port == 0 || port == envd_port || matches!(port, 22 | 2375 | 2376) { bail!( "template {} uses a reserved management port for ingress", template.template_id @@ -149,6 +203,17 @@ impl ProviderArgs { ) } } + template.resources = TemplateResources { + vcpus: Some(template.cpu), + memory_bytes: Some( + parse_incus_size(&template.memory) + .with_context(|| format!("template {} memory", template.template_id))?, + ), + disk_bytes: Some( + parse_incus_size(&template.disk) + .with_context(|| format!("template {} disk", template.template_id))?, + ), + }; let template_id = template.template_id.clone(); if templates.insert(template_id.clone(), template).is_some() { bail!("duplicate provider template: {template_id}") @@ -170,12 +235,14 @@ impl ProviderArgs { } Ok(Config(Arc::new(ConfigInner { controller_listen: raw.controller_listen, + credential, incus: raw.incus, templates, network: raw.network, - relay_idle_seconds: raw.relay_idle_seconds.unwrap_or(60).max(1), + relay_idle_seconds, dial_timeout_seconds: raw.dial_timeout_seconds.unwrap_or(10).max(1), - envd_port: raw.envd_port.unwrap_or(19091), + idle_check_seconds, + envd_port, ingress: raw.ingress, }))) } @@ -184,6 +251,102 @@ impl ProviderArgs { } } +/// The provider credential: the bearer token Lightspeed presents on +/// `/control` and on every data route. `Debug` is redacted. +#[derive(Clone)] +pub struct ProviderCredential(SecretString); + +impl ProviderCredential { + pub fn new(value: impl Into) -> anyhow::Result { + let value = value.into(); + if value.len() < MIN_CREDENTIAL_BYTES + || value + .chars() + .any(|character| character.is_whitespace() || character.is_control()) + { + bail!( + "the provider credential must be at least {MIN_CREDENTIAL_BYTES} bytes without whitespace" + ) + } + Ok(Self(SecretString::new(value))) + } + + async fn load(path: &std::path::Path) -> anyhow::Result { + let contents = tokio::fs::read_to_string(path) + .await + .with_context(|| format!("read credentialFile {}", path.display()))?; + Self::new(contents.trim_end_matches(['\r', '\n'])) + .with_context(|| format!("credentialFile {}", path.display())) + } + + pub fn expose(&self) -> &str { + self.0.expose() + } + + /// True when `presented` is this credential. Compares fixed-size digests + /// so the time taken reveals neither the credential's length nor how + /// much of it matched. + pub fn matches(&self, presented: &str) -> bool { + let expected = Sha256::digest(self.0.expose().as_bytes()); + let presented = Sha256::digest(presented.as_bytes()); + expected + .iter() + .zip(presented.iter()) + .fold(0u8, |difference, (left, right)| difference | (left ^ right)) + == 0 + } +} + +impl std::fmt::Debug for ProviderCredential { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("ProviderCredential()") + } +} + +/// A template's default idle policy: valid, and every power stage it sets +/// one this provider can move a target to. +fn validate_template_idle(idle: &IdlePolicy) -> anyhow::Result<()> { + idle.validate()?; + if let Some(action) = unsupported_idle_stage(idle) { + bail!("the Incus provider cannot carry out the {action} stage") + } + Ok(()) +} + +/// Bytes of an Incus size string: a whole number with an optional unit +/// (`B`, decimal `kB` to `EB`, binary `KiB` to `EiB`), as Incus accepts +/// for `limits.memory` and disk `size`. +fn parse_incus_size(value: &str) -> anyhow::Result { + const UNITS: [(&str, u64); 13] = [ + ("EiB", 1 << 60), + ("PiB", 1 << 50), + ("TiB", 1 << 40), + ("GiB", 1 << 30), + ("MiB", 1 << 20), + ("KiB", 1 << 10), + ("EB", 1_000_000_000_000_000_000), + ("PB", 1_000_000_000_000_000), + ("TB", 1_000_000_000_000), + ("GB", 1_000_000_000), + ("MB", 1_000_000), + ("kB", 1_000), + ("B", 1), + ]; + let (number, multiplier) = UNITS + .iter() + .find_map(|(unit, multiplier)| value.strip_suffix(unit).map(|number| (number, *multiplier))) + .unwrap_or((value, 1)); + if number.is_empty() || !number.bytes().all(|byte| byte.is_ascii_digit()) { + bail!("must be a whole number of bytes with an optional Incus unit such as GiB: {value}") + } + number + .parse::() + .ok() + .and_then(|number| number.checked_mul(multiplier)) + .filter(|bytes| *bytes > 0 && *bytes < (1 << 53)) + .ok_or_else(|| anyhow::anyhow!("size is out of range: {value}")) +} + fn validate_incus(incus: &IncusConfig) -> anyhow::Result<()> { if incus.endpoints.is_empty() { bail!("incus.endpoints must contain at least one HTTPS endpoint") @@ -212,7 +375,7 @@ fn validate_incus(incus: &IncusConfig) -> anyhow::Result<()> { .as_deref() .is_none_or(str::is_empty) { - bail!("incus cluster mode requires clusterNetworkUplink for OVN binding networks") + bail!("incus cluster mode requires clusterNetworkUplink for OVN universe networks") } if incus.mode == IncusMode::Single && incus.cluster_network_uplink.is_some() { bail!("incus clusterNetworkUplink is only valid in cluster mode") @@ -256,13 +419,6 @@ impl std::ops::Deref for Config { } impl Config { - pub fn template(&self, template_id: &str) -> anyhow::Result<&TemplatePolicy> { - self.templates - .get(template_id) - .filter(|template| !template.deprecated) - .ok_or_else(|| anyhow::anyhow!("template is not offered by this provider")) - } - pub fn incus_mode_name(&self) -> &'static str { match self.incus.mode { IncusMode::Single => "single", @@ -273,90 +429,225 @@ impl Config { #[cfg(test)] mod tests { + use environment_protocol::idle::IdlePolicyError; + use super::*; - #[tokio::test(flavor = "current_thread")] - async fn config_loads_provider_wide_templates_without_universe_bindings() { - let directory = tempfile::tempdir().expect("tempdir"); - let path = directory.path().join("provider.json"); - let document = serde_json::json!({ + const CREDENTIAL: &str = "0123456789abcdef0123456789abcdef"; + + fn base() -> serde_json::Value { + serde_json::json!({ "controllerListen":"127.0.0.1:0", "incus":{"mode":"single","endpoints":["https://incus.test"],"clientCertificatePem":"cert","clientPrivateKeyPem":"key","serverCaPem":"ca","storagePool":"default"}, "network":{"deniedEgressCidrs":["100.64.0.0/10"]}, "templates":[{"templateId":"dev-small-v1","displayName":"Small","imageFingerprint":"abc","cpu":2,"memory":"4GiB","disk":"40GiB"}] - }); + }) + } + + /// Write `document` and load it. With `credential`, `credentialFile` + /// points at a fresh file holding it. + async fn load( + directory: &tempfile::TempDir, + mut document: serde_json::Value, + credential: Option<&str>, + ) -> anyhow::Result { + if let Some(credential) = credential { + let credential_path = directory.path().join("credential"); + tokio::fs::write(&credential_path, format!("{credential}\n")) + .await + .unwrap(); + document["credentialFile"] = serde_json::json!(credential_path); + } + let path = directory.path().join("provider.json"); tokio::fs::write(&path, serde_json::to_vec(&document).unwrap()) .await .unwrap(); - let config = ProviderArgs { - config: path.clone(), - } - .load() - .await - .expect("config"); - assert!(config.template("dev-small-v1").is_ok()); + ProviderArgs { config: path }.load().await + } + + #[tokio::test(flavor = "current_thread")] + async fn config_loads_provider_wide_templates_without_universe_lists() { + let directory = tempfile::tempdir().expect("tempdir"); + let config = load(&directory, base(), Some(CREDENTIAL)) + .await + .expect("config"); + let template = &config.templates[&TemplateId::new("dev-small-v1").unwrap()]; + assert_eq!( + template.resources, + TemplateResources { + vcpus: Some(2), + memory_bytes: Some(4 << 30), + disk_bytes: Some(40 << 30), + } + ); + assert_eq!(template.idle, None); assert_eq!(config.network.denied_egress_cidrs, ["100.64.0.0/10"]); - let mut duplicate = document; + assert_eq!(config.relay_idle_seconds, 60); + assert_eq!(config.idle_check_seconds, 60); + assert!(config.credential.is_some()); + + let mut duplicate = base(); let first_template = duplicate["templates"][0].clone(); duplicate["templates"] .as_array_mut() .unwrap() .push(first_template); - tokio::fs::write(&path, serde_json::to_vec(&duplicate).unwrap()) - .await - .unwrap(); - assert!(ProviderArgs { config: path }.load().await.is_err()); + assert!(load(&directory, duplicate, Some(CREDENTIAL)).await.is_err()); } #[tokio::test(flavor = "current_thread")] async fn config_rejects_invalid_provider_network_cidr() { let directory = tempfile::tempdir().expect("tempdir"); - let path = directory.path().join("provider.json"); - let document = serde_json::json!({ - "controllerListen":"127.0.0.1:0", - "incus":{"mode":"single","endpoints":["https://incus.test"],"clientCertificatePem":"cert","clientPrivateKeyPem":"key","serverCaPem":"ca","storagePool":"default"}, - "network":{"deniedEgressCidrs":["not-a-cidr"]}, - "templates":[{"templateId":"dev-small-v1","displayName":"Small","imageFingerprint":"abc","cpu":2,"memory":"4GiB","disk":"40GiB"}] - }); - tokio::fs::write(&path, serde_json::to_vec(&document).unwrap()) - .await - .unwrap(); - assert!(ProviderArgs { config: path }.load().await.is_err()); + let mut document = base(); + document["network"]["deniedEgressCidrs"] = serde_json::json!(["not-a-cidr"]); + assert!(load(&directory, document, None).await.is_err()); } #[tokio::test(flavor = "current_thread")] async fn config_requires_explicit_valid_mode_topology() { let directory = tempfile::tempdir().expect("tempdir"); - let path = directory.path().join("provider.json"); - let base = serde_json::json!({ - "controllerListen":"127.0.0.1:0", - "incus":{"mode":"single","endpoints":["https://one.test","https://two.test"],"clientCertificatePem":"cert","clientPrivateKeyPem":"key","serverCaPem":"ca","storagePool":"default"}, - "network":{"deniedEgressCidrs":[]}, - "templates":[{"templateId":"t","displayName":"T","imageFingerprint":"abc","cpu":1,"memory":"1GiB","disk":"10GiB"}] - }); - tokio::fs::write(&path, serde_json::to_vec(&base).unwrap()) + let mut document = base(); + document["incus"]["endpoints"] = + serde_json::json!(["https://one.test", "https://two.test"]); + assert!(load(&directory, document.clone(), None).await.is_err()); + + document["incus"]["mode"] = serde_json::json!("cluster"); + document["incus"]["clusterNetworkUplink"] = serde_json::json!("UPLINK"); + document["templates"][0]["clusterGroup"] = serde_json::json!("x86-production"); + let config = load(&directory, document, None) .await - .unwrap(); + .expect("cluster config"); + assert_eq!(config.incus_mode_name(), "cluster"); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_configured_credential_must_be_strong() { + let directory = tempfile::tempdir().expect("tempdir"); + assert!(load(&directory, base(), Some("short")).await.is_err()); assert!( - ProviderArgs { - config: path.clone() - } - .load() + load( + &directory, + base(), + Some("0123456789abcdef 0123456789abcdef") + ) .await .is_err() ); + let mut missing_file = base(); + missing_file["credentialFile"] = serde_json::json!(directory.path().join("absent")); + assert!(load(&directory, missing_file, None).await.is_err()); + } - let mut clustered = base; - clustered["incus"]["mode"] = serde_json::json!("cluster"); - clustered["incus"]["clusterNetworkUplink"] = serde_json::json!("UPLINK"); - clustered["templates"][0]["clusterGroup"] = serde_json::json!("x86-production"); - tokio::fs::write(&path, serde_json::to_vec(&clustered).unwrap()) - .await - .unwrap(); - let config = ProviderArgs { config: path } - .load() + #[tokio::test(flavor = "current_thread")] + async fn the_credential_is_optional() { + let directory = tempfile::tempdir().expect("tempdir"); + let config = load(&directory, base(), None) .await - .expect("cluster config"); - assert_eq!(config.incus_mode_name(), "cluster"); + .expect("config without a credential"); + assert!(config.credential.is_none()); + } + + #[tokio::test(flavor = "current_thread")] + async fn the_data_route_base_is_not_provider_config() { + // Lightspeed derives the data route base from the controller URL or + // takes it from the provider record; the provider has no say. + let directory = tempfile::tempdir().expect("tempdir"); + let mut document = base(); + document["dataRouteBase"] = serde_json::json!("ws://127.0.0.1:19090/routes"); + assert!(load(&directory, document, None).await.is_err()); + } + + #[tokio::test(flavor = "current_thread")] + async fn relays_never_idle_close_below_the_protocol_minimum() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut document = base(); + document["relayIdleSeconds"] = serde_json::json!(59); + assert!(load(&directory, document.clone(), None).await.is_err()); + document["relayIdleSeconds"] = serde_json::json!(1900); + let config = load(&directory, document, None).await.expect("config"); + assert_eq!(config.relay_idle_seconds, 1900); + } + + #[tokio::test(flavor = "current_thread")] + async fn the_idle_check_interval_is_positive() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut document = base(); + document["idleCheckSeconds"] = serde_json::json!(0); + assert!(load(&directory, document.clone(), None).await.is_err()); + document["idleCheckSeconds"] = serde_json::json!(15); + let config = load(&directory, document, None).await.expect("config"); + assert_eq!(config.idle_check_seconds, 15); + } + + #[tokio::test(flavor = "current_thread")] + async fn template_idle_policies_are_validated_at_load() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut document = base(); + document["templates"][0]["idle"] = + serde_json::json!({"pauseAfterMs": 600000, "stopAfterMs": 3600000}); + let config = load(&directory, document, None).await.expect("config"); + assert_eq!( + config.templates[&TemplateId::new("dev-small-v1").unwrap()].idle, + Some(IdlePolicy { + pause_after_ms: Some(600_000), + stop_after_ms: Some(3_600_000), + ..IdlePolicy::default() + }) + ); + + for idle in [ + serde_json::json!({}), + serde_json::json!({"pauseAfterMs": 0}), + serde_json::json!({"pauseAfterMs": 600000, "stopAfterMs": 60000}), + // Incus targets cannot be suspended. + serde_json::json!({"suspendAfterMs": 600000}), + serde_json::json!({"pauseAfterMs": 600000, "unknown": 1}), + ] { + let mut document = base(); + document["templates"][0]["idle"] = idle.clone(); + assert!(load(&directory, document, None).await.is_err(), "{idle}"); + } + assert_eq!( + validate_template_idle(&IdlePolicy::default()) + .unwrap_err() + .downcast::() + .unwrap(), + IdlePolicyError::Empty + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn template_sizes_and_descriptions_are_validated_at_load() { + let directory = tempfile::tempdir().expect("tempdir"); + for (field, value) in [ + ("memory", serde_json::json!("50%")), + ("disk", serde_json::json!("lots")), + ("displayName", serde_json::json!("")), + ("description", serde_json::json!("a\u{0}b")), + ("templateId", serde_json::json!("")), + ] { + let mut document = base(); + document["templates"][0][field] = value; + assert!(load(&directory, document, None).await.is_err(), "{field}"); + } + } + + #[test] + fn incus_sizes_parse_decimal_and_binary_units() { + assert_eq!(parse_incus_size("4GiB").unwrap(), 4 << 30); + assert_eq!(parse_incus_size("512MB").unwrap(), 512_000_000); + assert_eq!(parse_incus_size("1024").unwrap(), 1024); + assert!(parse_incus_size("GiB").is_err()); + assert!(parse_incus_size("1.5GiB").is_err()); + assert!(parse_incus_size("0").is_err()); + } + + #[test] + fn credential_comparison_needs_the_exact_value() { + let credential = ProviderCredential::new(CREDENTIAL).unwrap(); + assert!(credential.matches(CREDENTIAL)); + assert!(!credential.matches(&CREDENTIAL[1..])); + assert!(!credential.matches("")); + assert!(!format!("{credential:?}").contains(CREDENTIAL)); } } diff --git a/crates/environment-provider-incus/src/edge.rs b/crates/environment-provider-incus/src/edge.rs index e3812d70d..7a9bcb9e1 100644 --- a/crates/environment-provider-incus/src/edge.rs +++ b/crates/environment-provider-incus/src/edge.rs @@ -134,7 +134,7 @@ async fn resolve( fn resolve_targets(targets: Vec, hostname: &str) -> Option { for target in targets { if target.ingress_hostname.as_deref() == Some(hostname) - && target.status == environment_protocol::control::targets::ProviderTargetStatus::Ready + && target.status == crate::incus::InstanceStatus::Running && let (Some(address), Some(port)) = (target.ipv4_address, target.ingress_port) { return Some(Route { address, port }); @@ -188,32 +188,20 @@ async fn proxy_websocket(mut client: WebSocket, endpoint: String) { #[cfg(test)] mod tests { use super::*; - use crate::incus::OwnedTarget; + use crate::incus::{InstanceStatus, OwnedTarget}; use axum::{Router, extract::WebSocketUpgrade, routing::get}; - use environment_protocol::{control::targets::ProviderTargetStatus, shared::ProviderTargetId}; use std::time::Duration; - fn target(status: ProviderTargetStatus, hostname: Option<&str>) -> OwnedTarget { + fn target(status: InstanceStatus, hostname: Option<&str>) -> OwnedTarget { OwnedTarget { - target_id: ProviderTargetId::new("target-a"), - name: "target-a".to_owned(), - universe_id: "universe-a".to_owned(), - binding_id: "binding-a".to_owned(), - environment_id: "environment-a".to_owned(), - incarnation_id: "incarnation-a".to_owned(), - request_id: "request-a".to_owned(), - template_id: "template-a".to_owned(), - image_fingerprint: "fingerprint-a".to_owned(), - adoption_source: None, status, - ipv4_address: Some("10.0.0.2".to_owned()), ingress_hostname: hostname.map(ToOwned::to_owned), ingress_port: Some(8080), - location: None, + ..crate::ingress::tests::target() } } - #[tokio::test] + #[tokio::test(flavor = "current_thread")] async fn ingress_proxy_closes_upstream_when_client_disappears() { let upstream_listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await @@ -268,7 +256,7 @@ mod tests { fn authorization_resolves_only_the_exact_ready_ingress_hostname() { let hostname = "route.env.example.com"; let route = resolve_targets( - vec![target(ProviderTargetStatus::Ready, Some(hostname))], + vec![target(InstanceStatus::Running, Some(hostname))], hostname, ) .expect("ready route"); @@ -277,14 +265,14 @@ mod tests { assert!( resolve_targets( - vec![target(ProviderTargetStatus::Starting, Some(hostname))], + vec![target(InstanceStatus::Starting, Some(hostname))], hostname, ) .is_none() ); assert!( resolve_targets( - vec![target(ProviderTargetStatus::Ready, Some(hostname))], + vec![target(InstanceStatus::Running, Some(hostname))], "other.env.example.com", ) .is_none() diff --git a/crates/environment-provider-incus/src/idle.rs b/crates/environment-provider-incus/src/idle.rs new file mode 100644 index 000000000..c3c36c58d --- /dev/null +++ b/crates/environment-provider-incus/src/idle.rs @@ -0,0 +1,604 @@ +//! The provider's idle loop: it powers idle targets down and closes them on +//! the provider's own clock. +//! +//! Every `idleCheckSeconds` the loop visits each owned instance that has an +//! idle policy (recorded as `user.lightspeed.idle` at `target/create`) and +//! no data connection in progress. A running instance's idle duration is +//! envd's `env/getIdle` `idleForMs`, read over the guest network, so running +//! processes, jobs and leftover process groups keep it awake. A paused or +//! stopped instance has no envd to ask: the loop continues from the idle +//! duration it recorded when it powered the instance down, plus the time +//! since, kept in memory. A provider restart, or an instance powered down +//! by anyone else, starts that clock at zero. +//! +//! A data route dial that woke an instance gives it a grace of at least one +//! `idleCheckSeconds` once the dial's connection ends ([`Wakes`]). Unpausing +//! does not reset envd's clock, and a dial that only observes (the +//! readiness probe's `initialize`) is no `use`, so without the grace the +//! next pass could power the instance down again before the caller's first +//! call arrives. +//! +//! The most escalated due stage ([`IdlePolicy::due_action`]) beyond the +//! instance's current power runs as the target's background job, so it is +//! serialized with controller calls and with the wake a data route dial +//! starts. An idle close deletes the instance; `target/get` then answers +//! `notFound`, which Lightspeed records as the provider ending the target. + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +use anyhow::{Context as _, bail}; +use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + control::target::PowerState, + data::{ + handshake::InitializeParams, + idle::{GetIdleParams, GetIdleResponse}, + }, + idle::{IdleAction, IdlePolicy}, + jsonrpc::{self, Id, Request}, + manifest::{DataInitialize, EnvGetIdle}, +}; +use futures_util::{SinkExt as _, StreamExt as _}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; +use tokio_tungstenite::tungstenite::Message; + +use crate::{ + Config, IncusBackend, + incus::{InstanceStatus, OwnedTarget, SUPPORTED_POWER_STATES}, + relay::{self, GuestSocket}, + server::{self, App, WorkKind}, +}; + +/// Bound on one guest's whole `initialize` and `env/getIdle` exchange. +const IDLE_READ_TIMEOUT: Duration = Duration::from_secs(10); + +/// The least wake grace, whatever `idleCheckSeconds`: room for a caller's +/// retries after a `waking` refusal and its first call. +const MIN_WAKE_GRACE: Duration = Duration::from_secs(30); + +/// The wake grace for an idle loop that looks every `idle_check_seconds`. +pub(crate) fn wake_grace(idle_check_seconds: u64) -> Duration { + Duration::from_secs(idle_check_seconds).max(MIN_WAKE_GRACE) +} + +/// When data route dials last woke instances, by instance name. The idle +/// loop leaves an instance alone for the grace after its last wake. +#[derive(Clone, Debug)] +pub(crate) struct Wakes { + grace: Duration, + at: Arc>>, +} + +impl Wakes { + pub(crate) fn new(grace: Duration) -> Self { + Self { + grace, + at: Arc::default(), + } + } + + /// Record that a dial which woke `name` ended at `at`. + pub(crate) fn record(&self, name: &str, at: Instant) { + let mut wakes = self.lock(); + let entry = wakes.entry(name.to_owned()).or_insert(at); + *entry = (*entry).max(at); + } + + /// True while `name` is within the grace of its last wake at `now`. + pub(crate) fn within_grace(&self, name: &str, now: Instant) -> bool { + self.lock() + .get(name) + .is_some_and(|at| now.saturating_duration_since(*at) < self.grace) + } + + /// Forget the wakes whose grace ended by `now`. + fn prune(&self, now: Instant) { + self.lock() + .retain(|_, at| now.saturating_duration_since(*at) < self.grace); + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.at + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +/// Run the idle loop until the task is aborted. +pub(crate) async fn run(app: App) { + let interval = Duration::from_secs(app.config.idle_check_seconds); + let mut clock = IdleClock::default(); + loop { + tokio::time::sleep(interval).await; + if let Err(error) = check_once(&app, &mut clock, Instant::now()).await { + eprintln!("warning: idle check failed: {error:#}"); + } + } +} + +/// Idle durations of the instances the loop powered down, by instance name. +#[derive(Debug, Default)] +pub(crate) struct IdleClock(HashMap); + +#[derive(Clone, Copy, Debug)] +struct PoweredDown { + /// The idle duration when the clock started. + idle_for_ms: u64, + since: Instant, +} + +impl IdleClock { + /// The idle duration of a powered-down instance at `now`, starting its + /// clock at zero when none runs. + fn powered_down_idle(&mut self, name: &str, now: Instant) -> u64 { + let entry = *self.0.entry(name.to_owned()).or_insert(PoweredDown { + idle_for_ms: 0, + since: now, + }); + let elapsed = u64::try_from(now.saturating_duration_since(entry.since).as_millis()) + .unwrap_or(u64::MAX); + entry.idle_for_ms.saturating_add(elapsed) + } +} + +/// One pass over every owned instance. +pub(crate) async fn check_once( + app: &App, + clock: &mut IdleClock, + now: Instant, +) -> anyhow::Result<()> { + let targets = app.backend.list_all_owned().await?; + app.wakes.prune(now); + clock + .0 + .retain(|name, _| targets.iter().any(|target| target.name() == name)); + for target in targets { + let name = target.name().to_owned(); + let Some(policy) = target.idle.as_ref() else { + clock.0.remove(&name); + continue; + }; + if app.relays.active(&name) { + // In use; a dial wakes the instance, so no powered-down clock + // applies any more. + clock.0.remove(&name); + continue; + } + if app.work.busy(&name) { + continue; + } + if app.wakes.within_grace(&name, now) { + // Just woken; its powered-down clock ended with the wake. + clock.0.remove(&name); + continue; + } + let (current, idle_for_ms) = match target.status { + InstanceStatus::Running => { + clock.0.remove(&name); + match read_idle(&app.config, &target).await { + Ok(report) => (PowerState::Running, report.idle_for_ms), + // Booting, or envd not answering: nothing to decide on. + Err(_) => continue, + } + } + InstanceStatus::Frozen => (PowerState::Paused, clock.powered_down_idle(&name, now)), + InstanceStatus::Stopped => (PowerState::Stopped, clock.powered_down_idle(&name, now)), + // Transitional, failed or unknown: look again next pass. + _ => continue, + }; + let Some(action) = escalation(policy, current, idle_for_ms) else { + continue; + }; + // A data connection may have started while envd was asked. + if app.relays.active(&name) || app.work.busy(&name) { + continue; + } + apply(app, &target, action); + match action.power_state() { + Some(_) => { + clock.0.insert( + name, + PoweredDown { + idle_for_ms, + since: now, + }, + ); + } + None => { + clock.0.remove(&name); + } + } + } + Ok(()) +} + +/// The due stage of `policy` that takes a target beyond `current`, if any: +/// a paused target may still be stopped or closed, a stopped one only +/// closed. +fn escalation(policy: &IdlePolicy, current: PowerState, idle_for_ms: u64) -> Option { + let action = policy.due_action(idle_for_ms, &SUPPORTED_POWER_STATES)?; + let depth = |power: PowerState| match power { + PowerState::Running => 0, + PowerState::Paused => 1, + PowerState::Suspended => 2, + PowerState::Stopped => 3, + }; + let target_depth = action.power_state().map_or(4, depth); + (target_depth > depth(current)).then_some(action) +} + +/// Start `action` as the target's background job. +fn apply(app: &App, target: &OwnedTarget, action: IdleAction) { + let universe_id = target.universe(); + let name = target.name(); + match action.power_state() { + Some(power) => app.work.start( + name, + WorkKind::Power(power), + server::power_job(&app.backend, universe_id, name, power), + ), + None => app.work.start( + name, + WorkKind::Close, + server::close_job(&app.backend, universe_id, name), + ), + } +} + +/// envd's idle report of a running guest, over a data connection of the +/// provider's own. `initialize` and `env/getIdle` are observations, so the +/// exchange does not reset the idle clock it reads. +pub(crate) async fn read_idle( + config: &Config, + target: &OwnedTarget, +) -> anyhow::Result { + tokio::time::timeout(IDLE_READ_TIMEOUT, async { + let mut guest = relay::dial_guest(config, target).await?; + let report = exchange(&mut guest).await; + let _ = guest.close(None).await; + report + }) + .await + .context("env/getIdle timed out")? +} + +async fn exchange(guest: &mut GuestSocket) -> anyhow::Result { + let initialized: Value = call( + guest, + 1, + DataInitialize::NAME, + &InitializeParams { + protocol_version: ENVD_PROTOCOL_VERSION, + client_info: server::server_info(), + }, + ) + .await + .context("initialize")?; + // envd refuses another version with `versionMismatch`; a matching + // version is all the provider needs from the handshake. + let version = initialized.get("protocolVersion").and_then(Value::as_u64); + if version != Some(u64::from(ENVD_PROTOCOL_VERSION)) { + bail!("envd answered initialize with protocol version {version:?}") + } + call(guest, 2, EnvGetIdle::NAME, &GetIdleParams::default()) + .await + .context("env/getIdle") +} + +/// Send one request and read frames until its response arrives. +async fn call( + guest: &mut GuestSocket, + id: i64, + method: &str, + params: &P, +) -> anyhow::Result { + let request = Request::new(Id::Number(id), method, params)?; + guest.send(Message::Text(request.to_text().into())).await?; + while let Some(frame) = guest.next().await { + let Message::Text(text) = frame? else { + continue; + }; + match jsonrpc::parse(text.as_str()) { + Ok(jsonrpc::Message::Response(response)) if response.id == Some(Id::Number(id)) => { + return Ok(response.decode()?); + } + // Notifications, and anything not answering this request. + Ok(_) => {} + Err(rejection) => bail!("envd sent an unreadable frame: {}", rejection.error), + } + } + bail!("envd closed the connection before answering {method}") +} + +#[cfg(test)] +mod tests { + use environment_protocol::jsonrpc::Response; + + use super::*; + use crate::{ + policy::{self, tests::owner}, + server::tests::{CREDENTIAL, FakeBackend, config_with}, + }; + + fn policy() -> IdlePolicy { + IdlePolicy { + pause_after_ms: Some(1_000), + stop_after_ms: Some(10_000), + close_after_ms: Some(100_000), + ..IdlePolicy::default() + } + } + + #[test] + fn only_stages_beyond_the_current_power_are_due() { + let policy = policy(); + assert_eq!(escalation(&policy, PowerState::Running, 999), None); + assert_eq!( + escalation(&policy, PowerState::Running, 1_000), + Some(IdleAction::Pause) + ); + assert_eq!(escalation(&policy, PowerState::Paused, 5_000), None); + assert_eq!( + escalation(&policy, PowerState::Paused, 10_000), + Some(IdleAction::Stop) + ); + assert_eq!( + escalation(&policy, PowerState::Running, 10_000), + Some(IdleAction::Stop) + ); + assert_eq!(escalation(&policy, PowerState::Stopped, 50_000), None); + assert_eq!( + escalation(&policy, PowerState::Stopped, 100_000), + Some(IdleAction::Close) + ); + // Incus cannot suspend, so a suspend stage never fires here. + let suspend = IdlePolicy { + suspend_after_ms: Some(1_000), + ..IdlePolicy::default() + }; + assert_eq!(escalation(&suspend, PowerState::Running, 5_000), None); + } + + #[test] + fn the_powered_down_clock_starts_at_zero_and_advances() { + let mut clock = IdleClock::default(); + let start = Instant::now(); + assert_eq!(clock.powered_down_idle("a", start), 0); + assert_eq!( + clock.powered_down_idle("a", start + Duration::from_millis(1_500)), + 1_500 + ); + clock.0.insert( + "b".to_owned(), + PoweredDown { + idle_for_ms: 4_000, + since: start, + }, + ); + assert_eq!( + clock.powered_down_idle("b", start + Duration::from_secs(1)), + 5_000 + ); + } + + fn app_with(port: u16) -> (App, FakeBackend) { + let backend = FakeBackend::default(); + ( + App::new(config_with(port, Some(CREDENTIAL)), backend.clone()), + backend, + ) + } + + async fn settle(app: &App, name: &str) { + tokio::time::timeout(Duration::from_secs(2), async { + while app.work.busy(name) { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("the idle job finishes"); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_powered_down_target_escalates_on_the_provider_clock_and_closes() { + let (app, backend) = app_with(19091); + let owner = owner("u", "e"); + let name = policy::instance_name(&owner); + backend.insert(&owner, "small", InstanceStatus::Frozen); + backend.update(&owner, |target| target.idle = Some(policy())); + let mut clock = IdleClock::default(); + let start = Instant::now(); + + // The first look starts the clock; nothing is due at zero. + check_once(&app, &mut clock, start).await.unwrap(); + assert!(backend.power_calls.lock().unwrap().is_empty()); + + check_once(&app, &mut clock, start + Duration::from_secs(10)) + .await + .unwrap(); + settle(&app, &name).await; + assert_eq!( + backend.power_calls.lock().unwrap().as_slice(), + [PowerState::Stopped] + ); + assert_eq!(backend.get(&owner).unwrap().status, InstanceStatus::Stopped); + + // The clock continues from the stop, not from zero. + check_once(&app, &mut clock, start + Duration::from_secs(100)) + .await + .unwrap(); + settle(&app, &name).await; + assert!(backend.get(&owner).is_none()); + assert_eq!(backend.deletes.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!(clock.0.is_empty()); + } + + #[tokio::test(flavor = "current_thread")] + async fn targets_without_a_policy_or_in_use_are_left_alone() { + let (app, backend) = app_with(19091); + let unpoliced = owner("u", "unpoliced"); + backend.insert(&unpoliced, "small", InstanceStatus::Frozen); + let relayed = owner("u", "relayed"); + backend.insert(&relayed, "small", InstanceStatus::Frozen); + backend.update(&relayed, |target| target.idle = Some(policy())); + let _hold = app.relays.hold(&policy::instance_name(&relayed)); + let mut clock = IdleClock::default(); + let start = Instant::now(); + for offset in [0, 1_000] { + check_once(&app, &mut clock, start + Duration::from_secs(offset)) + .await + .unwrap(); + } + assert!(backend.power_calls.lock().unwrap().is_empty()); + assert!(backend.get(&unpoliced).is_some()); + assert!(backend.get(&relayed).is_some()); + } + + /// A guest envd stand-in that answers `initialize` and reports + /// `idle_for_ms` from `env/getIdle`. Returns its port. + async fn idle_envd(idle_for_ms: u64) -> u16 { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(async move { + let Ok(mut socket) = tokio_tungstenite::accept_async(stream).await else { + return; + }; + while let Some(Ok(Message::Text(text))) = socket.next().await { + let Ok(jsonrpc::Message::Request(request)) = jsonrpc::parse(text.as_str()) + else { + continue; + }; + let result = match request.method.as_str() { + DataInitialize::NAME => { + serde_json::json!({"protocolVersion": ENVD_PROTOCOL_VERSION}) + } + EnvGetIdle::NAME => serde_json::to_value(GetIdleResponse { + idle_for_ms, + ..GetIdleResponse::default() + }) + .unwrap(), + _ => continue, + }; + let response = Response::success(request.id, &result); + if socket + .send(Message::Text(response.to_text().into())) + .await + .is_err() + { + return; + } + } + }); + } + }); + port + } + + fn running(backend: &FakeBackend, environment: &str) -> crate::incus::OwnedTarget { + let owner = owner("u", environment); + backend.insert(&owner, "small", InstanceStatus::Running); + backend.update(&owner, |target| { + target.idle = Some(policy()); + target.ipv4_address = Some("127.0.0.1".to_owned()); + }); + backend.get(&owner).unwrap() + } + + #[tokio::test(flavor = "current_thread")] + async fn a_running_target_is_paused_by_its_envd_idle_report() { + let port = idle_envd(5_000).await; + let (app, backend) = app_with(port); + let target = running(&backend, "e"); + assert_eq!( + read_idle(&app.config, &target).await.unwrap().idle_for_ms, + 5_000 + ); + let mut clock = IdleClock::default(); + let now = Instant::now(); + check_once(&app, &mut clock, now).await.unwrap(); + settle(&app, target.name()).await; + assert_eq!( + backend.power_calls.lock().unwrap().as_slice(), + [PowerState::Paused] + ); + // The provider clock continues from envd's report. + assert_eq!( + clock.powered_down_idle(target.name(), now + Duration::from_secs(5)), + 10_000 + ); + } + + #[test] + fn the_wake_grace_covers_at_least_one_idle_check() { + assert_eq!(wake_grace(1), MIN_WAKE_GRACE); + assert_eq!(wake_grace(600), Duration::from_secs(600)); + let wakes = Wakes::new(Duration::from_secs(60)); + let start = Instant::now(); + wakes.record("a", start + Duration::from_secs(10)); + // An earlier record does not shorten the grace. + wakes.record("a", start); + assert!(wakes.within_grace("a", start + Duration::from_secs(69))); + assert!(!wakes.within_grace("a", start + Duration::from_secs(70))); + assert!(!wakes.within_grace("b", start)); + wakes.prune(start + Duration::from_secs(70)); + assert!(wakes.lock().is_empty()); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_freshly_woken_target_is_left_alone_for_the_wake_grace() { + // Unpausing keeps envd's idle clock from before the pause. + let port = idle_envd(5_000).await; + let (app, backend) = app_with(port); + let target = running(&backend, "e"); + let woke = Instant::now(); + app.wakes.record(target.name(), woke); + let mut clock = IdleClock::default(); + check_once(&app, &mut clock, woke).await.unwrap(); + assert!(backend.power_calls.lock().unwrap().is_empty()); + + check_once( + &app, + &mut clock, + woke + wake_grace(app.config.idle_check_seconds), + ) + .await + .unwrap(); + settle(&app, target.name()).await; + assert_eq!( + backend.power_calls.lock().unwrap().as_slice(), + [PowerState::Paused] + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn running_work_reported_by_envd_keeps_a_target_awake() { + // envd holds idleForMs at zero while processes, jobs or leftover + // process groups run. + let port = idle_envd(0).await; + let (app, backend) = app_with(port); + running(&backend, "e"); + let mut clock = IdleClock::default(); + check_once(&app, &mut clock, Instant::now()).await.unwrap(); + assert!(backend.power_calls.lock().unwrap().is_empty()); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_running_target_without_a_listening_envd_is_left_alone() { + let port = { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener.local_addr().unwrap().port() + }; + let (app, backend) = app_with(port); + running(&backend, "e"); + let mut clock = IdleClock::default(); + check_once(&app, &mut clock, Instant::now()).await.unwrap(); + assert!(backend.power_calls.lock().unwrap().is_empty()); + } +} diff --git a/crates/environment-provider-incus/src/incus.rs b/crates/environment-provider-incus/src/incus.rs index 53a94f72f..3877af87e 100644 --- a/crates/environment-provider-incus/src/incus.rs +++ b/crates/environment-provider-incus/src/incus.rs @@ -15,11 +15,9 @@ use serde_json::{Value, json}; use thiserror::Error; use environment_protocol::{ - control::targets::{ - AdoptTargetParams, CreateTargetParams, PowerState, ProviderBindingContext, - ProviderTargetStatus, - }, - shared::ProviderTargetId, + control::target::{PowerState, TargetOwner}, + idle::{IdleAction, IdlePolicy}, + ids::{EnvironmentId, ProviderTargetId, UniverseId}, }; use crate::{ @@ -27,19 +25,17 @@ use crate::{ policy, }; +/// A Lightspeed-managed Incus instance, read from its ownership metadata. #[derive(Clone, Debug)] pub struct OwnedTarget { + /// The instance name, which is the deterministic name of `owner`. pub target_id: ProviderTargetId, - pub name: String, - pub universe_id: String, - pub binding_id: String, - pub environment_id: String, - pub incarnation_id: String, - pub request_id: String, + pub owner: TargetOwner, pub template_id: String, - pub image_fingerprint: String, - pub adoption_source: Option, - pub status: ProviderTargetStatus, + /// The idle policy the target was created with: the `target/create` + /// override or the template's default. `None` keeps it awake. + pub idle: Option, + pub status: InstanceStatus, pub ipv4_address: Option, pub ingress_hostname: Option, pub ingress_port: Option, @@ -48,54 +44,83 @@ pub struct OwnedTarget { pub location: Option, } +impl OwnedTarget { + pub fn name(&self) -> &str { + self.target_id.as_str() + } + + pub fn universe(&self) -> &UniverseId { + &self.owner.universe_id + } +} + +/// Incus instance status, reduced to what the provider acts on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InstanceStatus { + Running, + Starting, + Stopping, + Stopped, + /// `Frozen` or `Freezing`: a paused VM. + Frozen, + Error, + /// Any other status Incus reports. + Other, +} + +impl InstanceStatus { + fn from_incus(status: &str) -> Self { + match status { + "Running" | "Thawed" => Self::Running, + "Starting" => Self::Starting, + "Stopping" => Self::Stopping, + "Stopped" => Self::Stopped, + "Frozen" | "Freezing" => Self::Frozen, + "Error" => Self::Error, + _ => Self::Other, + } + } +} + +/// The Incus operations the controller needs. Mutations are slow (image +/// pulls, boots, graceful shutdowns); the controller runs them in the +/// background and observes the result with [`IncusBackend::get_owned`]. #[async_trait] pub trait IncusBackend: Clone + Send + Sync + 'static { async fn topology(&self) -> anyhow::Result; - async fn reconcile_binding(&self, binding: &ProviderBindingContext) -> anyhow::Result<()>; + /// Ensure the universe's project, network, ACL and profile exist. + async fn reconcile_universe(&self, universe_id: &UniverseId) -> anyhow::Result<()>; async fn ensure_image(&self, fingerprint: &str) -> anyhow::Result<()>; - async fn list_owned( - &self, - binding: &ProviderBindingContext, - ) -> anyhow::Result>; async fn list_all_owned(&self) -> anyhow::Result>; + /// The managed instance `name` in the universe's project. `None` when + /// the project or instance does not exist, when `name` is not an Incus + /// instance name, or when the instance carries no valid ownership + /// metadata. async fn get_owned( &self, - binding: &ProviderBindingContext, - target_id: &ProviderTargetId, + universe_id: &UniverseId, + name: &str, ) -> anyhow::Result>; + /// Create the owner's instance from the template, recording `idle` as + /// its idle policy, and start it. async fn create_vm( &self, - binding: &ProviderBindingContext, + owner: &TargetOwner, template: &TemplatePolicy, - params: &CreateTargetParams, - ) -> anyhow::Result; - async fn adopt_vm( - &self, - binding: &ProviderBindingContext, - params: &AdoptTargetParams, - ) -> anyhow::Result; - async fn delete_vm( - &self, - binding: &ProviderBindingContext, - target: &OwnedTarget, - force: bool, + idle: Option<&IdlePolicy>, ) -> anyhow::Result<()>; + /// Destroy the instance: a graceful stop within 30 s, then a forced stop, + /// then delete. Its ingress metadata goes with it. + async fn delete_vm(&self, target: &OwnedTarget) -> anyhow::Result<()>; + /// Record (or with `None` clear) the realized ingress hostname and port. async fn set_ingress( &self, - binding: &ProviderBindingContext, - target: &OwnedTarget, - hostname: Option<&str>, - port: Option, - ) -> anyhow::Result; - /// Converge one owned instance toward `power`. Idempotent by inventory; - /// transitional instances are reported as observed and converged on a - /// later call. - async fn set_power( - &self, - binding: &ProviderBindingContext, target: &OwnedTarget, - power: PowerState, - ) -> anyhow::Result; + route: Option<(&str, u16)>, + ) -> anyhow::Result<()>; + /// Converge the instance toward `power`. A stopped VM asked to pause is + /// started; a later call freezes it. + async fn set_power(&self, target: &OwnedTarget, power: PowerState) -> anyhow::Result<()>; } /// Power states the Incus provider can move a VM to. Stateful stop @@ -103,6 +128,26 @@ pub trait IncusBackend: Clone + Send + Sync + 'static { pub const SUPPORTED_POWER_STATES: [PowerState; 3] = [PowerState::Running, PowerState::Paused, PowerState::Stopped]; +/// The first stage of `idle` that moves a target to a power state this +/// provider cannot reach; such a stage would never fire. +pub fn unsupported_idle_stage(idle: &IdlePolicy) -> Option { + idle.stages() + .into_iter() + .map(|(action, _)| action) + .find(|action| { + action + .power_state() + .is_some_and(|state| !SUPPORTED_POWER_STATES.contains(&state)) + }) +} + +/// envd's state directory inside every VM: outside the agent's working tree +/// and writable by the service (`ReadWritePaths` in the image's unit). +pub const ENVD_STATE_DIR: &str = "/var/lib/lightspeed-envd"; + +/// Bound on one graceful Incus state action, in seconds. +const STATE_ACTION_TIMEOUT_SECONDS: i64 = 30; + #[derive(Clone, Debug, serde::Serialize)] #[serde(rename_all = "camelCase")] pub struct IncusTopology { @@ -444,6 +489,9 @@ impl IncusClient { } async fn instance(&self, project: &str, name: &str) -> anyhow::Result> { + if !policy::is_instance_name(name) { + return Ok(None); + } if !self .exists(&format!("/instances/{name}"), Some(project)) .await? @@ -467,13 +515,7 @@ impl IncusClient { ) .await .unwrap_or_default(); - owned_from_instance(instance, state).map(Some) - } - - async fn observed_instance(&self, project: &str, name: &str) -> anyhow::Result { - self.instance(project, name) - .await? - .ok_or_else(|| anyhow::anyhow!("Incus target disappeared during a power change")) + Ok(owned_from_instance(instance, state)) } async fn instance_state_action( @@ -481,7 +523,7 @@ impl IncusClient { project: &str, name: &str, action: &str, - timeout: u64, + timeout: i64, force: bool, ) -> anyhow::Result<()> { self.request_unit( @@ -493,21 +535,42 @@ impl IncusClient { .await } - async fn raw_instance(&self, project: &str, name: &str) -> anyhow::Result> { - if !self - .exists(&format!("/instances/{name}"), Some(project)) - .await? - { - return Ok(None); + /// Run a state action; when it fails, accept it anyway if the instance + /// already reached a status in `reached` (a stale read, a transport + /// failure after Incus accepted the action, or a concurrent change). + async fn converge_state( + &self, + project: &str, + name: &str, + action: &str, + force: bool, + reached: &[InstanceStatus], + ) -> anyhow::Result<()> { + let Err(error) = self + .instance_state_action(project, name, action, STATE_ACTION_TIMEOUT_SECONDS, force) + .await + else { + return Ok(()); + }; + match self.instance(project, name).await? { + Some(target) if reached.contains(&target.status) => Ok(()), + None if action == "stop" => Ok(()), + _ => Err(error), } - self.request( - Method::GET, - &format!("/instances/{name}"), - Some(project), - None, - ) - .await - .map(Some) + } + + /// Incus can briefly report the pre-operation `Stopped` status right + /// after a start operation succeeds. Wait a few seconds for the first + /// post-start status so the next observation is not stale. + async fn settle_after_start(&self, project: &str, name: &str) -> anyhow::Result<()> { + for _ in 0..10 { + match self.instance(project, name).await? { + Some(target) if target.status != InstanceStatus::Stopped => return Ok(()), + None => anyhow::bail!("the Incus instance disappeared while starting"), + Some(_) => tokio::time::sleep(Duration::from_millis(500)).await, + } + } + Ok(()) } async fn reconcile_network_acls(&self) -> anyhow::Result<()> { @@ -516,10 +579,10 @@ impl IncusClient { .await?; let mut networks = Vec::new(); for project in projects { - let Some(binding) = project.binding_context() else { + let Some(universe_id) = project.universe() else { continue; }; - let network_name = policy::network_name(&binding); + let network_name = policy::network_name(&universe_id); let network_project = network_resource_project(self.config.incus.mode, &project.name); if !self .exists(&format!("/networks/{network_name}"), network_project) @@ -555,7 +618,7 @@ impl IncusClient { .find(|denied| ipv4_cidrs_overlap(own_cidr, denied)) { anyhow::bail!( - "provider denied egress CIDR {denied} overlaps Incus-assigned binding network {own_cidr}" + "provider denied egress CIDR {denied} overlaps Incus-assigned universe network {own_cidr}" ) } } @@ -573,7 +636,7 @@ impl IncusClient { "action":"reject", "state":"enabled", "source":source, - "description":"block sibling Lightspeed binding network" + "description":"block sibling Lightspeed universe network" }) }) .collect(); @@ -595,7 +658,7 @@ impl IncusClient { "action":"reject", "state":"enabled", "destination":destination, - "description":"block sibling Lightspeed binding network" + "description":"block sibling Lightspeed universe network" }) })) .collect(); @@ -604,7 +667,7 @@ impl IncusClient { &format!("/network-acls/{project}-acl"), network_project, Some(json!({ - "description":"Lightspeed binding baseline", + "description":"Lightspeed universe baseline", "ingress":ingress, "egress":egress, "config":{} @@ -651,30 +714,30 @@ impl IncusBackend for IncusClient { }) } - async fn reconcile_binding(&self, binding: &ProviderBindingContext) -> anyhow::Result<()> { + async fn reconcile_universe(&self, universe_id: &UniverseId) -> anyhow::Result<()> { let _reconcile_guard = self.reconcile_lock.lock().await; - let project = policy::project_name(binding); + let project = policy::project_name(universe_id); let isolated_project_networks = self.config.incus.mode == IncusMode::Cluster; let network_project = network_resource_project(self.config.incus.mode, &project); - self.ensure_resource(&format!("/projects/{project}"), "/projects", None, json!({"name":project,"description":"Lightspeed managed binding","config":{"features.images":"false","features.networks":isolated_project_networks.to_string(),"features.profiles":"true","restricted":"true","restricted.devices.nic":"managed","restricted.devices.disk":"managed","restricted.networks.access":policy::network_name(binding),"user.lightspeed.managed":"true","user.lightspeed.universe":binding.universe_id,"user.lightspeed.binding":binding.binding_id}})).await?; - self.ensure_resource(&format!("/network-acls/{}", policy::acl_name(binding)), "/network-acls", network_project, json!({"name":policy::acl_name(binding),"description":"Lightspeed binding baseline","ingress":[],"egress":[],"config":{}})).await?; + self.ensure_resource(&format!("/projects/{project}"), "/projects", None, json!({"name":project,"description":"Lightspeed managed universe","config":{"features.images":"false","features.networks":isolated_project_networks.to_string(),"features.profiles":"true","restricted":"true","restricted.devices.nic":"managed","restricted.devices.disk":"managed","restricted.networks.access":policy::network_name(universe_id),"user.lightspeed.managed":"true","user.lightspeed.universe":universe_id.as_str()}})).await?; + self.ensure_resource(&format!("/network-acls/{}", policy::acl_name(universe_id)), "/network-acls", network_project, json!({"name":policy::acl_name(universe_id),"description":"Lightspeed universe baseline","ingress":[],"egress":[],"config":{}})).await?; let network = match self.config.incus.mode { IncusMode::Single => { - json!({"name":policy::network_name(binding),"type":"bridge","config":{"ipv4.address":"auto","ipv4.nat":"true","ipv6.address":"none","security.acls":policy::acl_name(binding),"security.acls.default.ingress.action":"allow","security.acls.default.egress.action":"allow"}}) + json!({"name":policy::network_name(universe_id),"type":"bridge","config":{"ipv4.address":"auto","ipv4.nat":"true","ipv6.address":"none","security.acls":policy::acl_name(universe_id),"security.acls.default.ingress.action":"allow","security.acls.default.egress.action":"allow"}}) } IncusMode::Cluster => { - json!({"name":policy::network_name(binding),"type":"ovn","config":{"network":self.config.incus.cluster_network_uplink.as_deref().expect("validated cluster uplink"),"ipv4.address":"auto","ipv4.nat":"true","ipv6.address":"none","security.acls":policy::acl_name(binding),"security.acls.default.ingress.action":"allow","security.acls.default.egress.action":"allow"}}) + json!({"name":policy::network_name(universe_id),"type":"ovn","config":{"network":self.config.incus.cluster_network_uplink.as_deref().expect("validated cluster uplink"),"ipv4.address":"auto","ipv4.nat":"true","ipv6.address":"none","security.acls":policy::acl_name(universe_id),"security.acls.default.ingress.action":"allow","security.acls.default.egress.action":"allow"}}) } }; self.ensure_resource( - &format!("/networks/{}", policy::network_name(binding)), + &format!("/networks/{}", policy::network_name(universe_id)), "/networks", network_project, network, ) .await?; self.reconcile_network_acls().await?; - self.ensure_resource(&format!("/profiles/{}", policy::profile_name(binding)), "/profiles", Some(&project), json!({"name":policy::profile_name(binding),"description":"Lightspeed VM baseline","config":{},"devices":{"eth0":{"type":"nic","network":policy::network_name(binding),"security.port_isolation":"true","security.ipv4_filtering":"true"},"root":{"type":"disk","pool":self.config.incus.storage_pool,"path":"/"}}})).await + self.ensure_resource(&format!("/profiles/{}", policy::profile_name(universe_id)), "/profiles", Some(&project), json!({"name":policy::profile_name(universe_id),"description":"Lightspeed VM baseline","config":{},"devices":{"eth0":{"type":"nic","network":policy::network_name(universe_id),"security.port_isolation":"true","security.ipv4_filtering":"true"},"root":{"type":"disk","pool":self.config.incus.storage_pool,"path":"/"}}})).await } async fn ensure_image(&self, fingerprint: &str) -> anyhow::Result<()> { @@ -693,35 +756,6 @@ impl IncusBackend for IncusClient { Ok(()) } - async fn list_owned( - &self, - binding: &ProviderBindingContext, - ) -> anyhow::Result> { - let project = policy::project_name(binding); - if !self.exists(&format!("/projects/{project}"), None).await? { - return Ok(Vec::new()); - } - let instances: Vec = self - .request(Method::GET, "/instances?recursion=1", Some(&project), None) - .await?; - let mut targets = Vec::new(); - for instance in instances { - let state = self - .request( - Method::GET, - &format!("/instances/{}/state", instance.name), - Some(&project), - None, - ) - .await - .unwrap_or_default(); - if let Ok(target) = owned_from_instance(instance, state) { - targets.push(target); - } - } - Ok(targets) - } - async fn list_all_owned(&self) -> anyhow::Result> { let projects: Vec = self .request(Method::GET, "/projects?recursion=1", None, None) @@ -746,14 +780,10 @@ impl IncusBackend for IncusClient { ) .await .unwrap_or_default(); - if let Ok(target) = owned_from_instance(instance, state) { - let binding = ProviderBindingContext { - universe_id: target.universe_id.clone(), - binding_id: target.binding_id.clone(), - }; - if project.name == policy::project_name(&binding) { - targets.push(target); - } + if let Some(target) = owned_from_instance(instance, state) + && project.name == policy::project_name(target.universe()) + { + targets.push(target); } } } @@ -762,69 +792,59 @@ impl IncusBackend for IncusClient { async fn get_owned( &self, - binding: &ProviderBindingContext, - target_id: &ProviderTargetId, + universe_id: &UniverseId, + name: &str, ) -> anyhow::Result> { - self.instance(&policy::project_name(binding), target_id.as_str()) - .await + if !policy::is_instance_name(name) { + return Ok(None); + } + let project = policy::project_name(universe_id); + if !self.exists(&format!("/projects/{project}"), None).await? { + return Ok(None); + } + Ok(self + .instance(&project, name) + .await? + .filter(|target| target.universe() == universe_id)) } async fn create_vm( &self, - binding: &ProviderBindingContext, + owner: &TargetOwner, template: &TemplatePolicy, - params: &CreateTargetParams, - ) -> anyhow::Result { - let project = policy::project_name(binding); - let name = policy::instance_name( - &binding.universe_id, - &binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); + idle: Option<&IdlePolicy>, + ) -> anyhow::Result<()> { + let project = policy::project_name(&owner.universe_id); + let name = policy::instance_name(owner); let created_at = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH)? .as_millis() .to_string(); let cloud_init = format!( - "#cloud-config\nwrite_files:\n - path: /etc/lightspeed-envd/provider.env\n permissions: '0600'\n content: |\n LIGHTSPEED_ENVD_LISTEN=0.0.0.0:{}\n LIGHTSPEED_ENVD_CWD=/workspace\n LIGHTSPEED_ENVD_FS_ROOT=/\nruncmd:\n - [mkdir, -p, /workspace]\n - [systemctl, enable, --now, lightspeed-envd]\n", - self.config.envd_port + "#cloud-config\nwrite_files:\n - path: /etc/lightspeed-envd/provider.env\n permissions: '0600'\n content: |\n LIGHTSPEED_ENVD_LISTEN=0.0.0.0:{}\n LIGHTSPEED_ENVD_CWD=/workspace\n LIGHTSPEED_ENVD_FS_ROOT=/\n LIGHTSPEED_ENVD_STATE_DIR={}\nruncmd:\n - [mkdir, -p, /workspace]\n - [systemctl, enable, --now, lightspeed-envd]\n", + self.config.envd_port, ENVD_STATE_DIR, ); - let config = BTreeMap::from([ + let mut config = ownership_config(owner); + config.extend([ ("limits.cpu".to_owned(), template.cpu.to_string()), ("limits.memory".to_owned(), template.memory.clone()), ("cloud-init.user-data".to_owned(), cloud_init), - ("user.lightspeed.managed".to_owned(), "true".to_owned()), - ( - "user.lightspeed.universe".to_owned(), - params.binding.universe_id.clone(), - ), - ( - "user.lightspeed.binding".to_owned(), - params.binding.binding_id.clone(), - ), - ( - "user.lightspeed.environment".to_owned(), - params.environment_id.clone(), - ), ( - "user.lightspeed.incarnation".to_owned(), - params.incarnation_id.clone(), - ), - ( - "user.lightspeed.request".to_owned(), - params.request_id.clone(), - ), - ( - "user.lightspeed.template".to_owned(), - template.template_id.clone(), + format!("{}template", policy::META_PREFIX), + template.template_id.to_string(), ), ( - "user.lightspeed.image".to_owned(), + format!("{}image", policy::META_PREFIX), template.image_fingerprint.clone(), ), - ("user.lightspeed.created_at_ms".to_owned(), created_at), + (format!("{}created_at_ms", policy::META_PREFIX), created_at), ]); + if let Some(idle) = idle { + config.insert( + format!("{}idle", policy::META_PREFIX), + serde_json::to_string(idle).context("encode the idle policy")?, + ); + } let path = if let Some(group) = &template.cluster_group { if !self .exists(&format!("/cluster/groups/{group}"), None) @@ -836,326 +856,140 @@ impl IncusBackend for IncusClient { } else { "/instances".to_owned() }; - if let Err(create_error) = self.request_unit(Method::POST, &path, Some(&project), Some(json!({"name":name,"type":"virtual-machine","start":false,"profiles":[policy::profile_name(binding)],"source":{"type":"image","fingerprint":template.image_fingerprint},"config":config,"devices":{"root":{"type":"disk","pool":self.config.incus.storage_pool,"path":"/","size":template.disk}}}))).await { + if let Err(create_error) = self.request_unit(Method::POST, &path, Some(&project), Some(json!({"name":name,"type":"virtual-machine","start":false,"profiles":[policy::profile_name(&owner.universe_id)],"source":{"type":"image","fingerprint":template.image_fingerprint},"config":config,"devices":{"root":{"type":"disk","pool":self.config.incus.storage_pool,"path":"/","size":template.disk}}}))).await // A transport can fail after Incus accepted the deterministic // create. Resolve that uncertainty before returning an error. - if let Some(target) = self.instance(&project, &name).await? { - return Ok(target); - } - return Err(create_error); - } - if let Err(error) = self - .request_unit( - Method::PUT, - &format!("/instances/{name}/state"), - Some(&project), - Some(json!({"action":"start","timeout":30,"force":false})), - ) - .await - { - let observed = self.instance(&project, &name).await?; - if !observed.as_ref().is_some_and(|target| { - matches!( - target.status, - ProviderTargetStatus::Starting | ProviderTargetStatus::Ready - ) - }) { - return Err(error); - } - } - let target = self - .instance(&project, &name) - .await? - .ok_or_else(|| anyhow::anyhow!("created Incus instance disappeared"))?; - Ok(target_after_accepted_start(target)) - } - - async fn adopt_vm( - &self, - binding: &ProviderBindingContext, - params: &AdoptTargetParams, - ) -> anyhow::Result { - let project = policy::project_name(binding); - let destination = policy::instance_name( - &binding.universe_id, - &binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); - if let Some(target) = self.instance(&project, &destination).await? { - verify_adopted_target(&target, params)?; - return ensure_instance_started(self, &project, &destination, target).await; - } - - let (source_project, source_name) = parse_source_target(¶ms.source_target)?; - let source = self - .raw_instance(source_project, source_name) - .await? - .ok_or_else(|| anyhow::anyhow!("Incus adoption source target not found"))?; - if source.kind != "virtual-machine" { - anyhow::bail!("only Incus virtual-machine instances can be adopted") - } - if source.ephemeral { - anyhow::bail!("ephemeral Incus instances cannot be adopted") - } - if source - .config - .get(&format!("{}managed", policy::META_PREFIX)) - .is_some_and(|value| value == "true") + && self.instance(&project, &name).await?.is_none() { - anyhow::bail!("Incus source target is already Lightspeed-managed") - } - - // Incus merges these overrides with the instance's existing local - // configuration during a cross-project move. User configuration is - // therefore preserved while ownership metadata is added. - let config = BTreeMap::from([ - (format!("{}managed", policy::META_PREFIX), "true".to_owned()), - ( - format!("{}universe", policy::META_PREFIX), - binding.universe_id.clone(), - ), - ( - format!("{}binding", policy::META_PREFIX), - binding.binding_id.clone(), - ), - ( - format!("{}environment", policy::META_PREFIX), - params.environment_id.clone(), - ), - ( - format!("{}incarnation", policy::META_PREFIX), - params.incarnation_id.clone(), - ), - ( - format!("{}request", policy::META_PREFIX), - params.request_id.clone(), - ), - ( - format!("{}template", policy::META_PREFIX), - "adopted".to_owned(), - ), - ( - format!("{}image", policy::META_PREFIX), - "adopted".to_owned(), - ), - ( - format!("{}adoption_source", policy::META_PREFIX), - params.source_target.clone(), - ), - ]); - - // Replace arbitrary source profiles with the binding baseline. Local - // NICs would override its managed eth0, so explicitly disable all - // source-local NICs except an eth0 override, which is rewritten onto - // the managed binding network. - let mut devices = BTreeMap::new(); - for (name, device) in &source.devices { - if device.get("type").and_then(Value::as_str) != Some("nic") { - continue; - } - let replacement = if name == "eth0" { - json!({ - "type":"nic", - "network":policy::network_name(binding), - "security.port_isolation":"true", - "security.ipv4_filtering":"true" - }) - } else { - json!({"type":"none"}) - }; - devices.insert(name.clone(), replacement); - } - - if source.status != "Stopped" { - self.request_unit( - Method::PUT, - &format!("/instances/{source_name}/state"), - Some(source_project), - Some(json!({"action":"stop","timeout":30,"force":true})), - ) - .await - .context("stop Incus source target before adoption")?; - } - - if let Err(move_error) = self - .request_unit( - Method::POST, - &format!("/instances/{source_name}"), - Some(source_project), - Some(json!({ - "name":destination, - "migration":true, - "live":false, - "project":project, - "pool":self.config.incus.storage_pool, - "instance_only":false, - "config":config, - "devices":devices, - "profiles":[policy::profile_name(binding)] - })), - ) - .await - && self.instance(&project, &destination).await?.is_none() - { - return Err(move_error); + return Err(create_error); } - let target = self - .instance(&project, &destination) - .await? - .ok_or_else(|| anyhow::anyhow!("adopted Incus target is not visible"))?; - verify_adopted_target(&target, params)?; - ensure_instance_started(self, &project, &destination, target).await + start_instance(self, &project, &name).await } - async fn delete_vm( - &self, - binding: &ProviderBindingContext, - target: &OwnedTarget, - force: bool, - ) -> anyhow::Result<()> { - let project = policy::project_name(binding); - if target.status != ProviderTargetStatus::Stopped { - let stop_result = self - .request_unit( - Method::PUT, - &format!("/instances/{}/state", target.name), - Some(&project), - Some(json!({ - "action":"stop", - "timeout": if force { 0 } else { 30 }, - "force": force - })), - ) + async fn delete_vm(&self, target: &OwnedTarget) -> anyhow::Result<()> { + let project = policy::project_name(target.universe()); + let name = target.name(); + if target.status != InstanceStatus::Stopped { + // Graceful first; a guest that ignores ACPI power-down (a hung + // kernel, an emergency shell) is then stopped by force, since its + // disk is destroyed either way. + let graceful = self + .converge_state(&project, name, "stop", false, &[InstanceStatus::Stopped]) .await; - if let Err(error) = stop_result { - let observed = self.instance(&project, &target.name).await?; - if observed - .as_ref() - .is_some_and(|target| target.status != ProviderTargetStatus::Stopped) - { - return Err(error); - } + if graceful.is_err() { + self.converge_state(&project, name, "stop", true, &[InstanceStatus::Stopped]) + .await + .context("force-stop Incus instance before delete")?; } } if let Err(error) = self .request_unit( Method::DELETE, - &format!("/instances/{}", target.name), + &format!("/instances/{name}"), Some(&project), None, ) .await - && self.instance(&project, &target.name).await?.is_some() + && self.instance(&project, name).await?.is_some() { return Err(error); } Ok(()) } - async fn set_power( - &self, - binding: &ProviderBindingContext, - target: &OwnedTarget, - power: PowerState, - ) -> anyhow::Result { + async fn set_power(&self, target: &OwnedTarget, power: PowerState) -> anyhow::Result<()> { if !SUPPORTED_POWER_STATES.contains(&power) { anyhow::bail!("Incus provider does not support power state {power}") } - let project = policy::project_name(binding); - let name = target.name.clone(); + let project = policy::project_name(target.universe()); + let name = target.name(); match (target.status, power) { - // Already there or already on the way: report the observation. - (ProviderTargetStatus::Ready | ProviderTargetStatus::Starting, PowerState::Running) - | (ProviderTargetStatus::Paused, PowerState::Paused) - | (ProviderTargetStatus::Stopped, PowerState::Stopped) => Ok(target.clone()), - (ProviderTargetStatus::Paused, PowerState::Running) => { - self.instance_state_action(&project, &name, "unfreeze", 30, false) - .await?; - let observed = self.observed_instance(&project, &name).await?; - Ok(target_after_accepted_resume(observed)) + // Already there or already on the way. + (InstanceStatus::Running | InstanceStatus::Starting, PowerState::Running) + | (InstanceStatus::Frozen, PowerState::Paused) + | (InstanceStatus::Stopped | InstanceStatus::Stopping, PowerState::Stopped) => Ok(()), + (InstanceStatus::Frozen, PowerState::Running) => { + self.converge_state( + &project, + name, + "unfreeze", + false, + &[InstanceStatus::Running], + ) + .await } - (ProviderTargetStatus::Stopped, PowerState::Running | PowerState::Paused) => { + (InstanceStatus::Stopped, PowerState::Running | PowerState::Paused) => { // A stopped VM cannot be frozen directly; converge through // running and let a later call freeze it. - ensure_instance_started(self, &project, &name, target.clone()).await + start_instance(self, &project, name).await } - (ProviderTargetStatus::Ready, PowerState::Paused) => { - self.instance_state_action(&project, &name, "freeze", 30, false) - .await?; - let observed = self.observed_instance(&project, &name).await?; - Ok(target_after_accepted_freeze(observed)) + (InstanceStatus::Running, PowerState::Paused) => { + self.converge_state(&project, name, "freeze", false, &[InstanceStatus::Frozen]) + .await } - (ProviderTargetStatus::Ready | ProviderTargetStatus::Paused, PowerState::Stopped) => { - if target.status == ProviderTargetStatus::Paused { - self.instance_state_action(&project, &name, "unfreeze", 30, false) - .await?; + (InstanceStatus::Running | InstanceStatus::Frozen, PowerState::Stopped) => { + if target.status == InstanceStatus::Frozen { + self.converge_state( + &project, + name, + "unfreeze", + false, + &[InstanceStatus::Running], + ) + .await?; } - if let Err(error) = self - .instance_state_action(&project, &name, "stop", 30, false) + self.converge_state(&project, name, "stop", false, &[InstanceStatus::Stopped]) .await - { - let observed = self.instance(&project, &name).await?; - if !observed - .as_ref() - .is_some_and(|target| target.status == ProviderTargetStatus::Stopped) - { - return Err(error); - } - } - let observed = self.observed_instance(&project, &name).await?; - Ok(target_after_accepted_stop(observed)) } - // Starting/creating/closing/failed/unknown: nothing safe to do - // until the instance settles; the caller re-observes later. - (_, _) => Ok(target.clone()), + // Transitional, failed or unknown: nothing safe to do until the + // instance settles; the caller re-observes later. + _ => Ok(()), } } async fn set_ingress( &self, - binding: &ProviderBindingContext, target: &OwnedTarget, - hostname: Option<&str>, - port: Option, - ) -> anyhow::Result { - if hostname.is_some() != port.is_some() { - anyhow::bail!("ingress hostname and port must be set together") - } - let project = policy::project_name(binding); + route: Option<(&str, u16)>, + ) -> anyhow::Result<()> { + let project = policy::project_name(target.universe()); + let name = target.name(); let instance: Instance = self .request( Method::GET, - &format!("/instances/{}", target.name), + &format!("/instances/{name}"), Some(&project), None, ) .await?; let mut config = instance.config; - match (hostname, port) { - (Some(hostname), Some(port)) => { - config.insert( - "user.lightspeed.ingress_hostname".to_owned(), - hostname.to_owned(), - ); - config.insert("user.lightspeed.ingress_port".to_owned(), port.to_string()); + let hostname_key = format!("{}ingress_hostname", policy::META_PREFIX); + let port_key = format!("{}ingress_port", policy::META_PREFIX); + match route { + Some((hostname, port)) => { + config.insert(hostname_key, hostname.to_owned()); + config.insert(port_key, port.to_string()); } - _ => { - config.remove("user.lightspeed.ingress_hostname"); - config.remove("user.lightspeed.ingress_port"); + None => { + config.remove(&hostname_key); + config.remove(&port_key); } } - let update = self.request_unit(Method::PUT, &format!("/instances/{}", target.name), Some(&project), Some(json!({"config":config,"description":instance.description,"profiles":instance.profiles,"devices":instance.devices,"ephemeral":instance.ephemeral}))).await; + let update = self.request_unit(Method::PUT, &format!("/instances/{name}"), Some(&project), Some(json!({"config":config,"description":instance.description,"profiles":instance.profiles,"devices":instance.devices,"ephemeral":instance.ephemeral}))).await; let observed = self - .instance(&project, &target.name) + .instance(&project, name) .await? .ok_or_else(|| anyhow::anyhow!("target disappeared while updating ingress"))?; - if observed.ingress_hostname.as_deref() != hostname || observed.ingress_port != port { + let realized = observed + .ingress_hostname + .as_deref() + .zip(observed.ingress_port); + if realized != route { return Err(update.err().unwrap_or_else(|| { anyhow::anyhow!("Incus did not realize the requested ingress metadata") })); } - Ok(observed) + Ok(()) } } @@ -1169,8 +1003,6 @@ struct Envelope { #[derive(Deserialize)] struct Instance { name: String, - #[serde(default, rename = "type")] - kind: String, #[serde(default)] description: String, #[serde(default)] @@ -1212,7 +1044,9 @@ struct Project { } impl Project { - fn binding_context(&self) -> Option { + /// The universe a provider-managed project isolates, or `None` for any + /// other project, including one whose name does not match its metadata. + fn universe(&self) -> Option { if self .config .get("user.lightspeed.managed") @@ -1220,15 +1054,13 @@ impl Project { { return None; } - let binding = ProviderBindingContext { - universe_id: self.config.get("user.lightspeed.universe")?.clone(), - binding_id: self.config.get("user.lightspeed.binding")?.clone(), - }; - (self.name == policy::project_name(&binding)).then_some(binding) + let universe_id = + UniverseId::new(self.config.get("user.lightspeed.universe")?.as_str()).ok()?; + (self.name == policy::project_name(&universe_id)).then_some(universe_id) } fn is_lightspeed_managed(&self) -> bool { - self.binding_context().is_some() + self.universe().is_some() } } @@ -1338,184 +1170,80 @@ fn network_resource_project(mode: IncusMode, project: &str) -> Option<&str> { } } -fn parse_source_target(value: &str) -> anyhow::Result<(&str, &str)> { - let (project, name) = value.split_once('/').unwrap_or(("default", value)); - if project.is_empty() - || name.is_empty() - || name.contains('/') - || [project, name].into_iter().any(|part| { - part.len() > 63 - || part.starts_with('-') - || part.ends_with('-') - || part - .chars() - .any(|character| !character.is_ascii_alphanumeric() && character != '-') - }) - { - anyhow::bail!("invalid Incus source target; expected instance or project/instance") - } - Ok((project, name)) -} - -async fn ensure_instance_started( - client: &IncusClient, - project: &str, - name: &str, - target: OwnedTarget, -) -> anyhow::Result { - if matches!( - target.status, - ProviderTargetStatus::Ready | ProviderTargetStatus::Starting - ) { - return Ok(target); - } - if target.status == ProviderTargetStatus::Paused { - client - .instance_state_action(project, name, "unfreeze", 30, false) - .await?; - let observed = client.observed_instance(project, name).await?; - return Ok(target_after_accepted_resume(observed)); - } - if let Err(error) = client - .request_unit( - Method::PUT, - &format!("/instances/{name}/state"), - Some(project), - Some(json!({"action":"start","timeout":30,"force":false})), - ) - .await - { - let observed = client.instance(project, name).await?; - if !observed.as_ref().is_some_and(|target| { - matches!( - target.status, - ProviderTargetStatus::Starting | ProviderTargetStatus::Ready - ) - }) { - return Err(error); - } - } +/// Start a stopped instance or unfreeze a frozen one; a running or starting +/// one needs nothing. +async fn start_instance(client: &IncusClient, project: &str, name: &str) -> anyhow::Result<()> { let target = client .instance(project, name) .await? - .ok_or_else(|| anyhow::anyhow!("Incus target disappeared while starting"))?; - Ok(target_after_accepted_start(target)) -} - -/// Incus can briefly return the pre-operation `Stopped` state immediately -/// after its asynchronous start operation succeeds. Keep the provisioning -/// lifecycle pending until a later observation sees the actual running state. -fn target_after_accepted_start(mut target: OwnedTarget) -> OwnedTarget { - target.status = status_after_accepted_start(target.status); - target -} - -fn status_after_accepted_start(status: ProviderTargetStatus) -> ProviderTargetStatus { - match status { - ProviderTargetStatus::Stopped => ProviderTargetStatus::Starting, - status => status, - } -} - -/// After an accepted unfreeze, a stale `Frozen` read means "resuming", not -/// "still paused". -fn target_after_accepted_resume(mut target: OwnedTarget) -> OwnedTarget { - if target.status == ProviderTargetStatus::Paused { - target.status = ProviderTargetStatus::Starting; - } - target -} - -/// After an accepted freeze, a stale `Running` read must not make the caller -/// freeze again; report the requested steady state. -fn target_after_accepted_freeze(mut target: OwnedTarget) -> OwnedTarget { - if target.status == ProviderTargetStatus::Ready { - target.status = ProviderTargetStatus::Paused; + .ok_or_else(|| anyhow::anyhow!("the Incus instance disappeared before it started"))?; + match target.status { + InstanceStatus::Running | InstanceStatus::Starting => Ok(()), + InstanceStatus::Frozen => { + client + .converge_state(project, name, "unfreeze", false, &[InstanceStatus::Running]) + .await + } + _ => { + client + .converge_state( + project, + name, + "start", + false, + &[InstanceStatus::Running, InstanceStatus::Starting], + ) + .await?; + client.settle_after_start(project, name).await + } } - target } -/// After an accepted stop, a stale `Running`/`Frozen` read is reported as -/// stopped; a later observation corrects it if the stop did not stick. -fn target_after_accepted_stop(mut target: OwnedTarget) -> OwnedTarget { - if matches!( - target.status, - ProviderTargetStatus::Ready | ProviderTargetStatus::Paused - ) { - target.status = ProviderTargetStatus::Stopped; - } - target +/// The `user.lightspeed.*` keys that make an instance the owner's. +fn ownership_config(owner: &TargetOwner) -> BTreeMap { + BTreeMap::from([ + (format!("{}managed", policy::META_PREFIX), "true".to_owned()), + ( + format!("{}universe", policy::META_PREFIX), + owner.universe_id.to_string(), + ), + ( + format!("{}environment", policy::META_PREFIX), + owner.environment_id.to_string(), + ), + ]) } -fn verify_adopted_target(target: &OwnedTarget, params: &AdoptTargetParams) -> anyhow::Result<()> { - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - || target.request_id != params.request_id - || target.template_id != "adopted" - || target.image_fingerprint != "adopted" - || target.adoption_source.as_deref() != Some(params.source_target.as_str()) - { - anyhow::bail!("existing target metadata conflicts with adoption request") - } - Ok(()) -} - -fn owned_from_instance(instance: Instance, state: InstanceState) -> anyhow::Result { - let name = instance.name.clone(); +/// The owned target an instance describes, or `None` when its ownership +/// metadata is missing, invalid or does not match its deterministic name. +fn owned_from_instance(instance: Instance, state: InstanceState) -> Option { let get = |key: &str| { instance .config .get(&format!("{}{}", policy::META_PREFIX, key)) .cloned() - .ok_or_else(|| anyhow::anyhow!("instance {name} is not owned: missing {key}")) }; if get("managed")? != "true" { - anyhow::bail!("instance is not Lightspeed-managed") + return None; } - let universe_id = get("universe")?; - let binding_id = get("binding")?; - let environment_id = get("environment")?; - let incarnation_id = get("incarnation")?; - if instance.name - != policy::instance_name(&universe_id, &binding_id, &environment_id, &incarnation_id) - { - anyhow::bail!("managed instance name does not match immutable ownership metadata") + let owner = TargetOwner { + universe_id: UniverseId::new(get("universe")?).ok()?, + environment_id: EnvironmentId::new(get("environment")?).ok()?, + }; + if instance.name != policy::instance_name(&owner) { + return None; } - let ipv4_address = managed_ipv4_address(&state); - Ok(OwnedTarget { - target_id: ProviderTargetId::new(instance.name.clone()), - name: instance.name, - universe_id, - binding_id, - environment_id, - incarnation_id, - request_id: get("request")?, + Some(OwnedTarget { + target_id: ProviderTargetId::new(instance.name.clone()).ok()?, + owner, template_id: get("template")?, - image_fingerprint: get("image")?, - adoption_source: instance - .config - .get(&format!("{}adoption_source", policy::META_PREFIX)) - .cloned(), - status: match instance.status.as_str() { - "Running" => ProviderTargetStatus::Ready, - "Starting" => ProviderTargetStatus::Starting, - "Stopped" => ProviderTargetStatus::Stopped, - // Incus reports a frozen container or paused VM as `Frozen`. - "Frozen" | "Freezing" => ProviderTargetStatus::Paused, - "Error" => ProviderTargetStatus::Failed, - _ => ProviderTargetStatus::Unknown, - }, - ipv4_address, - ingress_hostname: instance - .config - .get(&format!("{}ingress_hostname", policy::META_PREFIX)) - .cloned(), - ingress_port: instance - .config - .get(&format!("{}ingress_port", policy::META_PREFIX)) - .and_then(|value| value.parse().ok()), + // The provider writes this key itself; a value that does not decode + // leaves the target without an idle policy rather than unowned. + idle: get("idle").and_then(|value| serde_json::from_str(&value).ok()), + status: InstanceStatus::from_incus(&instance.status), + ipv4_address: managed_ipv4_address(&state), + ingress_hostname: get("ingress_hostname"), + ingress_port: get("ingress_port").and_then(|value| value.parse().ok()), location: (!instance.location.is_empty()).then_some(instance.location), }) } @@ -1542,20 +1270,17 @@ mod tests { } #[test] - fn managed_projects_require_matching_binding_metadata() { - let binding = ProviderBindingContext { - universe_id: "u".to_owned(), - binding_id: "b".to_owned(), - }; + fn managed_projects_require_matching_universe_metadata() { + let universe_id = UniverseId::new("u").unwrap(); let project = Project { - name: policy::project_name(&binding), + name: policy::project_name(&universe_id), config: BTreeMap::from([ ("user.lightspeed.managed".to_owned(), "true".to_owned()), ("user.lightspeed.universe".to_owned(), "u".to_owned()), - ("user.lightspeed.binding".to_owned(), "b".to_owned()), ]), }; assert!(project.is_lightspeed_managed()); + assert_eq!(project.universe(), Some(universe_id)); let spoofed = Project { name: "ls-spoofed".to_owned(), ..project @@ -1563,31 +1288,79 @@ mod tests { assert!(!spoofed.is_lightspeed_managed()); } + fn instance(name: &str, config: BTreeMap) -> Instance { + Instance { + name: name.to_owned(), + description: String::new(), + profiles: Vec::new(), + devices: BTreeMap::new(), + ephemeral: false, + status: "Running".to_owned(), + config, + location: String::new(), + } + } + #[test] - fn adoption_source_defaults_to_the_default_project() { - assert_eq!( - parse_source_target("manual-vm").unwrap(), - ("default", "manual-vm") - ); - assert_eq!( - parse_source_target("staging/manual-vm").unwrap(), - ("staging", "manual-vm") + fn ownership_metadata_must_match_the_deterministic_name() { + let owner = policy::tests::owner("u", "e"); + let mut config = ownership_config(&owner); + config.insert("user.lightspeed.template".to_owned(), "t".to_owned()); + config.insert("user.lightspeed.image".to_owned(), "f".to_owned()); + let name = policy::instance_name(&owner); + let owned = owned_from_instance(instance(&name, config.clone()), InstanceState::default()) + .expect("owned"); + assert_eq!(owned.owner, owner); + assert_eq!(owned.status, InstanceStatus::Running); + assert_eq!(owned.idle, None); + assert!( + owned_from_instance( + instance("ls-other", config.clone()), + InstanceState::default() + ) + .is_none() ); - assert!(parse_source_target("too/many/parts").is_err()); + config.insert("user.lightspeed.universe".to_owned(), "bad id!".to_owned()); + assert!(owned_from_instance(instance(&name, config), InstanceState::default()).is_none()); } #[test] fn standalone_network_resources_are_global_but_cluster_networks_are_project_local() { assert_eq!( - network_resource_project(IncusMode::Single, "binding-a"), + network_resource_project(IncusMode::Single, "universe-a"), None ); assert_eq!( - network_resource_project(IncusMode::Cluster, "binding-a"), - Some("binding-a") + network_resource_project(IncusMode::Cluster, "universe-a"), + Some("universe-a") ); } + #[test] + fn the_recorded_idle_policy_is_read_back() { + let owner = policy::tests::owner("u", "e"); + let mut config = ownership_config(&owner); + config.insert("user.lightspeed.template".to_owned(), "t".to_owned()); + let idle = IdlePolicy { + pause_after_ms: Some(600_000), + close_after_ms: Some(86_400_000), + ..IdlePolicy::default() + }; + config.insert( + "user.lightspeed.idle".to_owned(), + serde_json::to_string(&idle).unwrap(), + ); + let name = policy::instance_name(&owner); + let owned = owned_from_instance(instance(&name, config.clone()), InstanceState::default()) + .expect("owned"); + assert_eq!(owned.idle, Some(idle)); + + config.insert("user.lightspeed.idle".to_owned(), "not json".to_owned()); + let owned = owned_from_instance(instance(&name, config), InstanceState::default()) + .expect("an undecodable idle policy keeps ownership"); + assert_eq!(owned.idle, None); + } + #[test] fn managed_ipv4_ignores_guest_created_bridges() { let state = InstanceState { @@ -1620,53 +1393,18 @@ mod tests { } #[test] - fn accepted_power_changes_do_not_report_stale_pre_operation_states() { - let target = |status| OwnedTarget { - target_id: "t".into(), - name: "t".to_owned(), - universe_id: "u".to_owned(), - binding_id: "b".to_owned(), - environment_id: "e".to_owned(), - incarnation_id: "i".to_owned(), - request_id: "r".to_owned(), - template_id: "tpl".to_owned(), - image_fingerprint: "img".to_owned(), - adoption_source: None, - status, - ipv4_address: None, - ingress_hostname: None, - ingress_port: None, - location: None, - }; - assert_eq!( - target_after_accepted_freeze(target(ProviderTargetStatus::Ready)).status, - ProviderTargetStatus::Paused - ); - assert_eq!( - target_after_accepted_resume(target(ProviderTargetStatus::Paused)).status, - ProviderTargetStatus::Starting - ); + fn incus_statuses_reduce_to_provider_statuses() { assert_eq!( - target_after_accepted_resume(target(ProviderTargetStatus::Ready)).status, - ProviderTargetStatus::Ready + InstanceStatus::from_incus("Running"), + InstanceStatus::Running ); assert_eq!( - target_after_accepted_stop(target(ProviderTargetStatus::Paused)).status, - ProviderTargetStatus::Stopped + InstanceStatus::from_incus("Freezing"), + InstanceStatus::Frozen ); + assert_eq!(InstanceStatus::from_incus("Error"), InstanceStatus::Error); + assert_eq!(InstanceStatus::from_incus("Pending"), InstanceStatus::Other); assert!(SUPPORTED_POWER_STATES.contains(&PowerState::Paused)); assert!(!SUPPORTED_POWER_STATES.contains(&PowerState::Suspended)); } - - #[test] - fn accepted_start_does_not_report_a_stale_stopped_state() { - assert_eq!( - status_after_accepted_start(ProviderTargetStatus::Stopped), - ProviderTargetStatus::Starting - ); - assert_eq!( - status_after_accepted_start(ProviderTargetStatus::Ready), - ProviderTargetStatus::Ready - ); - } } diff --git a/crates/environment-provider-incus/src/ingress.rs b/crates/environment-provider-incus/src/ingress.rs index 5373c1328..43c4b568b 100644 --- a/crates/environment-provider-incus/src/ingress.rs +++ b/crates/environment-provider-incus/src/ingress.rs @@ -1,5 +1,7 @@ //! Provider-owned public hostname allocation. +use environment_protocol::routes::PublicUrl; + use crate::{Config, incus::OwnedTarget, policy}; pub fn hostname(target: &OwnedTarget, public_base_url: &str) -> anyhow::Result { @@ -10,15 +12,14 @@ pub fn hostname(target: &OwnedTarget, public_base_url: &str) -> anyhow::Result anyhow::Result { +pub fn public_endpoint(config: &Config, hostname: &str) -> anyhow::Result { let ingress = config .ingress .as_ref() @@ -26,36 +27,31 @@ pub fn public_endpoint(config: &Config, hostname: &str) -> anyhow::Result anyhow::Result<(String, String)> { +/// The hostname the provider allocates for the target. +pub fn allocated_hostname(config: &Config, target: &OwnedTarget) -> anyhow::Result { let ingress = config .ingress .as_ref() .ok_or_else(|| anyhow::anyhow!("provider ingress is not configured"))?; - let hostname = hostname(target, &ingress.public_base_url)?; - Ok((hostname.clone(), public_endpoint(config, &hostname)?)) + hostname(target, &ingress.public_base_url) } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - use environment_protocol::{control::targets::ProviderTargetStatus, shared::ProviderTargetId}; + use crate::incus::InstanceStatus; - fn target() -> OwnedTarget { + pub(crate) fn target() -> OwnedTarget { + let owner = policy::tests::owner("universe-a", "environment-a"); OwnedTarget { - target_id: ProviderTargetId::new("target-a"), - name: "target-a".to_owned(), - universe_id: "universe-a".to_owned(), - binding_id: "binding-a".to_owned(), - environment_id: "environment-a".to_owned(), - incarnation_id: "incarnation-a".to_owned(), - request_id: "request-a".to_owned(), + target_id: policy::instance_name(&owner).try_into().unwrap(), + owner, template_id: "template-a".to_owned(), - image_fingerprint: "fingerprint-a".to_owned(), - adoption_source: None, - status: ProviderTargetStatus::Ready, + idle: None, + status: InstanceStatus::Running, ipv4_address: Some("10.0.0.2".to_owned()), ingress_hostname: None, ingress_port: None, diff --git a/crates/environment-provider-incus/src/lib.rs b/crates/environment-provider-incus/src/lib.rs index 26cf2add2..3e9cc1e6b 100644 --- a/crates/environment-provider-incus/src/lib.rs +++ b/crates/environment-provider-incus/src/lib.rs @@ -1,5 +1,6 @@ pub mod config; pub mod edge; +mod idle; pub mod incus; pub mod ingress; pub mod policy; diff --git a/crates/environment-provider-incus/src/policy.rs b/crates/environment-provider-incus/src/policy.rs index e0bdfa7c8..0b72f7b93 100644 --- a/crates/environment-provider-incus/src/policy.rs +++ b/crates/environment-provider-incus/src/policy.rs @@ -1,9 +1,13 @@ use sha2::{Digest as _, Sha256}; -use environment_protocol::control::targets::ProviderBindingContext; +use environment_protocol::{control::target::TargetOwner, ids::UniverseId}; pub const META_PREFIX: &str = "user.lightspeed."; +/// Longest Incus instance, project or network name component the provider +/// addresses in an Incus API path. +const MAX_INCUS_NAME_BYTES: usize = 63; + pub fn stable_component(kind: &str, parts: &[&str]) -> String { let mut hash = Sha256::new(); hash.update(kind.as_bytes()); @@ -14,42 +18,59 @@ pub fn stable_component(kind: &str, parts: &[&str]) -> String { hex(&hash.finalize()[..10]) } -pub fn project_name(binding: &ProviderBindingContext) -> String { - format!( - "ls-{}", - stable_component("binding", &[&binding.universe_id, &binding.binding_id]) - ) +/// The isolation unit of every Incus resource the provider creates for a +/// universe: its project, network, ACL and profile all derive from it. +fn universe_component(universe_id: &UniverseId) -> String { + stable_component("universe", &[universe_id.as_str()]) } -pub fn network_name(binding: &ProviderBindingContext) -> String { - let component = stable_component("binding", &[&binding.universe_id, &binding.binding_id]); +pub fn project_name(universe_id: &UniverseId) -> String { + format!("ls-{}", universe_component(universe_id)) +} + +pub fn network_name(universe_id: &UniverseId) -> String { // A managed bridge's Incus network name also becomes its Linux interface // name, whose hard limit is 15 bytes. Keep 48 bits of the scoped digest // while making the resource kind visible within that limit. - format!("ls{}n", &component[..12]) + format!("ls{}n", &universe_component(universe_id)[..12]) } -pub fn profile_name(binding: &ProviderBindingContext) -> String { - format!("{}-vm", project_name(binding)) + +pub fn profile_name(universe_id: &UniverseId) -> String { + format!("{}-vm", project_name(universe_id)) } -pub fn acl_name(binding: &ProviderBindingContext) -> String { - format!("{}-acl", project_name(binding)) + +pub fn acl_name(universe_id: &UniverseId) -> String { + format!("{}-acl", project_name(universe_id)) } -pub fn instance_name( - universe_id: &str, - binding_id: &str, - environment_id: &str, - incarnation_id: &str, -) -> String { +/// The one instance name, and therefore `targetId`, an owner can hold. The +/// provider's owner index is this deterministic name inside the universe's +/// project, so create and close are idempotent per owner without any +/// provider-side state. +pub fn instance_name(owner: &TargetOwner) -> String { format!( "ls-{}", stable_component( "instance", - &[universe_id, binding_id, environment_id, incarnation_id] + &[owner.universe_id.as_str(), owner.environment_id.as_str()] ) ) } +/// True for a string the provider may place in an Incus API path as an +/// instance name: 1 to 63 ASCII letters, digits or hyphens, not starting or +/// ending with a hyphen. Every `targetId` this provider assigns qualifies; +/// any other opaque id names nothing here. +pub fn is_instance_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_INCUS_NAME_BYTES + && !value.starts_with('-') + && !value.ends_with('-') + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') +} + fn hex(bytes: &[u8]) -> String { const TABLE: &[u8; 16] = b"0123456789abcdef"; let mut output = String::with_capacity(bytes.len() * 2); @@ -61,27 +82,46 @@ fn hex(bytes: &[u8]) -> String { } #[cfg(test)] -mod tests { +pub(crate) mod tests { + use environment_protocol::ids::EnvironmentId; + use super::*; + + pub(crate) fn owner(universe: &str, environment: &str) -> TargetOwner { + TargetOwner { + universe_id: UniverseId::new(universe).unwrap(), + environment_id: EnvironmentId::new(environment).unwrap(), + } + } + #[test] fn names_are_stable_and_scoped() { - let a = instance_name("u", "b", "e", "i"); - assert_eq!(a, instance_name("u", "b", "e", "i")); - assert_ne!(a, instance_name("u2", "b", "e", "i")); + let a = instance_name(&owner("u", "e")); + assert_eq!(a, instance_name(&owner("u", "e"))); + assert_ne!(a, instance_name(&owner("u2", "e"))); + assert_ne!(a, instance_name(&owner("u", "e2"))); + assert!(is_instance_name(&a)); + } + + #[test] + fn universe_resources_share_one_isolation_unit() { + let universe = UniverseId::new("0d3d9e5e-2428-4e60-b66e-8d4520f64e5d").unwrap(); + let other = UniverseId::new("61092591-cd39-4504-b845-a817e3d9cb71").unwrap(); + + assert_eq!(network_name(&universe).len(), 15); + assert_ne!(network_name(&universe), network_name(&other)); + assert_ne!(project_name(&universe), project_name(&other)); + assert!(profile_name(&universe).starts_with(&project_name(&universe))); + assert!(acl_name(&universe).starts_with(&project_name(&universe))); + // The instance name never exposes the universe it belongs to. + assert!(!instance_name(&owner(universe.as_str(), "e")).contains(universe.as_str())); } #[test] - fn binding_network_names_fit_the_linux_interface_limit() { - let binding = ProviderBindingContext { - universe_id: "0d3d9e5e-2428-4e60-b66e-8d4520f64e5d".to_owned(), - binding_id: "hz02-incus".to_owned(), - }; - let other = ProviderBindingContext { - universe_id: "61092591-cd39-4504-b845-a817e3d9cb71".to_owned(), - binding_id: "hz02-incus".to_owned(), - }; - - assert_eq!(network_name(&binding).len(), 15); - assert_ne!(network_name(&binding), network_name(&other)); + fn only_incus_names_reach_an_api_path() { + assert!(is_instance_name("ls-0123456789abcdef0123")); + for value in ["", "-a", "a-", "a/b", "a?b", "default/pod", &"a".repeat(64)] { + assert!(!is_instance_name(value), "{value}"); + } } } diff --git a/crates/environment-provider-incus/src/relay.rs b/crates/environment-provider-incus/src/relay.rs index 6ac992bd1..c395b60b7 100644 --- a/crates/environment-provider-incus/src/relay.rs +++ b/crates/environment-provider-incus/src/relay.rs @@ -1,49 +1,123 @@ //! On-demand provider data connections. //! //! Lightspeed opens one provider WebSocket for a specific current target. The -//! provider validates ownership, dials the target's private envd, and proxies -//! the connection until it closes or becomes idle. There is no provider-initiated -//! connection and no provider route registry in Lightspeed. +//! provider validates the credential and ownership, wakes the target when it +//! is powered down, dials the target's private envd, and proxies the +//! connection until it closes or becomes idle. There is no +//! provider-initiated connection and no provider route registry in +//! Lightspeed. +//! +//! Every data connection, from the dial that may wake the target to the +//! relay's end, holds a [`RelayHold`] on its instance. The idle loop never +//! powers down or closes an instance while one is held. +//! +//! The relay is an intermediary under the protocol's liveness rule: it +//! forwards pings and pongs, counts every frame in either direction as +//! activity, and never idle-closes sooner than `relayIdleSeconds`, which +//! config loading keeps at or above the protocol minimum. Both legs carry the +//! protocol's message and frame limit. -use std::time::Duration; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, +}; use anyhow::Context as _; use axum::extract::ws::{Message as AxumMessage, WebSocket}; +use environment_protocol::limits::MAX_MESSAGE_BYTES; use futures_util::{SinkExt as _, StreamExt as _}; use tokio::net::TcpStream; use tokio_tungstenite::{ - MaybeTlsStream, WebSocketStream, connect_async, tungstenite::Message as TungsteniteMessage, + MaybeTlsStream, WebSocketStream, connect_async_with_config, + tungstenite::{Message as TungsteniteMessage, protocol::WebSocketConfig}, }; use crate::{Config, incus::OwnedTarget}; pub type GuestSocket = WebSocketStream>; +/// Data connections in progress, by instance name. +#[derive(Clone, Debug, Default)] +pub struct Relays(Arc>>); + +impl Relays { + /// Count one data connection to `name` until the hold is dropped. + pub fn hold(&self, name: &str) -> RelayHold { + *self.lock().entry(name.to_owned()).or_default() += 1; + RelayHold { + relays: self.clone(), + name: name.to_owned(), + } + } + + /// True while any data connection to `name` is held. + pub fn active(&self, name: &str) -> bool { + self.lock().get(name).is_some_and(|count| *count > 0) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.0 + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +/// One counted data connection; see [`Relays::hold`]. +#[derive(Debug)] +pub struct RelayHold { + relays: Relays, + name: String, +} + +impl Drop for RelayHold { + fn drop(&mut self) { + let mut relays = self.relays.lock(); + if let Some(count) = relays.get_mut(&self.name) { + *count -= 1; + if *count == 0 { + relays.remove(&self.name); + } + } + } +} + +/// The protocol's message limit, applied to both relay legs. +pub fn websocket_config() -> WebSocketConfig { + WebSocketConfig::default() + .max_message_size(Some(MAX_MESSAGE_BYTES as usize)) + .max_frame_size(Some(MAX_MESSAGE_BYTES as usize)) +} + pub async fn dial_guest(config: &Config, target: &OwnedTarget) -> anyhow::Result { + dial_guest_within( + config, + target, + Duration::from_secs(config.dial_timeout_seconds), + ) + .await +} + +/// Dial the guest's envd listener, giving up after `timeout`. +pub async fn dial_guest_within( + config: &Config, + target: &OwnedTarget, + timeout: Duration, +) -> anyhow::Result { let address = target .ipv4_address .as_deref() .ok_or_else(|| anyhow::anyhow!("target has no private IPv4 address"))?; let endpoint = format!("ws://{address}:{}/", config.envd_port); let (socket, _) = tokio::time::timeout( - Duration::from_secs(config.dial_timeout_seconds), - connect_async(endpoint), + timeout, + connect_async_with_config(endpoint, Some(websocket_config()), true), ) .await .context("envd dial timeout")??; Ok(socket) } -pub async fn probe_guest(config: &Config, target: &OwnedTarget) -> bool { - match dial_guest(config, target).await { - Ok(mut socket) => { - let _ = socket.close(None).await; - true - } - Err(_) => false, - } -} - pub async fn proxy(config: Config, mut lightspeed: WebSocket, mut guest: GuestSocket) { let idle = Duration::from_secs(config.relay_idle_seconds); proxy_with_idle(idle, &mut lightspeed, &mut guest).await; @@ -124,7 +198,22 @@ mod tests { use super::*; - #[tokio::test] + #[test] + fn relay_holds_count_per_instance_until_dropped() { + let relays = Relays::default(); + assert!(!relays.active("a")); + let first = relays.hold("a"); + let second = relays.hold("a"); + assert!(relays.active("a")); + assert!(!relays.active("b")); + drop(first); + assert!(relays.active("a")); + drop(second); + assert!(!relays.active("a")); + assert!(relays.lock().is_empty()); + } + + #[tokio::test(flavor = "current_thread")] async fn relay_closes_guest_when_lightspeed_disappears_without_a_close_frame() { let guest_listener = TcpListener::bind("127.0.0.1:0") .await @@ -144,7 +233,7 @@ mod tests { } false }); - let (guest, _) = connect_async(format!("ws://{guest_address}")) + let (guest, _) = tokio_tungstenite::connect_async(format!("ws://{guest_address}")) .await .expect("connect guest"); let guest = Arc::new(Mutex::new(Some(guest))); @@ -178,7 +267,7 @@ mod tests { .expect("relay server") }); - let (lightspeed, _) = connect_async(format!("ws://{relay_address}")) + let (lightspeed, _) = tokio_tungstenite::connect_async(format!("ws://{relay_address}")) .await .expect("connect lightspeed"); drop(lightspeed); @@ -190,4 +279,82 @@ mod tests { relay_server.abort(); assert!(saw_close, "relay dropped guest without a close frame"); } + + #[tokio::test(flavor = "current_thread")] + async fn relay_forwards_pings_and_counts_them_as_activity() { + let guest_listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("guest listener"); + let guest_address = guest_listener.local_addr().expect("guest address"); + let guest_observer = tokio::spawn(async move { + let (stream, _) = guest_listener.accept().await.expect("guest accept"); + let mut socket = tokio_tungstenite::accept_async(stream) + .await + .expect("guest websocket"); + let mut pings = 0; + while let Some(message) = socket.next().await { + match message { + Ok(TungsteniteMessage::Ping(_)) => pings += 1, + Ok(TungsteniteMessage::Text(text)) => return Some((pings, text.to_string())), + Ok(_) => {} + Err(_) => return None, + } + } + None + }); + let (guest, _) = tokio_tungstenite::connect_async(format!("ws://{guest_address}")) + .await + .expect("connect guest"); + let guest = Arc::new(Mutex::new(Some(guest))); + let relay_listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("relay listener"); + let relay_address = relay_listener.local_addr().expect("relay address"); + let app = Router::new().route( + "/", + get(move |upgrade: WebSocketUpgrade| { + let guest = guest.clone(); + async move { + let mut guest = guest.lock().await.take().expect("one relay connection"); + upgrade + .on_upgrade(move |mut socket| async move { + proxy_with_idle(Duration::from_millis(300), &mut socket, &mut guest) + .await; + }) + .into_response() + } + }), + ); + let relay_server = tokio::spawn(async move { + axum::serve(relay_listener, app) + .await + .expect("relay server") + }); + + let (mut lightspeed, _) = tokio_tungstenite::connect_async(format!("ws://{relay_address}")) + .await + .expect("connect lightspeed"); + // Eight pings 100 ms apart keep the 300 ms idle timer from firing + // across 800 ms with no data frame. + for _ in 0..8 { + lightspeed + .send(TungsteniteMessage::Ping(vec![1].into())) + .await + .expect("ping"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + lightspeed + .send(TungsteniteMessage::Text("after pings".into())) + .await + .expect("text"); + + let observed = tokio::time::timeout(Duration::from_secs(2), guest_observer) + .await + .expect("guest timeout") + .expect("guest observer"); + relay_server.abort(); + let (pings, text) = observed.expect("the relay closed the guest leg while pings flowed"); + assert_eq!(pings, 8); + assert_eq!(text, "after pings"); + } } diff --git a/crates/environment-provider-incus/src/server.rs b/crates/environment-provider-incus/src/server.rs index f6090c2e3..f543f8e6d 100644 --- a/crates/environment-provider-incus/src/server.rs +++ b/crates/environment-provider-incus/src/server.rs @@ -1,64 +1,184 @@ -use std::{collections::BTreeMap, sync::Arc}; +//! The provider's listener: the controller plane on `/control` and the data +//! routes under `/routes`, both behind the provider credential when one is +//! configured. +//! +//! Every target method answers promptly with the current observation. Slow +//! Incus work (an image pull, a boot, a graceful shutdown) runs as one +//! background job per target, serialized per target, and later calls observe +//! its progress: a running create reports `provisioning`, a running close +//! `closing`, and a failed create `failed` with its error as the `message`. +//! Job state is in memory; Incus itself, read through the deterministic owner +//! name, is the record of every target. +//! +//! The provider owns its targets' power. `target/setPower` serves manual +//! changes, the idle loop ([`crate::idle`]) powers idle targets down and +//! closes them on its own clock, and a data route dial wakes a powered-down +//! target before it relays. + +use std::{ + collections::{BTreeMap, HashMap}, + panic::AssertUnwindSafe, + sync::{ + Arc, Mutex as StdMutex, + atomic::{AtomicU64, Ordering}, + }, + time::{Duration, Instant}, +}; use axum::{ Json, Router, extract::{ Path, State, WebSocketUpgrade, - ws::{Message, WebSocket}, + ws::{Message, WebSocket, rejection::WebSocketUpgradeRejection}, }, - http::StatusCode, + http::{HeaderMap, HeaderValue, StatusCode, header}, response::{IntoResponse, Response}, routing::get, }; use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, control::{ handshake::{ ControllerCapabilities, ControllerInitializeParams, ControllerInitializeResponse, }, - ingress::{ - EnsureIngressParams, IngressResponse, ProviderIngressStatus, RemoveIngressParams, + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + CloseTargetParams, CreateTargetParams, GetTargetParams, IngressObservation, PowerState, + SetTargetPowerParams, TargetObservation, TargetOwner, TargetPhase, TargetRef, + TargetResponse, }, - methods::*, - targets::*, + template::{EnvironmentTemplate, ListTemplatesParams, ListTemplatesResponse}, }, - error::{EnvironmentProtocolError, EnvironmentProtocolErrorCode}, - shared::{ - CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentScope, ImplementationInfo, + error::{BusyReason, Capability, PreconditionReason, ProtocolError}, + handshake::{ImplementationInfo, Plane}, + idle::IdleAction, + ids::{EnvironmentId, ProviderTargetId, TemplateId, UniverseId}, + jsonrpc::{self, Request, Response as RpcResponse}, + limits::{MAX_DATA_ROUTE_HOLD_MS, MAX_MESSAGE_BYTES, RETRY_AFTER_MAX_SECS}, + manifest::{ + self, ControllerInitialize, IngressEnsure, IngressRemove, MessageKind, TargetClose, + TargetCreate, TargetGet, TargetSetPower, TemplateList, }, + metadata::{DescriptiveText, Metadata, validate_metadata_entry}, + routes::DataRouteRefusal, +}; +use futures_util::{FutureExt as _, SinkExt as _, StreamExt as _, future::BoxFuture}; +use serde::Serialize; +use serde_json::Value; +use tokio::sync::{Mutex as AsyncMutex, Semaphore, mpsc, watch}; + +use crate::{ + Config, IncusBackend, idle, + incus::{InstanceStatus, OwnedTarget, SUPPORTED_POWER_STATES, unsupported_idle_stage}, + ingress, policy, + relay::{self, GuestSocket, Relays}, }; -use futures_util::{StreamExt as _, stream}; -use serde::de::DeserializeOwned; -use serde_json::{Value, json}; -use crate::{Config, IncusBackend, incus::OwnedTarget, ingress, policy, relay}; +/// How long a mutation waits for its background job before it answers with +/// the current observation. +const PROMPT_WAIT: Duration = Duration::from_secs(5); + +/// Requests one controller connection may have in flight before `busy`. +const MAX_CONCURRENT_REQUESTS: usize = 32; + +const _: () = assert!( + MAX_CONCURRENT_REQUESTS >= environment_protocol::limits::MIN_CONCURRENT_REQUESTS as usize +); + +/// `Retry-After` of a data route's 503 refusal: about one guest boot step. +const RETRY_AFTER_SECS: u64 = 2; -type BindingKey = (String, String); -type BindingLock = Arc>; -type BindingLocks = Arc>>; +const _: () = assert!(RETRY_AFTER_SECS >= 1 && RETRY_AFTER_SECS <= RETRY_AFTER_MAX_SECS); + +/// Bound on one guest dial attempt while a data route dial waits, so the +/// wait re-reads the instance between attempts. +const DIAL_ATTEMPT: Duration = Duration::from_secs(1); + +/// Pause between two looks at the instance while a data route dial waits. +const HOLD_POLL: Duration = Duration::from_millis(250); #[derive(Clone)] -struct App { - config: Config, - backend: B, - binding_locks: BindingLocks, +pub(crate) struct App { + pub(crate) config: Config, + pub(crate) backend: B, + pub(crate) work: Arc, + pub(crate) relays: Relays, + /// When data route dials last woke instances. + pub(crate) wakes: idle::Wakes, + initialize: Arc, + /// How long a data route dial may wait for its target before it refuses. + data_route_hold: Duration, +} + +impl App { + pub(crate) fn new(config: Config, backend: B) -> Self { + let initialize = ControllerInitializeResponse { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + plane: Plane::Controller, + server_info: server_info(), + capabilities: ControllerCapabilities { + power: true, + ingress: config.ingress.is_some(), + }, + }; + let wakes = idle::Wakes::new(idle::wake_grace(config.idle_check_seconds)); + Self { + config, + backend, + work: Arc::default(), + relays: Relays::default(), + wakes, + initialize: Arc::new(initialize), + data_route_hold: Duration::from_millis(MAX_DATA_ROUTE_HOLD_MS), + } + } +} + +pub(crate) fn server_info() -> ImplementationInfo { + let info = ImplementationInfo::new("lightspeed-provider-incus") + .expect("the provider name is a valid implementation name"); + let info = info + .clone() + .with_version(release_info::VERSION) + .unwrap_or(info); + let info = info + .clone() + .with_git_sha(release_info::GIT_SHA) + .unwrap_or(info); + info.clone() + .with_target(release_info::TARGET) + .unwrap_or(info) } pub async fn serve(config: Config, backend: B) -> anyhow::Result<()> { let listener = tokio::net::TcpListener::bind(config.controller_listen).await?; - let app = Router::new() + serve_on(listener, config, backend).await +} + +/// Serve on an already bound listener, with the idle loop running beside +/// it. +pub async fn serve_on( + listener: tokio::net::TcpListener, + config: Config, + backend: B, +) -> anyhow::Result<()> { + let app = App::new(config, backend); + let idle_loop = tokio::spawn(idle::run(app.clone())); + let served = axum::serve(listener, router(app)).await; + idle_loop.abort(); + served?; + Ok(()) +} + +fn router(app: App) -> Router { + Router::new() .route("/health", get(health::)) - .route("/control", get(upgrade::)) + .route("/control", get(control_upgrade::)) .route( - "/routes/{universe}/{binding}/{environment}/{incarnation}/{target}", + "/routes/{universe}/{environment}/{target}", get(data_upgrade::), ) - .with_state(App { - config, - backend, - binding_locks: Arc::default(), - }); - axum::serve(listener, app).await?; - Ok(()) + .with_state(app) } async fn health(State(app): State>) -> Response { @@ -68,151 +188,371 @@ async fn health(State(app): State>) -> Response { } } -async fn upgrade( +/// True when no credential is configured, or when the request carries it as +/// a bearer token. +fn authorized(config: &Config, headers: &HeaderMap) -> bool { + let Some(credential) = &config.credential else { + return true; + }; + headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .is_some_and(|token| credential.matches(token)) +} + +fn unauthorized() -> Response { + ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, "Bearer")], + ) + .into_response() +} + +fn with_limits(upgrade: WebSocketUpgrade) -> WebSocketUpgrade { + upgrade + .max_message_size(MAX_MESSAGE_BYTES as usize) + .max_frame_size(MAX_MESSAGE_BYTES as usize) +} + +async fn control_upgrade( State(app): State>, - upgrade: WebSocketUpgrade, + headers: HeaderMap, + upgrade: Result, ) -> Response { - upgrade - .on_upgrade(move |socket| connection(app, socket)) - .into_response() + if !authorized(&app.config, &headers) { + return unauthorized(); + } + match upgrade { + Ok(upgrade) => with_limits(upgrade) + .on_upgrade(move |socket| connection(app, socket)) + .into_response(), + Err(rejection) => rejection.into_response(), + } } +/// A target's data route. A powered-down target is woken first; the dial +/// then waits, at most the protocol's hold bound, until envd answers, and +/// otherwise refuses with a typed [`DataRouteRefusal`]. The whole dial and +/// the relay after it hold the instance's [`RelayHold`](relay::RelayHold), +/// so the idle loop leaves it alone. async fn data_upgrade( State(app): State>, - Path((universe, binding_id, environment, incarnation, target_id)): Path<( - String, - String, - String, - String, - String, - )>, - upgrade: WebSocketUpgrade, + Path((universe, environment, target_id)): Path<(String, String, String)>, + headers: HeaderMap, + upgrade: Result, ) -> Response { - let binding = ProviderBindingContext { - universe_id: universe.clone(), - binding_id: binding_id.clone(), + if !authorized(&app.config, &headers) { + return unauthorized(); + } + let owner = match (UniverseId::new(universe), EnvironmentId::new(environment)) { + (Ok(universe_id), Ok(environment_id)) => TargetOwner { + universe_id, + environment_id, + }, + _ => return StatusCode::NOT_FOUND.into_response(), }; - let expected = policy::instance_name(&universe, &binding_id, &environment, &incarnation); - if expected != target_id { - return StatusCode::CONFLICT.into_response(); - } - let target_id = environment_protocol::shared::ProviderTargetId::new(target_id); - let target = match app.backend.get_owned(&binding, &target_id).await { - Ok(Some(target)) => target, - Ok(None) => return StatusCode::NOT_FOUND.into_response(), - Err(_) => return StatusCode::BAD_GATEWAY.into_response(), + let upgrade = match upgrade { + Ok(upgrade) => upgrade, + Err(rejection) => return rejection.into_response(), }; - if verify_binding(&target, &binding).is_err() - || target.environment_id != environment - || target.incarnation_id != incarnation - { - return StatusCode::CONFLICT.into_response(); + // Any other id names no target this owner can hold. + if target_id != policy::instance_name(&owner) { + return refuse(DataRouteRefusal::Gone {}); + } + let hold = app.relays.hold(&target_id); + let mut woke = false; + let dialed = tokio::time::timeout( + app.data_route_hold, + wake_and_dial(&app, &owner, &target_id, &mut woke), + ) + .await; + // A woken instance gets the wake grace once this connection ends, so + // the caller's next dial, or its first `use`, arrives before the idle + // loop looks again. + if woke && !matches!(dialed, Ok(Ok(_))) { + app.wakes.record(&target_id, Instant::now()); + } + match dialed { + Ok(Ok(guest)) => { + let config = app.config.clone(); + let wakes = woke.then(|| app.wakes.clone()); + with_limits(upgrade) + .on_upgrade(move |socket| async move { + let _hold = hold; + relay::proxy(config, socket, guest).await; + if let Some(wakes) = wakes { + wakes.record(&target_id, Instant::now()); + } + }) + .into_response() + } + Ok(Err(DialRefusal::Refused(refusal))) => refuse(refusal), + Ok(Err(DialRefusal::Backend)) => StatusCode::BAD_GATEWAY.into_response(), + Err(_) if woke => refuse(DataRouteRefusal::Waking {}), + Err(_) => refuse(DataRouteRefusal::Booting {}), } - let guest = match relay::dial_guest(&app.config, &target).await { - Ok(guest) => guest, - Err(_) => return StatusCode::SERVICE_UNAVAILABLE.into_response(), - }; - let config = app.config.clone(); - upgrade - .max_message_size(64 * 1024 * 1024) - .on_upgrade(move |socket| relay::proxy(config, socket, guest)) - .into_response() } -async fn connection(app: App, mut socket: WebSocket) { - while let Some(Ok(message)) = socket.recv().await { - match message { - Message::Text(text) => { - let response = dispatch(&app, &text).await; - if socket - .send(Message::Text(response.to_string().into())) - .await - .is_err() - { - break; +/// Why a data route dial does not upgrade. +enum DialRefusal { + Refused(DataRouteRefusal), + /// Incus could not be read; answered 502 without a body. + Backend, +} + +/// Wake the owner's instance when it is powered down, then dial its envd +/// until it answers. Runs until it succeeds or meets a terminal state; the +/// caller bounds it. `woke` records that this dial started a wake. +async fn wake_and_dial( + app: &App, + owner: &TargetOwner, + name: &str, + woke: &mut bool, +) -> Result { + loop { + let job = app + .work + .view(name) + .filter(Slot::running) + .map(|slot| slot.kind); + if job == Some(WorkKind::Close) { + return Err(DialRefusal::Refused(DataRouteRefusal::Gone {})); + } + // A wake runs only after the power-down it superseded, which the + // slot no longer shows: until the wake finished, the instance may + // still read running while it is about to freeze or stop. + if job == Some(WorkKind::Power(PowerState::Running)) { + *woke = true; + tokio::time::sleep(HOLD_POLL).await; + continue; + } + let target = match app.backend.get_owned(&owner.universe_id, name).await { + Ok(Some(target)) if target.owner == *owner => Some(target), + // A create still running has no instance to show yet. + Ok(_) if job == Some(WorkKind::Create) => None, + Ok(_) => return Err(DialRefusal::Refused(DataRouteRefusal::Gone {})), + Err(_) => return Err(DialRefusal::Backend), + }; + // A create still running starts the instance itself. + if let Some(target) = target.filter(|_| job != Some(WorkKind::Create)) { + let powering_down = + matches!(job, Some(WorkKind::Power(power)) if power != PowerState::Running); + match target.status { + InstanceStatus::Error => { + return Err(DialRefusal::Refused(DataRouteRefusal::Failed { + message: DescriptiveText::lossy(INSTANCE_ERROR) + .expect("the error text is descriptive"), + })); + } + InstanceStatus::Frozen | InstanceStatus::Stopped | InstanceStatus::Stopping => { + wake(app, owner, name); + *woke = true; + } + InstanceStatus::Running | InstanceStatus::Starting if powering_down => { + wake(app, owner, name); + *woke = true; } + InstanceStatus::Running | InstanceStatus::Starting => { + if target.ipv4_address.is_some() { + let attempt = + DIAL_ATTEMPT.min(Duration::from_secs(app.config.dial_timeout_seconds)); + if let Ok(guest) = + relay::dial_guest_within(&app.config, &target, attempt).await + { + return Ok(guest); + } + } + } + InstanceStatus::Other => {} + } + } + tokio::time::sleep(HOLD_POLL).await; + } +} + +/// Power the instance up as its background job, unless a wake is already +/// running. A power-down job still running finishes first, then this one +/// runs; [`wake_and_dial`] dials only once it finished. +fn wake(app: &App, owner: &TargetOwner, name: &str) { + app.work.start( + name, + WorkKind::Power(PowerState::Running), + power_job(&app.backend, &owner.universe_id, name, PowerState::Running), + ); +} + +/// A data route refusal: its status, its JSON body and, for a 503, +/// `Retry-After`. +fn refuse(refusal: DataRouteRefusal) -> Response { + let status = StatusCode::from_u16(refusal.status()).unwrap_or(StatusCode::SERVICE_UNAVAILABLE); + let retryable = refusal.is_retryable(); + let mut response = (status, Json(refusal)).into_response(); + if retryable { + response + .headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from(RETRY_AFTER_SECS)); + } + response +} + +/// One controller connection: the JSON-RPC envelope, the `initialize`-first +/// handshake, and pipelined requests answered in any order by one writer. +async fn connection(app: App, socket: WebSocket) { + let (mut sink, mut stream) = socket.split(); + let (outgoing, mut queue) = mpsc::channel::(MAX_CONCURRENT_REQUESTS * 2); + let writer = tokio::spawn(async move { + while let Some(message) = queue.recv().await { + if sink.send(message).await.is_err() { + break; } - Message::Ping(bytes) => { - let _ = socket.send(Message::Pong(bytes)).await; + } + let _ = sink.close().await; + }); + let permits = Arc::new(Semaphore::new(MAX_CONCURRENT_REQUESTS)); + let mut initialized = false; + while let Some(Ok(frame)) = stream.next().await { + let reply = match frame { + Message::Text(text) => match jsonrpc::parse(text.as_str()) { + Err(rejection) => rejection.into_response(), + Ok(jsonrpc::Message::Request(request)) => { + if request.method == ControllerInitialize::NAME { + Some(initialize(&app, &mut initialized, &request)) + } else if !initialized { + Some(RpcResponse::failure( + Some(request.id), + ProtocolError::not_initialized(), + )) + } else { + match permits.clone().try_acquire_owned() { + Ok(permit) => { + let app = app.clone(); + let outgoing = outgoing.clone(); + tokio::spawn(async move { + let response = handle(&app, request).await; + let _ = outgoing + .send(Message::Text(response.to_text().into())) + .await; + drop(permit); + }); + None + } + Err(_) => Some(RpcResponse::failure( + Some(request.id), + ProtocolError::busy( + Some(BusyReason::RequestLimit), + None, + format!( + "more than {MAX_CONCURRENT_REQUESTS} requests in flight" + ), + ), + )), + } + } + } + // The controller plane defines no notifications, and this + // server sends no requests, so no response answers anything. + Ok(jsonrpc::Message::Notification(_) | jsonrpc::Message::Response(_)) => None, + }, + Message::Binary(_) => Some(RpcResponse::failure( + None, + ProtocolError::invalid_request("binary frames are not part of any plane"), + )), + Message::Ping(payload) => { + if outgoing.send(Message::Pong(payload)).await.is_err() { + break; + } + None } + Message::Pong(_) => None, Message::Close(_) => break, - _ => {} + }; + if let Some(reply) = reply + && outgoing + .send(Message::Text(reply.to_text().into())) + .await + .is_err() + { + break; } } - let _ = socket.send(Message::Close(None)).await; + drop(outgoing); + let _ = writer.await; } -async fn dispatch(app: &App, text: &str) -> Value { - let request: Value = match serde_json::from_str(text) { - Ok(value) => value, - Err(decode_error) => { - return error( - Value::Null, - EnvironmentProtocolErrorCode::InvalidRequest, - decode_error.to_string(), - ); +fn initialize(app: &App, initialized: &mut bool, request: &Request) -> RpcResponse { + let id = Some(request.id.clone()); + if *initialized { + return RpcResponse::failure( + id, + ProtocolError::invalid_request("initialize was already called on this connection"), + ); + } + // The version decides before the rest of the params: a peer of another + // version may send another shape. + if let Some(version) = request + .params + .get("protocolVersion") + .and_then(Value::as_u64) + && version != u64::from(CONTROLLER_PROTOCOL_VERSION) + { + return RpcResponse::failure( + id, + ProtocolError::version_mismatch( + CONTROLLER_PROTOCOL_VERSION, + u32::try_from(version).unwrap_or(u32::MAX), + ), + ); + } + match request.params::() { + Ok(_) => { + *initialized = true; + RpcResponse::success(request.id.clone(), app.initialize.as_ref()) } + Err(error) => RpcResponse::failure(id, error), + } +} + +async fn handle(app: &App, request: Request) -> RpcResponse { + let result = route(app, &request).await; + RpcResponse { + id: Some(request.id), + result, + } +} + +async fn route(app: &App, request: &Request) -> Result { + let Some(spec) = manifest::find(Plane::Controller, &request.method) + .filter(|spec| spec.kind == MessageKind::Request) + else { + return Err(ProtocolError::method_not_found(&request.method)); }; - let id = request.get("id").cloned().unwrap_or(Value::Null); - let method = request - .get("method") - .and_then(Value::as_str) - .unwrap_or_default(); - let params = request.get("params").cloned().unwrap_or_else(|| json!({})); - let result = match method { - INITIALIZE_METHOD => decode::(params) - .and_then(|params| { - if params.protocol_version != CURRENT_PROTOCOL_VERSION { - anyhow::bail!("unsupported protocol version") - } - Ok(ControllerInitializeResponse { - protocol_version: CURRENT_PROTOCOL_VERSION, - capabilities: ControllerCapabilities { - list_templates: true, - list_targets: true, - create_target: true, - adopt_target: true, - get_target: true, - close_target: true, - set_target_power: true, - ingress: app.config.ingress.is_some(), - }, - implementation: ImplementationInfo { - name: "lightspeed-provider-incus".to_owned(), - version: Some(release_info::VERSION.to_owned()), - git_sha: Some(release_info::GIT_SHA.to_owned()), - target: Some(release_info::TARGET.to_owned()), - }, - }) - }) - .and_then(encode), - LIST_TEMPLATES_METHOD => list_templates(app, decode(params)).await.and_then(encode), - LIST_TARGETS_METHOD => list_targets(app, decode(params)).await.and_then(encode), - CREATE_TARGET_METHOD => create_target(app, decode(params)).await.and_then(encode), - ADOPT_TARGET_METHOD => adopt_target(app, decode(params)).await.and_then(encode), - GET_TARGET_METHOD => get_target(app, decode(params)).await.and_then(encode), - CLOSE_TARGET_METHOD => close_target(app, decode(params)).await.and_then(encode), - SET_TARGET_POWER_METHOD => set_target_power(app, decode(params)).await.and_then(encode), - ENSURE_INGRESS_METHOD => ensure_ingress(app, decode(params)).await.and_then(encode), - REMOVE_INGRESS_METHOD => remove_ingress(app, decode(params)).await.and_then(encode), - _ => Err(anyhow::anyhow!("unknown controller method: {method}")), - }; - match result { - Ok(result) => json!({"jsonrpc":"2.0","id":id,"result":result}), - Err(error_value) => error( - id, - EnvironmentProtocolErrorCode::Internal, - error_value.to_string(), - ), + if spec.capability == Some(Capability::Ingress) && !app.initialize.capabilities.ingress { + return Err(ProtocolError::capability_unavailable( + Capability::Ingress, + "this provider has no public ingress configured", + )); + } + match request.method.as_str() { + TemplateList::NAME => encode(list_templates(app, request.params()?)), + TargetCreate::NAME => encode(create_target(app, request.params()?).await?), + TargetGet::NAME => encode(get_target(app, request.params()?).await?), + TargetSetPower::NAME => encode(set_target_power(app, request.params()?).await?), + TargetClose::NAME => encode(close_target(app, request.params()?).await?), + IngressEnsure::NAME => encode(ensure_ingress(app, request.params()?).await?), + IngressRemove::NAME => encode(remove_ingress(app, request.params()?).await?), + _ => Err(ProtocolError::method_not_found(&request.method)), } } -async fn list_templates( - app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let _params = params?; - Ok(ListTemplatesResponse { +fn encode(value: T) -> Result { + serde_json::to_value(value) + .map_err(|error| ProtocolError::internal(format!("encode result: {error}"))) +} + +/// Every universe gets the same provider-wide templates. +fn list_templates(app: &App, _params: ListTemplatesParams) -> ListTemplatesResponse { + ListTemplatesResponse { templates: app .config .templates @@ -221,465 +561,1667 @@ async fn list_templates( template_id: template.template_id.clone(), display_name: template.display_name.clone(), description: template.description.clone(), + power_states: SUPPORTED_POWER_STATES.to_vec(), public_ingress: template.public_ingress, + idle: template.idle.clone(), deprecated: template.deprecated, - metadata: BTreeMap::from([ - ( - "imageFingerprint".to_owned(), - template.image_fingerprint.clone(), - ), - ("cpu".to_owned(), template.cpu.to_string()), - ("memory".to_owned(), template.memory.clone()), - ("disk".to_owned(), template.disk.clone()), - ]), + resources: Some(template.resources.clone()), + metadata: display_metadata([( + "imageFingerprint", + template.image_fingerprint.as_str(), + )]), }) .collect(), - }) + } } -async fn list_targets( +async fn get_target( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let config = app.config.clone(); - let mut targets = stream::iter(app.backend.list_owned(binding).await?) - .map(move |mut target| { - let config = config.clone(); - async move { - observe_daemon_readiness(&config, &mut target).await; - target - } - }) - .buffer_unordered(16) - .collect::>() - .await; - if let Some(status) = params.status { - targets.retain(|target| target.status == status); - } - Ok(ListTargetsResponse { - targets: targets.into_iter().map(summary).collect(), - }) + params: GetTargetParams, +) -> Result { + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + observation(app, owner, &name, &observed) + .map(|target| TargetResponse { target }) + .ok_or_else(|| ProtocolError::not_found(format!("no target exists for owner {name}"))) } -async fn get_target( - app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let mut target = app - .backend - .get_owned(binding, ¶ms.target_id) - .await? - .ok_or_else(|| anyhow::anyhow!("target not found"))?; - verify_binding(&target, binding)?; - observe_daemon_readiness(&app.config, &mut target).await; - Ok(GetTargetResponse { - target: summary(target), - }) +/// The wire path of an idle stage's threshold in `target/create` params. +fn idle_stage_field(action: IdleAction) -> &'static str { + match action { + IdleAction::Pause => "idle.pauseAfterMs", + IdleAction::Suspend => "idle.suspendAfterMs", + IdleAction::Stop => "idle.stopAfterMs", + IdleAction::Close => "idle.closeAfterMs", + } } async fn create_target( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = params.binding.clone(); - let binding_lock = { - let mut locks = app.binding_locks.lock().await; - locks - .entry(( - params.binding.universe_id.clone(), - params.binding.binding_id.clone(), + params: CreateTargetParams, +) -> Result { + if let Some(idle) = ¶ms.idle { + idle.validate() + .map_err(|error| ProtocolError::invalid_field("idle", error.to_string()))?; + if let Some(action) = unsupported_idle_stage(idle) { + return Err(ProtocolError::invalid_field( + idle_stage_field(action), + format!("the Incus provider cannot carry out the {action} stage"), + )); + } + } + let owner = params.owner; + let name = policy::instance_name(&owner); + let observed = observe(app, &owner, &name).await?; + if observed.holds_target() { + // Idempotent per owner: an owner that holds a target gets it back, + // and no creation policy runs again. + return observed_response(app, &owner, &name, &observed); + } + let template = app + .config + .templates + .get(¶ms.template_id) + .ok_or_else(|| { + ProtocolError::not_found(format!( + "template {} is not offered by this provider", + params.template_id )) - .or_default() - .clone() + })?; + if template.deprecated { + return Err(ProtocolError::failed_precondition( + PreconditionReason::TemplateDeprecated, + format!( + "template {} is deprecated and creates no new targets", + template.template_id + ), + )); + } + let template = template.clone(); + // The target keeps the policy it was created with, even when the + // template's default changes later. + let idle = params.idle.or_else(|| template.idle.clone()); + let backend = app.backend.clone(); + let job_owner = owner.clone(); + let job_name = name.clone(); + let job = async move { + let universe_id = &job_owner.universe_id; + if backend.get_owned(universe_id, &job_name).await?.is_some() { + return Ok(()); + } + backend.reconcile_universe(universe_id).await?; + backend.ensure_image(&template.image_fingerprint).await?; + backend + .create_vm(&job_owner, &template, idle.as_ref()) + .await }; - let _binding_guard = binding_lock.lock().await; - let template = app.config.template(¶ms.template_id)?; - app.backend.reconcile_binding(&binding).await?; - app.backend - .ensure_image(&template.image_fingerprint) - .await?; - let expected_target_id = policy::instance_name( - ¶ms.binding.universe_id, - ¶ms.binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); - if let Some(mut target) = app - .backend - .get_owned(&binding, &expected_target_id.clone().into()) + let target = mutate(app, &owner, &name, WorkKind::Create, job.boxed()) .await? - { - verify_create(&target, ¶ms, template)?; - observe_daemon_readiness(&app.config, &mut target).await; - return Ok(CreateTargetResponse { - target: summary(target), + .ok_or_else(|| ProtocolError::unavailable("the created Incus instance is not visible"))?; + Ok(TargetResponse { target }) +} + +async fn set_target_power( + app: &App, + params: SetTargetPowerParams, +) -> Result { + let power = params.power; + if !SUPPORTED_POWER_STATES.contains(&power) { + return Err(ProtocolError::invalid_field( + "power", + format!("this provider cannot move a target to {power}"), + )); + } + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + let current = observation(app, owner, &name, &observed) + .ok_or_else(|| ProtocolError::not_found(format!("target {name} does not exist")))?; + if current.phase != TargetPhase::Active { + return Err(ProtocolError::failed_precondition( + PreconditionReason::TargetNotActive, + format!("target {name} is {:?}, not active", current.phase), + )); + } + if current.power == Some(power) && !observed.slot.as_ref().is_some_and(Slot::running) { + return observed_response(app, owner, &name, &observed); + } + let job = power_job(&app.backend, &owner.universe_id, &name, power); + let target = mutate(app, owner, &name, WorkKind::Power(power), job) + .await? + .ok_or_else(|| ProtocolError::not_found(format!("target {name} disappeared")))?; + Ok(TargetResponse { target }) +} + +async fn close_target( + app: &App, + params: CloseTargetParams, +) -> Result { + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + if !observed.holds_target() { + app.work.forget(&name); + return Ok(TargetResponse { + target: closed(owner), }); } - let existing = app.backend.list_owned(&binding).await?; - if let Some(target) = existing - .iter() - .find(|target| target.request_id == params.request_id) - { - verify_create(target, ¶ms, template)?; - let mut target = target.clone(); - observe_daemon_readiness(&app.config, &mut target).await; - return Ok(CreateTargetResponse { - target: summary(target), + app.work.start( + &name, + WorkKind::Close, + close_job(&app.backend, &owner.universe_id, &name), + ); + wait_for_work(app, &name).await; + let observed = observe(app, owner, &name).await?; + if !observed.holds_target() { + return Ok(TargetResponse { + target: closed(owner), }); } - let mut target = app.backend.create_vm(&binding, template, ¶ms).await?; - verify_create(&target, ¶ms, template)?; - observe_daemon_readiness(&app.config, &mut target).await; - Ok(CreateTargetResponse { - target: summary(target), - }) + let mut target = observation(app, owner, &name, &observed).unwrap_or_else(|| closed(owner)); + // Until the instance is gone, a close answers closing, with the error of + // a failed attempt as its message; the next close tries again. + target.phase = TargetPhase::Closing; + target.power = None; + Ok(TargetResponse { target }) +} + +/// The background job that moves the instance `name` toward `power`. +pub(crate) fn power_job( + backend: &B, + universe_id: &UniverseId, + name: &str, + power: PowerState, +) -> BoxFuture<'static, anyhow::Result<()>> { + let backend = backend.clone(); + let universe_id = universe_id.clone(); + let name = name.to_owned(); + async move { + let target = backend + .get_owned(&universe_id, &name) + .await? + .ok_or_else(|| anyhow::anyhow!("the Incus instance disappeared"))?; + backend.set_power(&target, power).await + } + .boxed() +} + +/// The background job that destroys the instance `name`, if it still +/// exists. +pub(crate) fn close_job( + backend: &B, + universe_id: &UniverseId, + name: &str, +) -> BoxFuture<'static, anyhow::Result<()>> { + let backend = backend.clone(); + let universe_id = universe_id.clone(); + let name = name.to_owned(); + async move { + match backend.get_owned(&universe_id, &name).await? { + Some(target) => backend.delete_vm(&target).await, + None => Ok(()), + } + } + .boxed() } -async fn adopt_target( +async fn ensure_ingress( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = params.binding.clone(); - let binding_lock = { - let mut locks = app.binding_locks.lock().await; - locks - .entry((binding.universe_id.clone(), binding.binding_id.clone())) - .or_default() - .clone() + params: EnsureIngressParams, +) -> Result { + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + let Some(instance) = ingress_subject(app, owner, &name, &observed)? else { + return observed_response(app, owner, &name, &observed); }; - let _binding_guard = binding_lock.lock().await; - app.backend.reconcile_binding(&binding).await?; - let mut target = app.backend.adopt_vm(&binding, ¶ms).await?; - verify_adoption(&target, ¶ms)?; - observe_daemon_readiness(&app.config, &mut target).await; - Ok(AdoptTargetResponse { - target: summary(target), - }) + let port = TemplateId::new(instance.template_id.as_str()) + .ok() + .and_then(|template_id| app.config.templates.get(&template_id)) + .and_then(|template| template.ingress_port) + .ok_or_else(|| { + ProtocolError::failed_precondition( + PreconditionReason::IngressUnsupported, + format!("the template of target {name} has no public ingress"), + ) + })?; + let hostname = ingress::allocated_hostname(&app.config, &instance).map_err(|error| { + ProtocolError::internal(format!("allocate ingress hostname: {error:#}")) + })?; + if instance.ingress_hostname.as_deref() == Some(hostname.as_str()) + && instance.ingress_port == Some(port) + { + return observed_response(app, owner, &name, &observed); + } + let backend = app.backend.clone(); + let universe_id = owner.universe_id.clone(); + let job_name = name.clone(); + let job = async move { + let target = backend + .get_owned(&universe_id, &job_name) + .await? + .ok_or_else(|| anyhow::anyhow!("the Incus instance disappeared"))?; + backend.set_ingress(&target, Some((&hostname, port))).await + }; + let target = mutate( + app, + owner, + &name, + WorkKind::Ingress { enabled: true }, + job.boxed(), + ) + .await? + .ok_or_else(|| ProtocolError::not_found(format!("target {name} disappeared")))?; + Ok(TargetResponse { target }) } -async fn close_target( +async fn remove_ingress( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let expected_target_id = policy::instance_name( - ¶ms.binding.universe_id, - ¶ms.binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); - if params.target_id.as_str() != expected_target_id { - anyhow::bail!("target handle does not match deterministic ownership identity") + params: RemoveIngressParams, +) -> Result { + let owner = ¶ms.target.owner; + let name = resolve(app, ¶ms.target).await?; + let observed = observe(app, owner, &name).await?; + let Some(instance) = ingress_subject(app, owner, &name, &observed)? else { + return observed_response(app, owner, &name, &observed); + }; + if instance.ingress_hostname.is_none() && instance.ingress_port.is_none() { + return observed_response(app, owner, &name, &observed); } - let Some(target) = app.backend.get_owned(binding, ¶ms.target_id).await? else { - return Ok(CloseTargetResponse { - target_id: params.target_id, - status: ProviderTargetStatus::Closed, - }); + let backend = app.backend.clone(); + let universe_id = owner.universe_id.clone(); + let job_name = name.clone(); + let job = async move { + let target = backend + .get_owned(&universe_id, &job_name) + .await? + .ok_or_else(|| anyhow::anyhow!("the Incus instance disappeared"))?; + backend.set_ingress(&target, None).await }; - verify_binding(&target, binding)?; - if target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - { - anyhow::bail!("target ownership does not match close request") - } - app.backend - .delete_vm(binding, &target, params.force) - .await?; - Ok(CloseTargetResponse { - target_id: params.target_id, - status: ProviderTargetStatus::Closed, + let target = mutate( + app, + owner, + &name, + WorkKind::Ingress { enabled: false }, + job.boxed(), + ) + .await? + .ok_or_else(|| ProtocolError::not_found(format!("target {name} disappeared")))?; + Ok(TargetResponse { target }) +} + +/// The instance whose ingress may change, `None` when the target is closing +/// (the caller answers with the observation), or an error when there is no +/// instance yet or at all. +fn ingress_subject( + app: &App, + owner: &TargetOwner, + name: &str, + observed: &Observed, +) -> Result, ProtocolError> { + let current = observation(app, owner, name, observed) + .ok_or_else(|| ProtocolError::not_found(format!("target {name} does not exist")))?; + if current.phase == TargetPhase::Closing { + return Ok(None); + } + observed.instance.clone().map(Some).ok_or_else(|| { + ProtocolError::failed_precondition( + PreconditionReason::TargetNotActive, + format!("target {name} has no instance yet"), + ) }) } -async fn set_target_power( +/// The instance name a reference resolves to. The provider assigns exactly +/// one name per owner; a `targetId` naming any other instance is another +/// owner's target or none at all. +async fn resolve( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let expected_target_id = policy::instance_name( - ¶ms.binding.universe_id, - ¶ms.binding.binding_id, - ¶ms.environment_id, - ¶ms.incarnation_id, - ); - if params.target_id.as_str() != expected_target_id { - anyhow::bail!("target handle does not match deterministic ownership identity") + target: &TargetRef, +) -> Result { + let name = policy::instance_name(&target.owner); + if let Some(target_id) = &target.target_id + && target_id.as_str() != name + { + // A target being created is known by its job before its instance + // exists. + let foreign = app.work.view(target_id.as_str()).is_some() + || app + .backend + .get_owned(&target.owner.universe_id, target_id.as_str()) + .await + .map_err(unavailable)? + .is_some(); + return Err(if foreign { + owner_mismatch(target_id.as_str()) + } else { + ProtocolError::not_found(format!("target {target_id} does not exist")) + }); } - let target = app + Ok(name) +} + +/// What the provider knows about one owner's target. +struct Observed { + instance: Option, + slot: Option, +} + +impl Observed { + /// The owner holds something: an instance or work in progress. + fn holds_target(&self) -> bool { + self.instance.is_some() || self.slot.as_ref().is_some_and(Slot::running) + } +} + +async fn observe( + app: &App, + owner: &TargetOwner, + name: &str, +) -> Result { + let instance = app .backend - .get_owned(binding, ¶ms.target_id) - .await? - .ok_or_else(|| anyhow::anyhow!("target not found"))?; - verify_binding(&target, binding)?; - if target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id + .get_owned(&owner.universe_id, name) + .await + .map_err(unavailable)?; + if let Some(instance) = &instance + && instance.owner != *owner { - anyhow::bail!("target ownership does not match power request") + return Err(owner_mismatch(name)); } - let mut target = app - .backend - .set_power(binding, &target, params.power) - .await?; - observe_daemon_readiness(&app.config, &mut target).await; - Ok(SetTargetPowerResponse { - target: summary(target), + Ok(Observed { + instance, + slot: app.work.view(name), }) } -async fn ensure_ingress( +fn observed_response( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let mut target = app - .backend - .get_owned(binding, ¶ms.target_id) - .await? - .ok_or_else(|| anyhow::anyhow!("target not found"))?; - verify_ingress_target( - &target, - binding, - ¶ms.environment_id, - ¶ms.incarnation_id, - )?; - let template = app - .config - .templates - .get(&target.template_id) - .ok_or_else(|| anyhow::anyhow!("target template is no longer configured"))?; - let port = template - .ingress_port - .ok_or_else(|| anyhow::anyhow!("template does not permit public ingress"))?; - let (hostname, endpoint) = ingress::endpoint(&app.config, &target)?; - target = app - .backend - .set_ingress(binding, &target, Some(&hostname), Some(port)) - .await?; - verify_ingress_target( - &target, - binding, - ¶ms.environment_id, - ¶ms.incarnation_id, - )?; - Ok(IngressResponse { - status: ProviderIngressStatus::Ready, - public_endpoint: Some(endpoint), - }) + owner: &TargetOwner, + name: &str, + observed: &Observed, +) -> Result { + observation(app, owner, name, observed) + .map(|target| TargetResponse { target }) + .ok_or_else(|| ProtocolError::not_found(format!("target {name} does not exist"))) } -async fn remove_ingress( +/// The phase, power and message of an observation. +struct Condition { + phase: TargetPhase, + power: Option, + message: Option, +} + +impl Condition { + fn phase(phase: TargetPhase, message: &str) -> Self { + Self { + phase, + power: None, + message: DescriptiveText::lossy(message), + } + } +} + +/// The observation of a target, or `None` when the owner holds nothing. +/// +/// `targetId` is reported only once the Incus instance exists. Until then +/// the runtime keeps sending the idempotent `target/create`, which starts +/// the job again when it failed or was lost with a provider restart; a +/// failed attempt is reported as `provisioning` with its error as +/// `message`, so the runtime retries it until its provisioning deadline +/// instead of failing at the first Incus hiccup. Whether envd answers is +/// not part of an observation: the runtime's open handshake over the data +/// route proves it. +fn observation( app: &App, - params: anyhow::Result, -) -> anyhow::Result { - let params = params?; - let binding = ¶ms.binding; - let Some(target) = app.backend.get_owned(binding, ¶ms.target_id).await? else { - return Ok(IngressResponse { - status: ProviderIngressStatus::Disabled, - public_endpoint: None, - }); + owner: &TargetOwner, + name: &str, + observed: &Observed, +) -> Option { + let failure = observed.slot.as_ref().and_then(Slot::failure); + let running = observed + .slot + .as_ref() + .filter(|slot| slot.running()) + .map(|slot| slot.kind); + let failed_creation = observed + .slot + .as_ref() + .is_some_and(|slot| !slot.running() && slot.kind == WorkKind::Create); + let condition = match (running, &observed.instance) { + (Some(WorkKind::Close), _) => { + Condition::phase(TargetPhase::Closing, "closing the Incus instance") + } + (Some(WorkKind::Create), _) => Condition::phase( + TargetPhase::Provisioning, + "creating and starting the Incus instance", + ), + (_, Some(instance)) => { + let mut condition = instance_condition(instance); + if condition.message.is_none() { + condition.message = failure.and_then(DescriptiveText::lossy); + } + condition + } + (_, None) if failed_creation => Condition::phase( + TargetPhase::Provisioning, + &format!( + "the last attempt failed and the next target/create retries it: {}", + failure.unwrap_or("unknown error") + ), + ), + (_, None) => return None, }; - verify_ingress_target( - &target, - binding, - ¶ms.environment_id, - ¶ms.incarnation_id, - )?; - app.backend - .set_ingress(binding, &target, None, None) - .await?; - Ok(IngressResponse { - status: ProviderIngressStatus::Disabled, - public_endpoint: None, + let target_id = match &observed.instance { + Some(_) => Some(ProviderTargetId::new(name).ok()?), + None => None, + }; + let ingress = observed + .instance + .as_ref() + .and_then(|instance| instance.ingress_hostname.as_deref()) + .and_then(|hostname| ingress::public_endpoint(&app.config, hostname).ok()) + .map(|endpoint| IngressObservation { endpoint }); + Some(TargetObservation { + target_id, + owner: owner.clone(), + phase: condition.phase, + power: condition.power, + message: condition.message, + ingress, }) } -fn verify_ingress_target( - target: &OwnedTarget, - binding: &ProviderBindingContext, - environment: &str, - incarnation: &str, -) -> anyhow::Result<()> { - verify_binding(target, binding)?; - if target.environment_id != environment || target.incarnation_id != incarnation { - anyhow::bail!("target ownership does not match ingress request") +const INSTANCE_ERROR: &str = "Incus reports the instance in the Error state"; + +/// The phase and power an Incus instance status maps to. A starting VM is +/// reported running and a stopping one stopped: `power` names where the +/// target is going. +fn instance_condition(instance: &OwnedTarget) -> Condition { + let active = |power| Condition { + phase: TargetPhase::Active, + power: Some(power), + message: None, + }; + match instance.status { + InstanceStatus::Running | InstanceStatus::Starting => active(PowerState::Running), + InstanceStatus::Frozen => active(PowerState::Paused), + InstanceStatus::Stopped | InstanceStatus::Stopping => active(PowerState::Stopped), + InstanceStatus::Error => Condition::phase(TargetPhase::Failed, INSTANCE_ERROR), + InstanceStatus::Other => Condition::phase( + TargetPhase::Unknown, + "Incus reports an instance status this provider does not recognize", + ), } - Ok(()) } -fn verify_binding(target: &OwnedTarget, binding: &ProviderBindingContext) -> anyhow::Result<()> { - if target.universe_id != binding.universe_id || target.binding_id != binding.binding_id { - anyhow::bail!("target belongs to another binding") +fn closed(owner: &TargetOwner) -> TargetObservation { + TargetObservation { + target_id: None, + owner: owner.clone(), + phase: TargetPhase::Closed, + power: None, + message: None, + ingress: None, } - Ok(()) } -fn verify_create( - target: &OwnedTarget, - params: &CreateTargetParams, - template: &crate::config::TemplatePolicy, -) -> anyhow::Result<()> { - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - || target.request_id != params.request_id - || target.template_id != template.template_id - || target.image_fingerprint != template.image_fingerprint - { - anyhow::bail!("existing target metadata conflicts with provision request") + +/// Provider display facts; entries outside the metadata bounds are left out. +fn display_metadata<'a>(entries: impl IntoIterator) -> Metadata { + let entries: BTreeMap = entries + .into_iter() + .filter(|(key, value)| validate_metadata_entry(key, value).is_ok()) + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect(); + Metadata::new(entries).unwrap_or_default() +} + +fn unavailable(error: anyhow::Error) -> ProtocolError { + ProtocolError::unavailable(format!("Incus: {error:#}")) +} + +fn owner_mismatch(target_id: &str) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::OwnerMismatch, + format!("target {target_id} belongs to another owner"), + ) +} + +/// Start `job` for the target and wait up to the prompt bound, then +/// observe. +async fn mutate( + app: &App, + owner: &TargetOwner, + name: &str, + kind: WorkKind, + job: BoxFuture<'static, anyhow::Result<()>>, +) -> Result, ProtocolError> { + app.work.start(name, kind, job); + wait_for_work(app, name).await; + let observed = observe(app, owner, name).await?; + Ok(observation(app, owner, name, &observed)) +} + +async fn wait_for_work(app: &App, name: &str) { + if let Some(mut done) = app.work.done(name) { + let _ = tokio::time::timeout(PROMPT_WAIT, done.wait_for(|finished| *finished)).await; } - Ok(()) } -fn verify_adoption(target: &OwnedTarget, params: &AdoptTargetParams) -> anyhow::Result<()> { - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - || target.request_id != params.request_id - || target.template_id != "adopted" - || target.image_fingerprint != "adopted" - || target.adoption_source.as_deref() != Some(params.source_target.as_str()) - { - anyhow::bail!("existing target metadata conflicts with adoption request") +/// The background job kinds; at most one job per target is current. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WorkKind { + Create, + Power(PowerState), + Ingress { enabled: bool }, + Close, +} + +#[derive(Clone, Debug)] +enum SlotState { + Running, + Failed(String), +} + +/// The current job of one target: running, or failed with its error. A job +/// that succeeds leaves no slot, because Incus then holds the truth. +#[derive(Clone, Debug)] +struct Slot { + id: u64, + kind: WorkKind, + state: SlotState, + done: watch::Receiver, +} + +impl Slot { + fn running(&self) -> bool { + matches!(self.state, SlotState::Running) + } + + fn failure(&self) -> Option<&str> { + match &self.state { + SlotState::Failed(message) => Some(message), + SlotState::Running => None, + } } - Ok(()) } -async fn observe_daemon_readiness(config: &Config, target: &mut OwnedTarget) { - if target.status == ProviderTargetStatus::Ready && !relay::probe_guest(config, target).await { - target.status = ProviderTargetStatus::Starting; - } -} -fn summary(target: OwnedTarget) -> ProviderTargetSummary { - let mut metadata = BTreeMap::from([ - ("templateId".to_owned(), target.template_id), - ("imageFingerprint".to_owned(), target.image_fingerprint), - ]); - if let Some(location) = target.location { - metadata.insert("incusMember".to_owned(), location); - } - let mut capabilities = EnvironmentCapabilities::filesystem(true, true) - .with_filesystem_search() - .with_process() - .with_jobs(); - // Keep the controller-plane summary aligned with the envd handshake. The - // runtime consumes this summary before opening the data-plane connection. - capabilities.network = true; - capabilities.filesystem_glob = true; - capabilities.filesystem_ranged_read = true; - capabilities.process_output_notifications = false; - capabilities.process_pty = true; - ProviderTargetSummary { - target_id: target.target_id, - display_name: Some(target.environment_id), - status: target.status, - scope: EnvironmentScope::Default, - capabilities, - default_cwd: None, - power_states: crate::incus::SUPPORTED_POWER_STATES.to_vec(), - metadata, - } -} -fn decode(value: Value) -> anyhow::Result { - Ok(serde_json::from_value(value)?) -} -fn encode(value: T) -> anyhow::Result { - Ok(serde_json::to_value(value)?) -} -fn error(id: Value, code: EnvironmentProtocolErrorCode, message: String) -> Value { - json!({"jsonrpc":"2.0","id":id,"error":EnvironmentProtocolError::new(code,message)}) + +/// Background jobs by instance name. Jobs of one target run one at a time +/// under that target's lock; a newer request of another kind replaces the +/// current slot (a close supersedes a running create) and its job runs once +/// the earlier one finished. +#[derive(Default)] +pub(crate) struct Work { + next_id: AtomicU64, + inner: StdMutex, +} + +#[derive(Default)] +struct WorkInner { + slots: HashMap, + locks: HashMap>>, +} + +impl Work { + fn lock(&self) -> std::sync::MutexGuard<'_, WorkInner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// Run `job` as the target's current job unless a job of the same kind + /// is already running. + pub(crate) fn start( + self: &Arc, + name: &str, + kind: WorkKind, + job: BoxFuture<'static, anyhow::Result<()>>, + ) { + let mut inner = self.lock(); + if inner + .slots + .get(name) + .is_some_and(|slot| slot.kind == kind && slot.running()) + { + return; + } + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let (finished, done) = watch::channel(false); + let lock = inner.locks.entry(name.to_owned()).or_default().clone(); + inner.slots.insert( + name.to_owned(), + Slot { + id, + kind, + state: SlotState::Running, + done, + }, + ); + drop(inner); + let work = Arc::clone(self); + let name = name.to_owned(); + tokio::spawn(async move { + let outcome = { + let _guard = lock.lock().await; + AssertUnwindSafe(job) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(anyhow::anyhow!("the background job panicked"))) + }; + drop(lock); + work.finish(&name, id, outcome); + let _ = finished.send(true); + }); + } + + fn finish(&self, name: &str, id: u64, outcome: anyhow::Result<()>) { + let mut inner = self.lock(); + if inner.slots.get(name).is_some_and(|slot| slot.id == id) { + match outcome { + Ok(()) => { + inner.slots.remove(name); + } + Err(error) => { + if let Some(slot) = inner.slots.get_mut(name) { + slot.state = SlotState::Failed(format!("{error:#}")); + } + } + } + } + Self::release_lock(&mut inner, name); + } + + /// Drop the target's remembered failure once it holds nothing. + fn forget(&self, name: &str) { + let mut inner = self.lock(); + if inner.slots.get(name).is_some_and(|slot| !slot.running()) { + inner.slots.remove(name); + } + Self::release_lock(&mut inner, name); + } + + fn release_lock(inner: &mut WorkInner, name: &str) { + if !inner.slots.contains_key(name) + && inner + .locks + .get(name) + .is_some_and(|lock| Arc::strong_count(lock) == 1) + { + inner.locks.remove(name); + } + } + + fn view(&self, name: &str) -> Option { + self.lock().slots.get(name).cloned() + } + + /// True while a job of the target runs. + pub(crate) fn busy(&self, name: &str) -> bool { + self.lock().slots.get(name).is_some_and(Slot::running) + } + + fn done(&self, name: &str) -> Option> { + self.lock() + .slots + .get(name) + .filter(|slot| slot.running()) + .map(|slot| slot.done.clone()) + } } #[cfg(test)] -mod tests { +pub(crate) mod tests { + use std::sync::atomic::AtomicUsize; + + use async_trait::async_trait; + use environment_client::{ConnectOptions, DataClient, conformance::data_route::check_refusal}; + use environment_protocol::{ + error::{ErrorData, ProtocolErrorKind}, + idle::IdlePolicy, + jsonrpc::Id, + }; + use super::*; + use crate::{ + IncusTopology, + config::{ + ConfigInner, IncusConfig, IncusMode, NetworkPolicy, ProviderCredential, TemplatePolicy, + }, + incus::IncusMemberStatus, + policy::tests::owner, + }; - fn target() -> OwnedTarget { - OwnedTarget { - target_id: "target-a".into(), - name: "target-a".to_owned(), - universe_id: "u-a".to_owned(), - binding_id: "b-a".to_owned(), - environment_id: "e-a".to_owned(), - incarnation_id: "i-a".to_owned(), - request_id: "r-a".to_owned(), - template_id: "dev-small-v1".to_owned(), - image_fingerprint: "fingerprint-a".to_owned(), - adoption_source: None, - status: ProviderTargetStatus::Ready, - ipv4_address: Some("10.0.0.2".to_owned()), - ingress_hostname: None, - ingress_port: None, - location: Some("member-a".to_owned()), + pub(crate) const CREDENTIAL: &str = "0123456789abcdef0123456789abcdef"; + + /// An in-memory Incus: instances by name, every mutation instant. + #[derive(Clone, Default)] + pub(crate) struct FakeBackend { + pub(crate) instances: Arc>>, + pub(crate) creates: Arc, + pub(crate) deletes: Arc, + /// Every `set_power` target state, in order. + pub(crate) power_calls: Arc>>, + } + + impl FakeBackend { + pub(crate) fn insert( + &self, + owner: &TargetOwner, + template_id: &str, + status: InstanceStatus, + ) { + let name = policy::instance_name(owner); + self.instances.lock().unwrap().insert( + name.clone(), + OwnedTarget { + target_id: ProviderTargetId::new(name).unwrap(), + owner: owner.clone(), + template_id: template_id.to_owned(), + idle: None, + status, + ipv4_address: None, + ingress_hostname: None, + ingress_port: None, + location: None, + }, + ); + } + + pub(crate) fn update(&self, owner: &TargetOwner, change: impl FnOnce(&mut OwnedTarget)) { + let name = policy::instance_name(owner); + change(self.instances.lock().unwrap().get_mut(&name).unwrap()); + } + + pub(crate) fn get(&self, owner: &TargetOwner) -> Option { + let name = policy::instance_name(owner); + self.instances.lock().unwrap().get(&name).cloned() } } - #[test] - fn ownership_checks_fence_cross_binding_and_changed_image_adoption() { - let binding = ProviderBindingContext { - universe_id: "u-a".to_owned(), - binding_id: "b-a".to_owned(), - }; - assert!(verify_binding(&target(), &binding).is_ok()); - let other = ProviderBindingContext { - universe_id: "u-b".to_owned(), - ..binding.clone() - }; - assert!(verify_binding(&target(), &other).is_err()); - let params = CreateTargetParams { - request_id: "r-a".to_owned(), - environment_id: "e-a".to_owned(), - incarnation_id: "i-a".to_owned(), - binding: ProviderBindingContext { - universe_id: "u-a".to_owned(), - binding_id: "b-a".to_owned(), - }, - template_id: "dev-small-v1".to_owned(), - }; - let mut template = crate::config::TemplatePolicy { - template_id: "dev-small-v1".to_owned(), - display_name: "Small".to_owned(), + #[async_trait] + impl IncusBackend for FakeBackend { + async fn topology(&self) -> anyhow::Result { + Ok(IncusTopology { + mode: "single", + members: vec![IncusMemberStatus { + name: "standalone".to_owned(), + status: "Online".to_owned(), + schedulable: true, + }], + }) + } + + async fn reconcile_universe(&self, _universe_id: &UniverseId) -> anyhow::Result<()> { + Ok(()) + } + + async fn ensure_image(&self, _fingerprint: &str) -> anyhow::Result<()> { + Ok(()) + } + + async fn list_all_owned(&self) -> anyhow::Result> { + Ok(self.instances.lock().unwrap().values().cloned().collect()) + } + + async fn get_owned( + &self, + universe_id: &UniverseId, + name: &str, + ) -> anyhow::Result> { + Ok(self + .instances + .lock() + .unwrap() + .get(name) + .filter(|target| target.universe() == universe_id) + .cloned()) + } + + async fn create_vm( + &self, + owner: &TargetOwner, + template: &TemplatePolicy, + idle: Option<&IdlePolicy>, + ) -> anyhow::Result<()> { + self.creates.fetch_add(1, Ordering::SeqCst); + self.insert( + owner, + template.template_id.as_str(), + InstanceStatus::Running, + ); + self.update(owner, |target| target.idle = idle.cloned()); + Ok(()) + } + + async fn delete_vm(&self, target: &OwnedTarget) -> anyhow::Result<()> { + self.deletes.fetch_add(1, Ordering::SeqCst); + self.instances.lock().unwrap().remove(target.name()); + Ok(()) + } + + async fn set_ingress( + &self, + target: &OwnedTarget, + route: Option<(&str, u16)>, + ) -> anyhow::Result<()> { + if let Some(instance) = self.instances.lock().unwrap().get_mut(target.name()) { + instance.ingress_hostname = route.map(|(hostname, _)| hostname.to_owned()); + instance.ingress_port = route.map(|(_, port)| port); + } + Ok(()) + } + + async fn set_power(&self, target: &OwnedTarget, power: PowerState) -> anyhow::Result<()> { + self.power_calls.lock().unwrap().push(power); + if let Some(instance) = self.instances.lock().unwrap().get_mut(target.name()) { + instance.status = match power { + PowerState::Running => InstanceStatus::Running, + PowerState::Paused => InstanceStatus::Frozen, + _ => InstanceStatus::Stopped, + }; + } + Ok(()) + } + } + + pub(crate) fn template(id: &str, deprecated: bool) -> TemplatePolicy { + TemplatePolicy { + template_id: TemplateId::new(id).unwrap(), + display_name: environment_protocol::metadata::DisplayName::new("Small").unwrap(), description: None, - image_fingerprint: "fingerprint-a".to_owned(), + image_fingerprint: "fingerprint".to_owned(), cpu: 2, memory: "4GiB".to_owned(), disk: "40GiB".to_owned(), cluster_group: None, public_ingress: false, ingress_port: None, - deprecated: false, + deprecated, + idle: None, + resources: Default::default(), + } + } + + /// A provider config whose guests' envd listens on `envd_port`. + pub(crate) fn config_with(envd_port: u16, credential: Option<&str>) -> Config { + let mut idling = template("idling", false); + idling.idle = Some(IdlePolicy { + pause_after_ms: Some(600_000), + ..IdlePolicy::default() + }); + let templates = [template("small", false), template("retired", true), idling] + .into_iter() + .map(|template| (template.template_id.clone(), template)) + .collect(); + Config(Arc::new(ConfigInner { + controller_listen: "127.0.0.1:0".parse().unwrap(), + credential: credential.map(|credential| ProviderCredential::new(credential).unwrap()), + incus: IncusConfig { + mode: IncusMode::Single, + endpoints: vec!["https://incus.test".to_owned()], + client_certificate_pem: "cert".into(), + client_private_key_pem: "key".into(), + server_ca_pem: "ca".into(), + storage_pool: "default".to_owned(), + cluster_network_uplink: None, + image_server_url: None, + }, + templates, + network: NetworkPolicy::default(), + relay_idle_seconds: 60, + dial_timeout_seconds: 1, + idle_check_seconds: 60, + envd_port, + ingress: None, + })) + } + + fn config() -> Config { + config_with(19091, Some(CREDENTIAL)) + } + + fn app() -> (App, FakeBackend) { + let backend = FakeBackend::default(); + (App::new(config(), backend.clone()), backend) + } + + async fn call( + app: &App, + method: &str, + params: &P, + ) -> Result { + let request = Request::new(Id::Number(1), method, params).unwrap(); + let value = route(app, &request).await?; + Ok(serde_json::from_value::(value) + .unwrap() + .target) + } + + fn by_owner(owner: &TargetOwner) -> TargetRef { + TargetRef { + owner: owner.clone(), + target_id: None, + } + } + + fn create(owner: &TargetOwner, template: &str) -> CreateTargetParams { + CreateTargetParams { + owner: owner.clone(), + template_id: TemplateId::new(template).unwrap(), + display_name: None, + idle: None, + } + } + + fn reason(error: &ProtocolError) -> Option { + match error.data { + ErrorData::FailedPrecondition { reason } => Some(reason), + _ => None, + } + } + + #[test] + fn initialize_enforces_version_and_runs_once() { + let (app, _) = app(); + let mut initialized = false; + let params = |version: u32| serde_json::json!({"protocolVersion": version, "clientInfo": {"name": "test"}}); + let other = CONTROLLER_PROTOCOL_VERSION + 1; + let request = Request::new(Id::Number(1), "initialize", ¶ms(other)).unwrap(); + let mismatch = initialize(&app, &mut initialized, &request) + .result + .unwrap_err(); + assert_eq!( + mismatch.data, + ErrorData::VersionMismatch { + expected: CONTROLLER_PROTOCOL_VERSION, + received: other + } + ); + assert!(!initialized); + + let request = Request::new( + Id::Number(2), + "initialize", + ¶ms(CONTROLLER_PROTOCOL_VERSION), + ) + .unwrap(); + let response: ControllerInitializeResponse = initialize(&app, &mut initialized, &request) + .decode() + .unwrap(); + assert!(initialized); + assert_eq!(response.plane, Plane::Controller); + assert!(response.capabilities.power); + assert!(!response.capabilities.ingress); + + let repeat = initialize(&app, &mut initialized, &request) + .result + .unwrap_err(); + assert_eq!(repeat.kind(), ProtocolErrorKind::InvalidRequest); + } + + #[tokio::test(flavor = "current_thread")] + async fn templates_offer_power_states_and_their_idle_default() { + let (app, _) = app(); + let request = Request::new( + Id::Number(1), + TemplateList::NAME, + &ListTemplatesParams { + universe_id: UniverseId::new("u").unwrap(), + }, + ) + .unwrap(); + let listed: ListTemplatesResponse = + serde_json::from_value(route(&app, &request).await.unwrap()).unwrap(); + let idling = listed + .templates + .iter() + .find(|template| template.template_id.as_str() == "idling") + .unwrap(); + assert_eq!(idling.power_states, SUPPORTED_POWER_STATES); + assert_eq!( + idling.idle.as_ref().and_then(|idle| idle.pause_after_ms), + Some(600_000) + ); + let small = listed + .templates + .iter() + .find(|template| template.template_id.as_str() == "small") + .unwrap(); + assert_eq!(small.idle, None); + } + + #[tokio::test(flavor = "current_thread")] + async fn unknown_and_unadvertised_methods_are_typed_errors() { + let (app, _) = app(); + let unknown = call(&app, "controller/listTargets", &serde_json::json!({})) + .await + .unwrap_err(); + assert_eq!(unknown.kind(), ProtocolErrorKind::MethodNotFound); + let adopt = call(&app, "target/adopt", &serde_json::json!({})) + .await + .unwrap_err(); + assert_eq!(adopt.kind(), ProtocolErrorKind::MethodNotFound); + let ingress = call( + &app, + IngressEnsure::NAME, + &EnsureIngressParams { + target: by_owner(&owner("u", "e")), + }, + ) + .await + .unwrap_err(); + assert_eq!( + ingress.data, + ErrorData::CapabilityUnavailable { + capability: Capability::Ingress + } + ); + let undecodable = call(&app, TargetGet::NAME, &serde_json::json!({"target": 7})) + .await + .unwrap_err(); + assert_eq!(undecodable.kind(), ProtocolErrorKind::InvalidParams); + } + + #[tokio::test(flavor = "current_thread")] + async fn get_never_creates_and_answers_not_found() { + let (app, backend) = app(); + let owner = owner("u", "e"); + for _ in 0..2 { + let error = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap_err(); + assert_eq!(error.kind(), ProtocolErrorKind::NotFound); + } + let unknown = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: Some(ProviderTargetId::new("default/pod-abc").unwrap()), + }, + }, + ) + .await + .unwrap_err(); + assert_eq!(unknown.kind(), ProtocolErrorKind::NotFound); + assert_eq!(backend.creates.load(Ordering::SeqCst), 0); + assert!(backend.instances.lock().unwrap().is_empty()); + } + + #[tokio::test(flavor = "current_thread")] + async fn create_is_idempotent_per_owner_and_applies_policy_only_once() { + let (app, backend) = app(); + let owner = owner("u", "e"); + let params = create(&owner, "small"); + let first = call(&app, TargetCreate::NAME, ¶ms).await.unwrap(); + let second = call(&app, TargetCreate::NAME, ¶ms).await.unwrap(); + assert_eq!(first.owner, owner); + assert_eq!(first.target_id, second.target_id); + assert_eq!( + first.target_id.as_ref().map(ProviderTargetId::as_str), + Some(policy::instance_name(&owner).as_str()) + ); + assert_eq!(second.phase, TargetPhase::Active); + assert_eq!(second.power, Some(PowerState::Running)); + assert_eq!(backend.creates.load(Ordering::SeqCst), 1); + + // A target that already exists comes back even when its template + // has been deprecated since: observation applies no creation policy. + let existing = crate::policy::tests::owner("u", "old"); + backend.insert(&existing, "retired", InstanceStatus::Stopped); + let observed = call(&app, TargetCreate::NAME, &create(&existing, "retired")) + .await + .unwrap(); + assert_eq!(observed.power, Some(PowerState::Stopped)); + assert_eq!(backend.creates.load(Ordering::SeqCst), 1); + + let fresh = crate::policy::tests::owner("u", "new"); + let refused = call(&app, TargetCreate::NAME, &create(&fresh, "retired")) + .await + .unwrap_err(); + assert_eq!( + reason(&refused), + Some(PreconditionReason::TemplateDeprecated) + ); + let missing = call(&app, TargetCreate::NAME, &create(&fresh, "absent")) + .await + .unwrap_err(); + assert_eq!(missing.kind(), ProtocolErrorKind::NotFound); + } + + #[tokio::test(flavor = "current_thread")] + async fn create_records_the_override_or_the_template_idle_default() { + let (app, backend) = app(); + let defaulted = owner("u", "defaulted"); + call(&app, TargetCreate::NAME, &create(&defaulted, "idling")) + .await + .unwrap(); + assert_eq!( + backend.get(&defaulted).unwrap().idle, + app.config.templates[&TemplateId::new("idling").unwrap()] + .idle + .clone() + ); + + let overridden = owner("u", "overridden"); + let idle = IdlePolicy { + stop_after_ms: Some(1_800_000), + ..IdlePolicy::default() }; - assert!(verify_create(&target(), ¶ms, &template).is_ok()); - template.image_fingerprint = "fingerprint-b".to_owned(); - assert!(verify_create(&target(), ¶ms, &template).is_err()); + let mut params = create(&overridden, "idling"); + params.idle = Some(idle.clone()); + call(&app, TargetCreate::NAME, ¶ms).await.unwrap(); + assert_eq!(backend.get(&overridden).unwrap().idle, Some(idle)); + + let none = owner("u", "none"); + call(&app, TargetCreate::NAME, &create(&none, "small")) + .await + .unwrap(); + assert_eq!(backend.get(&none).unwrap().idle, None); + } + + #[tokio::test(flavor = "current_thread")] + async fn create_refuses_an_invalid_idle_override() { + let (app, backend) = app(); + let owner = owner("u", "e"); + let mut params = create(&owner, "small"); + params.idle = Some(IdlePolicy { + pause_after_ms: Some(600_000), + stop_after_ms: Some(60_000), + ..IdlePolicy::default() + }); + let refused = call(&app, TargetCreate::NAME, ¶ms).await.unwrap_err(); + assert_eq!(refused.kind(), ProtocolErrorKind::InvalidParams); + assert_eq!(backend.creates.load(Ordering::SeqCst), 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn create_refuses_an_idle_stage_incus_cannot_reach() { + let (app, backend) = app(); + let owner = owner("u", "e"); + let mut params = create(&owner, "small"); + params.idle = Some(IdlePolicy { + pause_after_ms: Some(60_000), + suspend_after_ms: Some(600_000), + ..IdlePolicy::default() + }); + let refused = call(&app, TargetCreate::NAME, ¶ms).await.unwrap_err(); + assert_eq!(refused.kind(), ProtocolErrorKind::InvalidParams); + assert!(matches!( + &refused.data, + ErrorData::InvalidParams { field: Some(field), .. } if field == "idle.suspendAfterMs" + )); + assert_eq!(backend.creates.load(Ordering::SeqCst), 0); + assert!(backend.get(&owner).is_none()); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_foreign_target_id_is_an_owner_mismatch() { + let (app, backend) = app(); + let holder = owner("u", "holder"); + backend.insert(&holder, "small", InstanceStatus::Running); + let error = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: TargetRef { + owner: owner("u", "other"), + target_id: Some(ProviderTargetId::new(policy::instance_name(&holder)).unwrap()), + }, + }, + ) + .await + .unwrap_err(); + assert_eq!(reason(&error), Some(PreconditionReason::OwnerMismatch)); + } + + #[tokio::test(flavor = "current_thread")] + async fn close_by_owner_destroys_and_is_idempotent() { + let (app, backend) = app(); + let owner = owner("u", "e"); + backend.insert(&owner, "small", InstanceStatus::Running); + let closed = call( + &app, + TargetClose::NAME, + &CloseTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap(); + assert_eq!(closed.phase, TargetPhase::Closed); + assert_eq!(closed.target_id, None); + assert_eq!(closed.owner, owner); + let again = call( + &app, + TargetClose::NAME, + &CloseTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap(); + assert_eq!(again.phase, TargetPhase::Closed); + assert_eq!(backend.deletes.load(Ordering::SeqCst), 1); + let gone = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap_err(); + assert_eq!(gone.kind(), ProtocolErrorKind::NotFound); + } + + #[tokio::test(flavor = "current_thread")] + async fn power_changes_need_an_active_target_and_a_supported_state() { + let (app, backend) = app(); + let owner = owner("u", "e"); + let set = |power| SetTargetPowerParams { + target: by_owner(&owner), + power, + }; + let missing = call(&app, TargetSetPower::NAME, &set(PowerState::Paused)) + .await + .unwrap_err(); + assert_eq!(missing.kind(), ProtocolErrorKind::NotFound); + + backend.insert(&owner, "small", InstanceStatus::Running); + let unsupported = call(&app, TargetSetPower::NAME, &set(PowerState::Suspended)) + .await + .unwrap_err(); + assert_eq!(unsupported.kind(), ProtocolErrorKind::InvalidParams); + let paused = call(&app, TargetSetPower::NAME, &set(PowerState::Paused)) + .await + .unwrap(); + assert_eq!(paused.phase, TargetPhase::Active); + assert_eq!(paused.power, Some(PowerState::Paused)); + + backend.insert(&owner, "small", InstanceStatus::Error); + let failed = call(&app, TargetSetPower::NAME, &set(PowerState::Running)) + .await + .unwrap_err(); + assert_eq!(reason(&failed), Some(PreconditionReason::TargetNotActive)); + } + + #[tokio::test(flavor = "current_thread")] + async fn observations_carry_power_only_while_active() { + let (app, backend) = app(); + let owner = owner("u", "e"); + let get = || GetTargetParams { + target: by_owner(&owner), + }; + for (status, power) in [ + (InstanceStatus::Running, Some(PowerState::Running)), + (InstanceStatus::Starting, Some(PowerState::Running)), + (InstanceStatus::Frozen, Some(PowerState::Paused)), + (InstanceStatus::Stopping, Some(PowerState::Stopped)), + (InstanceStatus::Stopped, Some(PowerState::Stopped)), + ] { + backend.insert(&owner, "small", status); + let observed = call(&app, TargetGet::NAME, &get()).await.unwrap(); + assert_eq!(observed.phase, TargetPhase::Active, "{status:?}"); + assert_eq!(observed.power, power, "{status:?}"); + } + backend.insert(&owner, "small", InstanceStatus::Error); + let failed = call(&app, TargetGet::NAME, &get()).await.unwrap(); + assert_eq!(failed.phase, TargetPhase::Failed); + assert_eq!(failed.power, None); + assert!(failed.message.is_some()); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_failed_create_is_provisioning_without_an_id_and_retried_by_create() { + let (app, backend) = app(); + let owner = owner("u", "e"); + let name = policy::instance_name(&owner); + app.work.start( + &name, + WorkKind::Create, + async { Err(anyhow::anyhow!("image server unreachable")) }.boxed(), + ); + wait_for_work(&app, &name).await; + let observed = call( + &app, + TargetGet::NAME, + &GetTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap(); + // No instance exists, so no id is handed out and the runtime keeps + // sending target/create. + assert_eq!(observed.phase, TargetPhase::Provisioning); + assert_eq!(observed.target_id, None); + assert!( + observed + .message + .as_ref() + .is_some_and(|message| message.as_str().contains("image server unreachable")) + ); + let created = call(&app, TargetCreate::NAME, &create(&owner, "small")) + .await + .unwrap(); + assert_eq!(created.phase, TargetPhase::Active); + assert_eq!( + created.target_id.as_ref().map(ProviderTargetId::as_str), + Some(name.as_str()) + ); + assert_eq!(backend.creates.load(Ordering::SeqCst), 1); + } + + #[tokio::test(flavor = "current_thread")] + async fn closing_an_owner_whose_create_failed_forgets_the_failure() { + let (app, _) = app(); + let owner = owner("u", "e"); + let name = policy::instance_name(&owner); + app.work.start( + &name, + WorkKind::Create, + async { Err(anyhow::anyhow!("image pull failed")) }.boxed(), + ); + wait_for_work(&app, &name).await; + // Closing an owner whose create failed before any instance existed + // holds nothing, so it closes at once and forgets the failure. + let closed = call( + &app, + TargetClose::NAME, + &CloseTargetParams { + target: by_owner(&owner), + }, + ) + .await + .unwrap(); + assert_eq!(closed.phase, TargetPhase::Closed); + assert!(app.work.view(&name).is_none()); + } + + #[test] + fn only_the_exact_bearer_credential_is_authorized() { + let config = config(); + let mut headers = HeaderMap::new(); + assert!(!authorized(&config, &headers)); + headers.insert(header::AUTHORIZATION, "Bearer wrong".parse().unwrap()); + assert!(!authorized(&config, &headers)); + headers.insert(header::AUTHORIZATION, CREDENTIAL.parse().unwrap()); + assert!(!authorized(&config, &headers)); + headers.insert( + header::AUTHORIZATION, + format!("Bearer {CREDENTIAL}").parse().unwrap(), + ); + assert!(authorized(&config, &headers)); } #[test] - fn summary_reports_member_without_changing_target_identity() { - let summary = summary(target()); - assert_eq!(summary.target_id.as_str(), "target-a"); + fn without_a_credential_every_request_is_authorized() { + let config = config_with(19091, None); + let mut headers = HeaderMap::new(); + assert!(authorized(&config, &headers)); + headers.insert(header::AUTHORIZATION, "Bearer anything".parse().unwrap()); + assert!(authorized(&config, &headers)); + } + + /// A guest envd stand-in on loopback that accepts WebSocket upgrades + /// and answers nothing. Returns its port. + pub(crate) async fn silent_envd() -> u16 { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(async move { + if let Ok(mut socket) = tokio_tungstenite::accept_async(stream).await { + while let Some(Ok(_)) = socket.next().await {} + } + }); + } + }); + port + } + + /// Serve `app`'s routes on loopback with a short data route hold. + /// Returns the data route URL of `owner`'s target. + async fn serve_routes(mut app: App, owner: &TargetOwner) -> String { + app.data_route_hold = Duration::from_millis(1_000); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, router(app)).await }); + format!( + "ws://{address}/routes/{}/{}/{}", + owner.universe_id, + owner.environment_id, + policy::instance_name(owner) + ) + } + + fn data_options() -> ConnectOptions { + ConnectOptions::new(ImplementationInfo::new("test").unwrap()).with_bearer_token(CREDENTIAL) + } + + /// Dial a data route that must refuse, and return its conforming refusal. + async fn refused(url: &str) -> DataRouteRefusal { + let error = match DataClient::connect_initialized(url, data_options()).await { + Ok(_) => panic!("the data route upgraded"), + Err(error) => error, + }; + check_refusal(&error) + .unwrap_or_else(|problem| panic!("{problem}")) + .clone() + } + + #[tokio::test(flavor = "current_thread")] + async fn a_data_route_without_a_target_is_gone() { + let (app, _) = app(); + let owner = owner("u", "e"); + let url = serve_routes(app, &owner).await; + assert_eq!(refused(&url).await, DataRouteRefusal::Gone {}); + let foreign = format!("{}x", url); + assert_eq!(refused(&foreign).await, DataRouteRefusal::Gone {}); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_broken_instance_refuses_as_failed() { + let (app, backend) = app(); + let owner = owner("u", "e"); + backend.insert(&owner, "small", InstanceStatus::Error); + let url = serve_routes(app, &owner).await; + assert!(matches!( + refused(&url).await, + DataRouteRefusal::Failed { .. } + )); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_running_target_whose_envd_does_not_answer_is_booting() { + let (app, backend) = app(); + let owner = owner("u", "e"); + // No guest address yet: envd cannot be reached. + backend.insert(&owner, "small", InstanceStatus::Running); + let url = serve_routes(app, &owner).await; + assert_eq!(refused(&url).await, DataRouteRefusal::Booting {}); + assert!(backend.power_calls.lock().unwrap().is_empty()); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_stopped_target_is_woken_and_waking_until_envd_answers() { + let (app, backend) = app(); + let owner = owner("u", "e"); + backend.insert(&owner, "small", InstanceStatus::Stopped); + let wakes = app.wakes.clone(); + let url = serve_routes(app, &owner).await; + assert_eq!(refused(&url).await, DataRouteRefusal::Waking {}); assert_eq!( - summary.metadata.get("incusMember").map(String::as_str), - Some("member-a") + backend.power_calls.lock().unwrap().as_slice(), + [PowerState::Running] ); - assert!(summary.capabilities.network); - assert!(summary.capabilities.filesystem_glob); - assert!(summary.capabilities.filesystem_ranged_read); - assert!(summary.capabilities.process_pty); - assert!(!summary.capabilities.process_output_notifications); + assert_eq!(backend.get(&owner).unwrap().status, InstanceStatus::Running); + assert!(wakes.within_grace(&policy::instance_name(&owner), Instant::now())); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_paused_target_wakes_on_dial_and_relays() { + let port = silent_envd().await; + let backend = FakeBackend::default(); + let app = App::new(config_with(port, Some(CREDENTIAL)), backend.clone()); + let relays = app.relays.clone(); + let owner = owner("u", "e"); + backend.insert(&owner, "small", InstanceStatus::Frozen); + backend.update(&owner, |target| { + target.ipv4_address = Some("127.0.0.1".to_owned()); + }); + let wakes = app.wakes.clone(); + let url = serve_routes(app, &owner).await; + let socket = dial_route(&url).await; + assert_eq!(backend.get(&owner).unwrap().status, InstanceStatus::Running); assert_eq!( - summary.power_states, - vec![PowerState::Running, PowerState::Paused, PowerState::Stopped] + backend.power_calls.lock().unwrap().as_slice(), + [PowerState::Running] + ); + // The relay holds the instance while it runs. + let name = policy::instance_name(&owner); + assert!(relays.active(&name)); + drop(socket); + tokio::time::timeout(Duration::from_secs(2), async { + while relays.active(&name) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the relay releases the instance when it ends"); + // The woken instance has its grace from the end of the connection. + assert!(wakes.within_grace(&name, Instant::now())); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_dial_during_a_power_down_waits_for_the_wake_behind_it() { + let port = silent_envd().await; + let backend = FakeBackend::default(); + let app = App::new(config_with(port, Some(CREDENTIAL)), backend.clone()); + let owner = owner("u", "e"); + let name = policy::instance_name(&owner); + backend.insert(&owner, "small", InstanceStatus::Running); + backend.update(&owner, |target| { + target.ipv4_address = Some("127.0.0.1".to_owned()); + }); + // A pause still in progress: the instance reads running until it + // freezes. + let pausing = backend.clone(); + let target = backend.get(&owner).unwrap(); + app.work.start( + &name, + WorkKind::Power(PowerState::Paused), + async move { + tokio::time::sleep(Duration::from_millis(300)).await; + pausing.set_power(&target, PowerState::Paused).await + } + .boxed(), + ); + let url = serve_routes(app, &owner).await; + let _socket = dial_route(&url).await; + // The dial upgraded only after the wake that followed the pause. + assert_eq!( + backend.power_calls.lock().unwrap().as_slice(), + [PowerState::Paused, PowerState::Running] + ); + assert_eq!(backend.get(&owner).unwrap().status, InstanceStatus::Running); + } + + /// Open the data route `url` with the configured credential and expect + /// it to upgrade. + async fn dial_route(url: &str) -> GuestSocket { + let mut request = + tokio_tungstenite::tungstenite::client::IntoClientRequest::into_client_request(url) + .unwrap(); + request.headers_mut().insert( + header::AUTHORIZATION, + format!("Bearer {CREDENTIAL}").parse().unwrap(), + ); + let (socket, response) = tokio_tungstenite::connect_async(request) + .await + .expect("the target relays"); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + socket + } + + #[tokio::test(flavor = "current_thread")] + async fn a_data_route_needs_the_configured_credential() { + let (app, backend) = app(); + let owner = owner("u", "e"); + backend.insert(&owner, "small", InstanceStatus::Running); + let url = serve_routes(app, &owner).await; + let options = ConnectOptions::new(ImplementationInfo::new("test").unwrap()); + match DataClient::connect_initialized(&url, options).await { + Err(environment_client::ClientError::UpgradeRejected { status: 401, .. }) => {} + Err(error) => panic!("expected HTTP 401, got {error}"), + Ok(_) => panic!("the data route upgraded without the credential"), + } + } + + /// The client's controller conformance suite against this server over + /// a real WebSocket, with the in-memory backend standing in for Incus. + #[tokio::test(flavor = "current_thread")] + async fn the_controller_passes_the_conformance_suite() { + use environment_client::conformance::controller::{ControllerConformance, run}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(serve_on(listener, config(), FakeBackend::default())); + let counter = Arc::new(AtomicUsize::new(0)); + let owner_factory: Arc TargetOwner + Send + Sync> = Arc::new(move || { + let environment = format!("env-{}", counter.fetch_add(1, Ordering::SeqCst)); + owner("conformance", &environment) + }); + let mut suite = ControllerConformance::new( + format!("ws://{address}/control"), + data_options(), + TemplateId::new("small").unwrap(), + owner_factory, + ); + suite.poll_interval = Duration::from_millis(10); + suite.settle_timeout = Duration::from_secs(10); + let report = run(&suite).await; + server.abort(); + report.into_result().expect("the controller conforms"); + } + + /// The same suite against a provider configured without a credential. + #[tokio::test(flavor = "current_thread")] + async fn the_controller_conforms_without_a_credential() { + use environment_client::conformance::controller::{ControllerConformance, run}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(serve_on( + listener, + config_with(19091, None), + FakeBackend::default(), + )); + let counter = Arc::new(AtomicUsize::new(0)); + let owner_factory: Arc TargetOwner + Send + Sync> = Arc::new(move || { + let environment = format!("env-{}", counter.fetch_add(1, Ordering::SeqCst)); + owner("conformance", &environment) + }); + let mut suite = ControllerConformance::new( + format!("ws://{address}/control"), + ConnectOptions::new(ImplementationInfo::new("conformance").unwrap()), + TemplateId::new("small").unwrap(), + owner_factory, ); + suite.poll_interval = Duration::from_millis(10); + suite.settle_timeout = Duration::from_secs(10); + let report = run(&suite).await; + server.abort(); + report.into_result().expect("the controller conforms"); } } diff --git a/crates/environment-provider-incus/tests/incus_live.rs b/crates/environment-provider-incus/tests/incus_live.rs index 72c46323d..a7593ba3d 100644 --- a/crates/environment-provider-incus/tests/incus_live.rs +++ b/crates/environment-provider-incus/tests/incus_live.rs @@ -1,19 +1,45 @@ -use std::path::PathBuf; +use std::{ + path::PathBuf, + sync::Arc, + time::{Duration, Instant}, +}; -use environment_provider_incus::{IncusBackend as _, IncusClient, ProviderArgs}; +use environment_client::{ + ConnectOptions, ControllerClient, DataClient, + conformance::controller::{ControllerConformance, run}, +}; +use environment_protocol::{ + control::target::{ + CloseTargetParams, CreateTargetParams, GetTargetParams, PowerState, SetTargetPowerParams, + TargetObservation, TargetOwner, TargetPhase, TargetRef, + }, + handshake::ImplementationInfo, + idle::IdlePolicy, + ids::{EnvironmentId, ProviderTargetId, TemplateId, UniverseId}, + paths::EnvironmentPath, + routes::{DataRouteBase, provider_data_route}, +}; +use environment_provider_incus::{ + Config, IncusBackend as _, IncusClient, ProviderArgs, policy, server, +}; +use tools::environment_protocol::conformance::{self as data_conformance, DataConformance}; -#[tokio::test(flavor = "current_thread")] -#[ignore = "requires a reachable standalone Incus server or Incus cluster"] -async fn configured_topology_and_owned_inventory_are_live() { +async fn live_config() -> Config { let config_path = std::env::var("LIGHTSPEED_INCUS_PROVIDER_CONFIG") .map(PathBuf::from) .expect("LIGHTSPEED_INCUS_PROVIDER_CONFIG must name a live provider config"); - let config = ProviderArgs { + ProviderArgs { config: config_path, } .load() .await - .expect("load live provider config"); + .expect("load live provider config") +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a reachable standalone Incus server or Incus cluster"] +async fn configured_topology_and_owned_inventory_are_live() { + let config = live_config().await; let backend = IncusClient::new(config.clone()) .await .expect("connect and validate live Incus topology"); @@ -29,5 +55,374 @@ async fn configured_topology_and_owned_inventory_are_live() { if topology.mode == "cluster" { assert!(target.location.is_some()); } + // Every owned instance carries its owner's deterministic name. + assert_eq!(target.name(), policy::instance_name(&target.owner)); } } + +/// One owner in the fixed conformance universe, so repeated runs reuse +/// one Incus project and network. +fn conformance_owner() -> TargetOwner { + TargetOwner { + universe_id: UniverseId::new("lightspeed-conformance").expect("universe id"), + environment_id: EnvironmentId::new(uuid::Uuid::new_v4().to_string()) + .expect("environment id"), + } +} + +fn first_template(config: &Config) -> TemplateId { + config + .templates + .values() + .find(|template| !template.deprecated) + .map(|template| template.template_id.clone()) + .expect("the live config offers a template") +} + +/// Connect options carrying the provider credential when one is +/// configured, as Lightspeed sends it. +fn connect_options(config: &Config) -> ConnectOptions { + let options = ConnectOptions::new( + ImplementationInfo::new("environment-provider-incus-live").expect("client info"), + ); + match &config.credential { + Some(credential) => options.with_bearer_token(credential.expose().to_owned()), + None => options, + } +} + +/// A provider on loopback against live Incus, with its controller +/// connected. Returns the controller, the data route base Lightspeed +/// derives from its URL, and the server task. +async fn start_provider( + config: &Config, +) -> ( + ControllerClient, + DataRouteBase, + tokio::task::JoinHandle>, +) { + let backend = IncusClient::new(config.clone()) + .await + .expect("connect and validate live Incus topology"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind the controller listener"); + let address = listener.local_addr().expect("listener address"); + let server = tokio::spawn(server::serve_on(listener, config.clone(), backend)); + let endpoint = format!("ws://{address}/control"); + let (controller, initialized) = + ControllerClient::connect_initialized(&endpoint, connect_options(config)) + .await + .expect("controller handshake"); + assert!(initialized.capabilities.power); + let base = DataRouteBase::derived_from_controller(&endpoint).expect("derived data route base"); + (controller, base, server) +} + +fn by_owner(owner: &TargetOwner, target_id: Option<&ProviderTargetId>) -> TargetRef { + TargetRef { + owner: owner.clone(), + target_id: target_id.cloned(), + } +} + +/// Create a target for `owner` the way the runtime does: `target/create` +/// until the observation carries a `targetId`, then `target/get` until it +/// is active. +async fn create_until_active( + controller: &ControllerClient, + owner: &TargetOwner, + template_id: &TemplateId, + idle: Option, + timeout: Duration, +) -> anyhow::Result { + let deadline = Instant::now() + timeout; + let params = CreateTargetParams { + owner: owner.clone(), + template_id: template_id.clone(), + display_name: None, + idle, + }; + let mut observation = controller.create_target(¶ms).await?.target; + loop { + if observation.phase == TargetPhase::Failed { + anyhow::bail!("the target failed: {:?}", observation.message); + } + if observation.phase == TargetPhase::Active + && let Some(target_id) = observation.target_id.clone() + { + // The provider names the instance after its owner. + assert_eq!(target_id.as_str(), policy::instance_name(owner)); + return Ok(target_id); + } + if Instant::now() >= deadline { + anyhow::bail!("the target did not become active within {timeout:?}: {observation:?}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + observation = match &observation.target_id { + Some(target_id) => { + controller + .get_target(&GetTargetParams { + target: by_owner(owner, Some(target_id)), + }) + .await? + .target + } + None => controller.create_target(¶ms).await?.target, + }; + } +} + +async fn get( + controller: &ControllerClient, + owner: &TargetOwner, + target_id: &ProviderTargetId, +) -> anyhow::Result { + Ok(controller + .get_target(&GetTargetParams { + target: by_owner(owner, Some(target_id)), + }) + .await? + .target) +} + +/// Dial the target's data route until envd answers, honoring the +/// provider's `waking` and `booting` refusals and their `Retry-After`, as +/// the runtime's open handshake does. +async fn dial_until_ready( + route: &str, + options: ConnectOptions, + timeout: Duration, +) -> anyhow::Result<(DataClient, Vec)> { + let deadline = Instant::now() + timeout; + let mut refusals = Vec::new(); + loop { + match DataClient::connect_initialized(route, options.clone()).await { + Ok((client, _)) => return Ok((client, refusals)), + Err(error) => { + let Some(refusal) = error.data_route_refusal() else { + anyhow::bail!("the data route failed without a typed refusal: {error}"); + }; + if !refusal.is_retryable() { + anyhow::bail!("the data route refused for good: {refusal}"); + } + refusals.push(refusal.state().to_owned()); + if Instant::now() >= deadline { + anyhow::bail!("envd did not answer within {timeout:?}: {refusal}"); + } + let wait = error.retry_after_ms().unwrap_or(2_000); + tokio::time::sleep(Duration::from_millis(wait)).await; + } + } + } +} + +/// Close by owner until the provider answers `closed`. +async fn close_until_closed( + controller: &ControllerClient, + owner: &TargetOwner, + timeout: Duration, +) -> anyhow::Result<()> { + let deadline = Instant::now() + timeout; + loop { + let observation = controller + .close_target(&CloseTargetParams { + target: by_owner(owner, None), + }) + .await? + .target; + if observation.phase == TargetPhase::Closed { + return Ok(()); + } + if Instant::now() >= deadline { + anyhow::bail!("the target did not close within {timeout:?}"); + } + tokio::time::sleep(Duration::from_secs(2)).await; + } +} + +/// The tools' data conformance suite against envd inside a live Incus VM, +/// reached through this provider's data route and relay, under the data +/// route base Lightspeed derives from the controller URL, exactly as the +/// runtime reaches it. Creates one VM and closes it. +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a reachable standalone Incus server or Incus cluster; creates and deletes one VM"] +async fn the_guest_envd_conforms_on_live_incus() { + let config = live_config().await; + let template_id = first_template(&config); + let options = connect_options(&config); + let (controller, base, server) = start_provider(&config).await; + let owner = conformance_owner(); + let outcome = async { + let target_id = create_until_active( + &controller, + &owner, + &template_id, + None, + Duration::from_secs(900), + ) + .await?; + let route = provider_data_route(&base, &owner, &target_id); + let (client, _) = + dial_until_ready(&route, options.clone(), Duration::from_secs(600)).await?; + let suite = DataConformance::new( + client.clone(), + EnvironmentPath::new("/workspace").expect("work directory"), + ); + let report = data_conformance::run(&suite).await; + let _ = client.close().await; + report + .into_result() + .map(|_| ()) + .map_err(anyhow::Error::from) + } + .await; + let closed = close_until_closed(&controller, &owner, Duration::from_secs(300)).await; + let _ = controller.close().await; + server.abort(); + outcome.expect("the guest envd conforms"); + closed.expect("the conformance target closes"); +} + +/// A paused target wakes when its data route is dialed: the dial holds or +/// refuses `waking`, and then relays to envd. Creates one VM and closes it. +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a reachable standalone Incus server or Incus cluster; creates and deletes one VM"] +async fn a_paused_target_wakes_on_dial_on_live_incus() { + let config = live_config().await; + let template_id = first_template(&config); + let options = connect_options(&config); + let (controller, base, server) = start_provider(&config).await; + let owner = conformance_owner(); + let outcome = async { + let target_id = create_until_active( + &controller, + &owner, + &template_id, + None, + Duration::from_secs(900), + ) + .await?; + let route = provider_data_route(&base, &owner, &target_id); + let (client, _) = + dial_until_ready(&route, options.clone(), Duration::from_secs(600)).await?; + let _ = client.close().await; + + let paused = controller + .set_target_power(&SetTargetPowerParams { + target: by_owner(&owner, Some(&target_id)), + power: PowerState::Paused, + }) + .await? + .target; + anyhow::ensure!( + paused.power == Some(PowerState::Paused), + "setPower did not pause the target: {paused:?}" + ); + + let (client, refusals) = + dial_until_ready(&route, options.clone(), Duration::from_secs(120)).await?; + let _ = client.close().await; + anyhow::ensure!( + refusals + .iter() + .all(|state| state == "waking" || state == "booting"), + "unexpected refusals while waking: {refusals:?}" + ); + let woken = get(&controller, &owner, &target_id).await?; + anyhow::ensure!( + woken.power == Some(PowerState::Running), + "the dial did not wake the target: {woken:?}" + ); + Ok(()) + } + .await; + let closed = close_until_closed(&controller, &owner, Duration::from_secs(300)).await; + let _ = controller.close().await; + server.abort(); + outcome.expect("the paused target wakes on dial"); + closed.expect("the target closes"); +} + +/// The idle loop pauses a target whose envd reports it idle past the +/// create-time `idle` override. Takes up to a few `idleCheckSeconds`. +/// Creates one VM and closes it. +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a reachable standalone Incus server or Incus cluster; creates and deletes one VM"] +async fn the_idle_loop_pauses_an_idle_target_on_live_incus() { + let config = live_config().await; + let template_id = first_template(&config); + let options = connect_options(&config); + let (controller, base, server) = start_provider(&config).await; + let owner = conformance_owner(); + let idle = IdlePolicy { + pause_after_ms: Some(1_000), + ..IdlePolicy::default() + }; + let wait = Duration::from_secs(config.idle_check_seconds * 3 + 60); + let outcome = async { + let target_id = create_until_active( + &controller, + &owner, + &template_id, + Some(idle), + Duration::from_secs(900), + ) + .await?; + // Wait until envd answers, so the idle loop can read it. + let route = provider_data_route(&base, &owner, &target_id); + let (client, _) = + dial_until_ready(&route, options.clone(), Duration::from_secs(600)).await?; + let _ = client.close().await; + let deadline = Instant::now() + wait; + loop { + let observed = get(&controller, &owner, &target_id).await?; + if observed.power == Some(PowerState::Paused) { + return Ok(()); + } + anyhow::ensure!( + Instant::now() < deadline, + "the idle loop did not pause the target within {wait:?}: {observed:?}" + ); + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + .await; + let closed = close_until_closed(&controller, &owner, Duration::from_secs(300)).await; + let _ = controller.close().await; + server.abort(); + outcome.expect("the idle loop pauses the target"); + closed.expect("the target closes"); +} + +/// The client's controller conformance suite against this provider on live +/// Incus. It creates one VM from the first offered template and closes it; +/// every owner lives in one fixed conformance universe, so repeated runs +/// reuse one Incus project and network. +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires a reachable standalone Incus server or Incus cluster; creates and deletes one VM"] +async fn the_controller_conforms_on_live_incus() { + let config = live_config().await; + let template_id = first_template(&config); + let backend = IncusClient::new(config.clone()) + .await + .expect("connect and validate live Incus topology"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind the controller listener"); + let address = listener.local_addr().expect("listener address"); + let options = connect_options(&config); + let server = tokio::spawn(server::serve_on(listener, config, backend)); + + let owner = Arc::new(conformance_owner); + let mut suite = ControllerConformance::new( + format!("ws://{address}/control"), + options, + template_id, + owner, + ); + suite.settle_timeout = Duration::from_secs(900); + let report = run(&suite).await; + server.abort(); + report.into_result().expect("the Incus controller conforms"); +} diff --git a/crates/environments/src/endpoint.rs b/crates/environments/src/endpoint.rs new file mode 100644 index 000000000..ff711fa1f --- /dev/null +++ b/crates/environments/src/endpoint.rs @@ -0,0 +1,270 @@ +//! Environment endpoints: where Lightspeed dials a provider controller or an +//! environment's data plane. +//! +//! An endpoint is one validated URL whose scheme selects the dialer from a +//! closed set; there is no separate transport enum. Every scheme carries +//! WebSocket text frames running the environment protocol's JSON-RPC +//! envelope: +//! +//! - `ws://host[:port][/path]` and `wss://host[:port][/path]`: TCP, plain or +//! TLS. +//! - `unix:///absolute/socket/path`: a local Unix socket (envd's Unix +//! listener). The dialer connects to the socket and performs the +//! WebSocket upgrade with request target `/` and host `localhost`. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use thiserror::Error; + +/// Longest endpoint URL in bytes. +pub const MAX_ENDPOINT_BYTES: usize = 2048; + +/// The dialer an endpoint selects. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum EndpointScheme { + Ws, + Wss, + Unix, +} + +impl EndpointScheme { + pub fn as_str(self) -> &'static str { + match self { + Self::Ws => "ws", + Self::Wss => "wss", + Self::Unix => "unix", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum EndpointError { + #[error("endpoint is empty")] + Empty, + + #[error("endpoint is longer than {MAX_ENDPOINT_BYTES} bytes")] + TooLong, + + #[error("endpoint must not contain whitespace or control characters: {0}")] + InvalidCharacter(String), + + #[error("endpoint scheme must be ws://, wss:// or unix://: {0}")] + Scheme(String), + + #[error("endpoint has no host: {0}")] + NoHost(String), + + #[error("endpoint must not carry user information: {0}")] + UserInfo(String), + + #[error("endpoint must not carry a fragment: {0}")] + Fragment(String), + + #[error("endpoint has an invalid port: {0}")] + Port(String), + + #[error("unix endpoint must name an absolute socket path without query: {0}")] + UnixPath(String), +} + +/// A validated endpoint URL. Serializes as the URL string. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EnvironmentEndpoint { + url: String, + scheme: EndpointScheme, +} + +impl EnvironmentEndpoint { + pub fn parse(value: &str) -> Result { + if value.is_empty() { + return Err(EndpointError::Empty); + } + if value.len() > MAX_ENDPOINT_BYTES { + return Err(EndpointError::TooLong); + } + if value + .chars() + .any(|character| character.is_whitespace() || character.is_control()) + { + return Err(EndpointError::InvalidCharacter(value.to_owned())); + } + let (scheme, rest) = value + .split_once("://") + .ok_or_else(|| EndpointError::Scheme(value.to_owned()))?; + let scheme = match scheme { + "ws" => EndpointScheme::Ws, + "wss" => EndpointScheme::Wss, + "unix" => EndpointScheme::Unix, + _ => return Err(EndpointError::Scheme(value.to_owned())), + }; + if rest.contains('#') { + return Err(EndpointError::Fragment(value.to_owned())); + } + match scheme { + EndpointScheme::Unix => { + if !rest.starts_with('/') || rest.len() == 1 || rest.contains('?') { + return Err(EndpointError::UnixPath(value.to_owned())); + } + } + EndpointScheme::Ws | EndpointScheme::Wss => { + let authority = rest.split(['/', '?']).next().unwrap_or_default(); + if authority.contains('@') { + return Err(EndpointError::UserInfo(value.to_owned())); + } + let host = + host_of(authority).ok_or_else(|| EndpointError::Port(value.to_owned()))?; + if host.is_empty() || host == "[]" { + return Err(EndpointError::NoHost(value.to_owned())); + } + } + } + Ok(Self { + url: value.to_owned(), + scheme, + }) + } + + pub fn as_str(&self) -> &str { + &self.url + } + + pub fn scheme(&self) -> EndpointScheme { + self.scheme + } + + /// The socket path of a `unix://` endpoint. + pub fn unix_socket_path(&self) -> Option<&str> { + match self.scheme { + EndpointScheme::Unix => self.url.strip_prefix("unix://"), + EndpointScheme::Ws | EndpointScheme::Wss => None, + } + } + + /// The host of a `ws://` or `wss://` endpoint, brackets kept for IPv6. + pub fn host(&self) -> Option<&str> { + let rest = match self.scheme { + EndpointScheme::Ws => self.url.strip_prefix("ws://")?, + EndpointScheme::Wss => self.url.strip_prefix("wss://")?, + EndpointScheme::Unix => return None, + }; + host_of(rest.split(['/', '?']).next().unwrap_or_default()) + } + + /// True for a Unix socket or a `localhost` or loopback-address host: + /// endpoints whose traffic never leaves the machine. + pub fn is_local(&self) -> bool { + match self.scheme { + EndpointScheme::Unix => true, + EndpointScheme::Ws | EndpointScheme::Wss => self.host().is_some_and(|host| { + let host = host.trim_start_matches('[').trim_end_matches(']'); + host.eq_ignore_ascii_case("localhost") + || host + .parse::() + .is_ok_and(|address| address.is_loopback()) + }), + } + } +} + +fn host_of(authority: &str) -> Option<&str> { + if let Some(rest) = authority.strip_prefix('[') { + let (address, after) = rest.split_once(']')?; + if !after.is_empty() { + after.strip_prefix(':')?.parse::().ok()?; + } + return Some(&authority[..address.len() + 2]); + } + match authority.rsplit_once(':') { + Some((host, port)) => { + port.parse::().ok()?; + Some(host) + } + None => Some(authority), + } +} + +impl fmt::Display for EnvironmentEndpoint { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.url.fmt(formatter) + } +} + +impl FromStr for EnvironmentEndpoint { + type Err = EndpointError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } +} + +impl Serialize for EnvironmentEndpoint { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.url) + } +} + +impl<'de> Deserialize<'de> for EnvironmentEndpoint { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + Self::parse(&value).map_err(de::Error::custom) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_scheme_selects_the_dialer() { + let tcp = EnvironmentEndpoint::parse("wss://incus.example.com:19090/control").unwrap(); + assert_eq!(tcp.scheme(), EndpointScheme::Wss); + assert_eq!(tcp.host(), Some("incus.example.com")); + assert!(!tcp.is_local()); + let local = + EnvironmentEndpoint::parse("ws://127.0.0.1:18080/environment-gateway/routes/u/e/i") + .unwrap(); + assert!(local.is_local()); + let unix = EnvironmentEndpoint::parse("unix:///run/lightspeed/envd.sock").unwrap(); + assert_eq!(unix.scheme(), EndpointScheme::Unix); + assert_eq!(unix.unix_socket_path(), Some("/run/lightspeed/envd.sock")); + assert!(unix.is_local()); + assert!( + EnvironmentEndpoint::parse("ws://[::1]:9/x") + .unwrap() + .is_local() + ); + } + + #[test] + fn other_schemes_and_malformed_urls_are_rejected() { + for bad in [ + "", + "http://host/control", + "ssh://host", + "stdio", + "ws://", + "ws://user@host", + "ws://host:port", + "ws://host/a b", + "ws://host/#frag", + "unix://relative.sock", + "unix:///", + ] { + assert!(EnvironmentEndpoint::parse(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn endpoints_serialize_as_strings_and_validate_on_decode() { + let endpoint: EnvironmentEndpoint = + serde_json::from_value(serde_json::json!("wss://gw.example.com/x")).unwrap(); + assert_eq!( + serde_json::to_value(&endpoint).unwrap(), + "wss://gw.example.com/x" + ); + assert!( + serde_json::from_value::(serde_json::json!("http://x")).is_err() + ); + } +} diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 85001130d..d4d7d2b4d 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -1,17 +1,35 @@ //! Environment-compute domain contracts. //! -//! Physical providers are deployment-scoped, provider bindings and -//! environments are universe-scoped, and provider target facts live on an -//! environment incarnation rather than on stable environment identity. +//! Physical providers are deployment-scoped; provider bindings, registration +//! keys and environments are universe-scoped. One `environments` row covers +//! three kinds: provisioned (a provider target), external (an `envd` +//! Lightspeed dials at a fixed endpoint) and registered (an `envd` that +//! dialed the gateway). Lightspeed owns a five-state lifecycle and the +//! decisions to create and to close; everything a provider does while a +//! target runs (power, idle, wake, ingress) is the provider's, and the row +//! only caches what the provider last reported, for display. +//! +//! Every write is a request type with a pure `apply` that the stores share: +//! a lifecycle write whose expected `from` state no longer holds is skipped +//! and leaves the record as it is, so a close accepted while a create was in +//! flight wins. Nothing that only reads a row writes to it; whether a row +//! may be used is [`EnvironmentRecord::use_gate`]. use std::{collections::BTreeMap, fmt, str::FromStr}; use async_trait::async_trait; use auth::{AuthGrantId, AuthProviderId, SecretId, SecretValue}; -pub use environment_protocol::control::targets::PowerState; +pub use environment_protocol::{ + control::target::{PowerState, TargetPhase}, + idle::IdlePolicy, +}; use environment_protocol::{ - control::targets::EnvironmentTemplate, - shared::{EnvironmentTransport, ProviderTargetId}, + control::template::EnvironmentTemplate, + handshake::ImplementationInfo, + ids::ProviderTargetId, + metadata::{DescriptiveText, validate_caller_metadata, validate_descriptive_text}, + registration::DaemonPublicKey, + routes::{DataRouteBase, DataRouteRefusal}, }; pub use harness::EnvironmentId; use harness::{StringIdError, validate_general_string_id}; @@ -94,30 +112,12 @@ macro_rules! registry_string_id { } registry_string_id!(EnvironmentProviderId); -registry_string_id!(EnvironmentProviderBindingId); -registry_string_id!(EnvironmentIncarnationId); registry_string_id!(EnvironmentProvisionRequestId); registry_string_id!(EnvironmentTemplateId); registry_string_id!(EnvironmentJobGroupId); registry_string_id!(EnvironmentRegistrationKeyId); registry_string_id!(EnvironmentDaemonId); -impl EnvironmentProvisionRequestId { - /// The deterministic request id of the one environment a daemon identity - /// may register, so a retried first registration converges on the same - /// environment through the `(universe, request_id)` unique key. - pub fn for_daemon(daemon_id: &EnvironmentDaemonId) -> Self { - Self::new(format!( - "{DAEMON_PROVISION_REQUEST_PREFIX}{}", - daemon_id.as_str() - )) - } -} - -/// Prefix of the request id a registered environment derives from its daemon -/// identity. -pub const DAEMON_PROVISION_REQUEST_PREFIX: &str = "daemon:"; - /// Prefix of every daemon id derived from an `envd` public key. pub const DAEMON_ID_PREFIX: &str = "daemon_"; @@ -151,21 +151,13 @@ pub enum EnvironmentRegistryError { #[error("environment registry {kind} not found: {id}")] NotFound { kind: &'static str, id: String }, - #[error( - "environment registry revision conflict for {kind} {id}: expected {expected:?}, actual {actual:?}" - )] - RevisionConflict { - kind: &'static str, - id: String, - expected: Option, - actual: Option, - }, - #[error("invalid environment registry request: {message}")] InvalidInput { message: String }, /// The registration key exists but no longer admits new daemon - /// identities: revoked or past its expiry. + /// identities (revoked or past its expiry), the secret an admission + /// presented was retired by a rotation (`rotated`), or a revoked key + /// was asked to rotate. #[error("environment registration key {registration_key_id} is {reason}")] RegistrationKeyUnavailable { registration_key_id: String, @@ -186,98 +178,184 @@ pub enum EnvironmentRegistryError { Store { message: String }, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentConnectionSpec { - pub endpoint: String, - pub transport: EnvironmentTransport, -} +// --------------------------------------------------------------------------- +// Timing constants +// --------------------------------------------------------------------------- -impl EnvironmentConnectionSpec { - pub fn new(endpoint: impl Into, transport: EnvironmentTransport) -> Self { - Self { - endpoint: endpoint.into(), - transport, - } - } +/// Default disconnect grace for ephemeral registered environments whose key +/// does not set one: long enough for a pod restart or a brief network +/// interruption, short enough that leaked benchmark sandboxes disappear. +pub const DEFAULT_EPHEMERAL_DISCONNECT_GRACE_MS: u64 = 5 * 60 * 1_000; - pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { - validate_endpoint("provider controller endpoint", &self.endpoint) - } -} +/// How long a caller waits for the disconnected daemon of a persistent +/// registered environment to reconnect before the environment counts as +/// away. Covers a rolling restart without hiding a dead pod. Ephemeral +/// daemons get no wait. +pub const REGISTERED_RECONNECT_WAIT_MS: i64 = 2 * 60 * 1_000; + +/// Default re-check interval of a [`EnvironmentUse::Wait`] when no +/// `Retry-After` says otherwise. +pub const WAIT_RETRY_AFTER_MS: u64 = 2_000; + +/// A connected registered environment's `last_seen_at_ms` is written at +/// most this often; connects and disconnects are always written. +pub const REGISTERED_LIVENESS_STAMP_MS: i64 = 5 * 60 * 1_000; + +/// The registered sweep marks a row disconnected when it reads connected but +/// its `last_seen_at_ms` is older than this: the gateway that held the +/// connection stopped without recording the disconnect. +pub const REGISTERED_STALE_AFTER_MS: i64 = 15 * 60 * 1_000; + +/// Longest stored `state_message`, in bytes. +pub const STATE_MESSAGE_MAX_BYTES: usize = 1024; + +/// Default page size of [`ListEnvironments`]. +pub const LIST_ENVIRONMENTS_DEFAULT_LIMIT: u32 = 100; + +/// Largest page size of [`ListEnvironments`]. +pub const LIST_ENVIRONMENTS_MAX_LIMIT: u32 = 500; + +// --------------------------------------------------------------------------- +// Providers and bindings +// --------------------------------------------------------------------------- #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct EnvironmentProviderRecord { pub provider_id: EnvironmentProviderId, pub display_name: Option, - pub controller_connection: EnvironmentConnectionSpec, - pub metadata: BTreeMap, + /// Where Lightspeed dials the provider's controller. + pub controller_endpoint: EnvironmentEndpoint, + /// Operator-configured base of the provider's data routes; `None` + /// derives it from the controller endpoint + /// ([`DataRouteBase::derived_from_controller`]). + pub data_route_base: Option, + /// A bearer credential is stored; without one no `Authorization` header + /// is sent on the controller or data routes. + pub has_credential: bool, pub created_at_ms: i64, pub updated_at_ms: i64, } impl EnvironmentProviderRecord { + /// The data route base the gateway and the reconciler dial: the + /// configured one, or the one derived from a `ws`/`wss` controller + /// endpoint. A `unix://` controller has no derivable base. + pub fn effective_data_route_base(&self) -> Result { + if let Some(base) = &self.data_route_base { + return Ok(base.clone()); + } + if self.controller_endpoint.scheme() == EndpointScheme::Unix { + return invalid( + "a provider with a unix:// controller endpoint needs a configured data route base", + ); + } + DataRouteBase::derived_from_controller(self.controller_endpoint.as_str()) + .map_err(|error| invalid_error(format!("data route base: {error}"))) + } + pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { validate_nonempty_optional("display_name", self.display_name.as_deref())?; - self.controller_connection.validate()?; - validate_metadata(&self.metadata)?; validate_timestamps(self.created_at_ms, self.updated_at_ms)?; + self.effective_data_route_base()?; + if let Some(base) = &self.data_route_base + && self.controller_endpoint.scheme() != EndpointScheme::Unix + { + base.check_for_controller(self.controller_endpoint.as_str()) + .map_err(|error| invalid_error(format!("data route base: {error}")))?; + } Ok(()) } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// What a provider put does with the stored bearer credential. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProviderCredentialChange { + /// Keep the stored credential, or none for a new provider. + Keep, + /// Store this credential, replacing any stored one. + Set(SecretValue), + /// Remove the stored credential. + Clear, +} + +/// Register or replace a provider. The provider credential is write-only: +/// the store keeps it sealed and hands it out only through +/// [`EnvironmentProviderStore::read_provider_credential`]. +#[derive(Clone, Debug, PartialEq, Eq)] pub struct PutEnvironmentProvider { pub provider_id: EnvironmentProviderId, pub display_name: Option, - pub controller_connection: EnvironmentConnectionSpec, - pub metadata: BTreeMap, + pub controller_endpoint: EnvironmentEndpoint, + pub data_route_base: Option, + pub credential: ProviderCredentialChange, pub updated_at_ms: i64, } impl PutEnvironmentProvider { - pub fn into_record(self) -> Result { + /// The record this put writes for a provider whose current record is + /// `existing`: the creation time is kept, and `has_credential` follows + /// the credential change. A new provider needs no credential. + pub fn into_record( + self, + existing: Option<&EnvironmentProviderRecord>, + ) -> Result { validate_nonnegative_i64(self.updated_at_ms, "updated_at_ms")?; + let has_credential = match &self.credential { + ProviderCredentialChange::Keep => existing.is_some_and(|record| record.has_credential), + ProviderCredentialChange::Set(credential) => { + validate_provider_credential(credential)?; + true + } + ProviderCredentialChange::Clear => false, + }; + let created_at_ms = existing.map_or(self.updated_at_ms, |record| record.created_at_ms); let record = EnvironmentProviderRecord { provider_id: self.provider_id, display_name: self.display_name, - controller_connection: self.controller_connection, - metadata: self.metadata, - created_at_ms: self.updated_at_ms, - updated_at_ms: self.updated_at_ms, + controller_endpoint: self.controller_endpoint, + data_route_base: self.data_route_base, + has_credential, + created_at_ms, + updated_at_ms: self.updated_at_ms.max(created_at_ms), }; record.validate()?; Ok(record) } } -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct ListEnvironmentProviders {} +/// Longest accepted provider credential, in bytes. +pub const PROVIDER_CREDENTIAL_MAX_BYTES: usize = 4096; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum EnvironmentProviderBindingStatus { - Enabled, - Disabled, +/// A provider credential travels as an HTTP bearer token: 1 to +/// [`PROVIDER_CREDENTIAL_MAX_BYTES`] bytes of visible ASCII. +pub fn validate_provider_credential( + credential: &SecretValue, +) -> Result<(), EnvironmentRegistryError> { + let value = credential.expose(); + if value.is_empty() + || value.len() > PROVIDER_CREDENTIAL_MAX_BYTES + || !value.bytes().all(|byte| byte.is_ascii_graphic()) + { + return invalid(format!( + "provider credential must be 1..={PROVIDER_CREDENTIAL_MAX_BYTES} bytes of visible ASCII" + )); + } + Ok(()) } +/// A universe's use of a provider. At most one per `(universe, provider)`; +/// `enabled` admits creates, template listing and data routes. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct EnvironmentProviderBindingRecord { pub universe_id: Uuid, - pub binding_id: EnvironmentProviderBindingId, pub provider_id: EnvironmentProviderId, - pub status: EnvironmentProviderBindingStatus, - pub revision: u64, - pub metadata: BTreeMap, + pub enabled: bool, pub created_at_ms: i64, pub updated_at_ms: i64, } impl EnvironmentProviderBindingRecord { pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { - if self.revision == 0 { - return invalid("provider binding revision must be positive"); - } - validate_metadata(&self.metadata)?; validate_timestamps(self.created_at_ms, self.updated_at_ms) } } @@ -285,24 +363,48 @@ impl EnvironmentProviderBindingRecord { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct PutEnvironmentProviderBinding { pub universe_id: Uuid, - pub binding_id: EnvironmentProviderBindingId, pub provider_id: EnvironmentProviderId, - pub status: EnvironmentProviderBindingStatus, - pub metadata: BTreeMap, - pub expected_revision: Option, + pub enabled: bool, pub updated_at_ms: i64, } +impl PutEnvironmentProviderBinding { + /// The record this put writes over `existing`, keeping its creation + /// time. + pub fn into_record( + self, + existing: Option<&EnvironmentProviderBindingRecord>, + ) -> Result { + validate_nonnegative_i64(self.updated_at_ms, "updated_at_ms")?; + let created_at_ms = existing.map_or(self.updated_at_ms, |record| record.created_at_ms); + let record = EnvironmentProviderBindingRecord { + universe_id: self.universe_id, + provider_id: self.provider_id, + enabled: self.enabled, + created_at_ms, + updated_at_ms: self.updated_at_ms.max(created_at_ms), + }; + record.validate()?; + Ok(record) + } +} + +// --------------------------------------------------------------------------- +// Registration keys +// --------------------------------------------------------------------------- + /// What Lightspeed does with a registered environment while its daemon is -/// disconnected. A property of the registration key, copied onto every -/// environment the key admits; `envd` neither knows nor chooses it. +/// disconnected. A property of the registration key, read from the key for +/// every environment it admitted; `envd` neither knows nor chooses it. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum RegisteredIdentityMode { - /// The environment stays `Offline` until it is explicitly closed. + /// The environment stays open while its daemon is away: callers wait up + /// to [`REGISTERED_RECONNECT_WAIT_MS`] for a reconnect, and the key's + /// `offline_close_after_ms` may close it eventually. Persistent, /// The environment is closed once its daemon has been away longer than - /// the key's disconnect grace. + /// the key's disconnect grace; callers never wait for a reconnect. Ephemeral, } @@ -323,395 +425,6 @@ impl RegisteredIdentityMode { } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum EnvironmentSource { - Provisioned { - provider_id: EnvironmentProviderId, - binding_id: EnvironmentProviderBindingId, - }, - External { - connection: EnvironmentConnectionSpec, - }, - /// An `envd` that dialed the gateway outbound and was admitted by a - /// registration key. The daemon public key is the identity: one key - /// maps to at most one environment in the deployment, ever, so a closed - /// environment's daemon can never register again without a new key. - Registered { - registration_key_id: EnvironmentRegistrationKeyId, - daemon_id: EnvironmentDaemonId, - /// Lowercase hex of the daemon's raw Ed25519 public key. - daemon_public_key: String, - identity_mode: RegisteredIdentityMode, - }, -} - -impl EnvironmentSource { - pub fn provider_id(&self) -> Option<&EnvironmentProviderId> { - match self { - Self::Provisioned { provider_id, .. } => Some(provider_id), - Self::External { .. } | Self::Registered { .. } => None, - } - } - - pub fn binding_id(&self) -> Option<&EnvironmentProviderBindingId> { - match self { - Self::Provisioned { binding_id, .. } => Some(binding_id), - Self::External { .. } | Self::Registered { .. } => None, - } - } - - pub fn registration_key_id(&self) -> Option<&EnvironmentRegistrationKeyId> { - match self { - Self::Registered { - registration_key_id, - .. - } => Some(registration_key_id), - Self::Provisioned { .. } | Self::External { .. } => None, - } - } - - pub fn identity_mode(&self) -> Option { - match self { - Self::Registered { identity_mode, .. } => Some(*identity_mode), - Self::Provisioned { .. } | Self::External { .. } => None, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum EnvironmentStatus { - Provisioning, - Booting, - Ready, - /// Execution frozen with RAM resident; wakes on use. - Paused, - /// Execution state saved to disk; wakes on use. - Suspended, - /// Powered off with disk retained; wakes on use for provisioned - /// environments whose provider supports power control. - Offline, - Closing, - Closed, - Failed, - Unknown, -} - -impl EnvironmentStatus { - /// The steady power state this observed status corresponds to, or `None` - /// while the environment is transitioning, failed, or gone. - pub fn power_state(self) -> Option { - match self { - Self::Ready => Some(PowerState::Running), - Self::Paused => Some(PowerState::Paused), - Self::Suspended => Some(PowerState::Suspended), - Self::Offline => Some(PowerState::Stopped), - Self::Provisioning - | Self::Booting - | Self::Closing - | Self::Closed - | Self::Failed - | Self::Unknown => None, - } - } - - /// True for the observed states a power change can start from or wake - /// out of. - pub fn is_powered_down(self) -> bool { - matches!(self, Self::Paused | Self::Suspended | Self::Offline) - } -} - -/// Staged idle policy: each threshold is measured against the daemon's -/// reported idle duration and must be non-decreasing in the order -/// pause → suspend → stop → close. Every stage is optional. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentIdlePolicy { - pub pause_after_ms: Option, - pub suspend_after_ms: Option, - pub stop_after_ms: Option, - pub close_after_ms: Option, -} - -/// One stage of an idle policy in escalation order. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum IdleAction { - Pause, - Suspend, - Stop, - Close, -} - -impl IdleAction { - /// The power state this stage requests; `None` for close. - pub fn power_state(self) -> Option { - match self { - Self::Pause => Some(PowerState::Paused), - Self::Suspend => Some(PowerState::Suspended), - Self::Stop => Some(PowerState::Stopped), - Self::Close => None, - } - } -} - -impl EnvironmentIdlePolicy { - pub fn is_empty(&self) -> bool { - self.pause_after_ms.is_none() - && self.suspend_after_ms.is_none() - && self.stop_after_ms.is_none() - && self.close_after_ms.is_none() - } - - /// Stages in escalation order with their thresholds. - pub fn stages(&self) -> Vec<(IdleAction, u64)> { - [ - (IdleAction::Pause, self.pause_after_ms), - (IdleAction::Suspend, self.suspend_after_ms), - (IdleAction::Stop, self.stop_after_ms), - (IdleAction::Close, self.close_after_ms), - ] - .into_iter() - .filter_map(|(action, threshold)| threshold.map(|threshold| (action, threshold))) - .collect() - } - - pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { - if self.is_empty() { - return invalid("idle policy must set at least one stage"); - } - let mut previous: Option<(IdleAction, u64)> = None; - for (action, threshold) in self.stages() { - if threshold == 0 { - return invalid(format!("idle policy {action:?} threshold must be positive")); - } - if let Some((earlier, earlier_threshold)) = previous - && threshold < earlier_threshold - { - return invalid(format!( - "idle policy {action:?} threshold must not be below {earlier:?}" - )); - } - previous = Some((action, threshold)); - } - Ok(()) - } - - /// The most escalated stage whose threshold `idle_for_ms` has crossed and - /// whose power state the provider supports (`close` is always - /// supported). Returns `None` while no stage applies. - pub fn due_action(&self, idle_for_ms: u64, supported: &[PowerState]) -> Option { - self.stages() - .into_iter() - .filter(|(_, threshold)| idle_for_ms >= *threshold) - .filter(|(action, _)| { - action - .power_state() - .is_none_or(|state| supported.contains(&state)) - }) - .map(|(action, _)| action) - .next_back() - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentIncarnationRecord { - pub incarnation_id: EnvironmentIncarnationId, - pub provision_request_id: Option, - pub provider_target_id: Option, - pub template_id: Option, - /// Provider-native reference used only while explicitly adopting an - /// existing target. The provider converts it into `provider_target_id`. - pub adoption_source_target: Option, - /// Power states the provider reported this target supports; observed - /// with the target, empty until first observation or when the provider - /// offers no power control. - pub power_states: Vec, - pub created_at_ms: i64, - pub updated_at_ms: i64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentRecord { - pub environment_id: EnvironmentId, - pub request_id: EnvironmentProvisionRequestId, - pub source: EnvironmentSource, - pub display_name: Option, - pub status: EnvironmentStatus, - /// Lightspeed-owned power intent. The lifecycle reconciler - /// converges the provider target toward it; observed state is `status`. - pub desired_power: PowerState, - /// Optional staged idle policy the power reaper applies from the - /// daemon's idle report. - pub idle_policy: Option, - pub incarnation: EnvironmentIncarnationRecord, - pub public_ingress_enabled: bool, - pub public_endpoint: Option, - pub metadata: BTreeMap, - /// Registered environments only: when the gateway last saw the daemon's - /// control connection (connect, heartbeat). A stale stamp under `Ready` - /// means the gateway stopped without recording the disconnect. - pub last_seen_at_ms: Option, - pub created_at_ms: i64, - pub updated_at_ms: i64, -} - -impl EnvironmentRecord { - pub fn registration_key_id(&self) -> Option<&EnvironmentRegistrationKeyId> { - self.source.registration_key_id() - } - - pub fn is_registered(&self) -> bool { - matches!(self.source, EnvironmentSource::Registered { .. }) - } - - /// True for a registered environment whose daemon is not currently - /// connected as far as durable state knows, including a `Ready` row - /// whose heartbeat stamp is older than `stale_after_ms`. - pub fn registered_daemon_absent(&self, now_ms: i64, stale_after_ms: i64) -> bool { - if !self.is_registered() { - return false; - } - match self.status { - EnvironmentStatus::Ready => self - .last_seen_at_ms - .is_none_or(|seen| now_ms.saturating_sub(seen) > stale_after_ms), - EnvironmentStatus::Offline => true, - _ => false, - } - } - /// True when the observed steady power state differs from the desired - /// one and the environment is in a state a power change can act on. - pub fn power_diverges(&self) -> bool { - matches!(self.source, EnvironmentSource::Provisioned { .. }) - && self - .status - .power_state() - .is_some_and(|observed| observed != self.desired_power) - } - - pub fn provider_id(&self) -> Option<&EnvironmentProviderId> { - self.source.provider_id() - } - - pub fn binding_id(&self) -> Option<&EnvironmentProviderBindingId> { - self.source.binding_id() - } - - pub fn observed_at_ms(&self) -> i64 { - self.updated_at_ms - } - - pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { - validate_nonempty_optional("display_name", self.display_name.as_deref())?; - validate_metadata(&self.metadata)?; - validate_timestamps(self.created_at_ms, self.updated_at_ms)?; - validate_timestamps( - self.incarnation.created_at_ms, - self.incarnation.updated_at_ms, - )?; - validate_nonempty_optional("public_endpoint", self.public_endpoint.as_deref())?; - if let Some(policy) = &self.idle_policy { - policy.validate()?; - } - if self.public_ingress_enabled != self.public_endpoint.is_some() { - return invalid("public ingress requires exactly one public endpoint when enabled"); - } - if self.public_ingress_enabled - && !matches!(self.source, EnvironmentSource::Provisioned { .. }) - { - return invalid("only provisioned environments can have provider-managed ingress"); - } - if self.last_seen_at_ms.is_some() && !self.is_registered() { - return invalid("only registered environments carry a last-seen stamp"); - } - if let Some(seen) = self.last_seen_at_ms { - validate_nonnegative_i64(seen, "last_seen_at_ms")?; - } - if let Some(target_id) = &self.incarnation.provider_target_id { - validate_provider_target_id(target_id)?; - } - match &self.source { - EnvironmentSource::Provisioned { .. } => { - if self.incarnation.provision_request_id.is_none() { - return invalid("provisioned incarnation requires a provision request id"); - } - if self.incarnation.template_id.is_some() - == self.incarnation.adoption_source_target.is_some() - { - return invalid( - "provisioned incarnation requires exactly one template or adoption source", - ); - } - if let Some(source) = self.incarnation.adoption_source_target.as_deref() - && (source.is_empty() - || source.len() > 255 - || source.chars().any(char::is_control)) - { - return invalid( - "adoption_source_target must be non-empty, at most 255 bytes, and contain no control characters", - ); - } - } - EnvironmentSource::External { connection } => { - connection.validate()?; - self.validate_no_provider_linkage("external")?; - } - EnvironmentSource::Registered { - daemon_id, - daemon_public_key, - .. - } => { - validate_daemon_public_key(daemon_public_key)?; - if daemon_id - != &EnvironmentDaemonId::from_public_key(&decode_hex(daemon_public_key)?) - { - return invalid("registered daemon id does not match its public key"); - } - if self.request_id != EnvironmentProvisionRequestId::for_daemon(daemon_id) { - return invalid( - "registered environment request id must derive from its daemon id", - ); - } - if matches!( - self.status, - EnvironmentStatus::Provisioning - | EnvironmentStatus::Booting - | EnvironmentStatus::Paused - | EnvironmentStatus::Suspended - ) { - return invalid( - "registered environments are ready, offline, closing, or closed", - ); - } - self.validate_no_provider_linkage("registered")?; - } - } - Ok(()) - } - - fn validate_no_provider_linkage(&self, kind: &str) -> Result<(), EnvironmentRegistryError> { - if self.incarnation.provision_request_id.is_some() - || self.incarnation.provider_target_id.is_some() - || self.incarnation.template_id.is_some() - || self.incarnation.adoption_source_target.is_some() - || !self.incarnation.power_states.is_empty() - { - return invalid(format!("{kind} incarnation must not have provider linkage")); - } - if self.desired_power != PowerState::Running || self.idle_policy.is_some() { - return invalid(format!("{kind} environments have no power control")); - } - Ok(()) - } -} - -/// Default disconnect grace for ephemeral registered environments whose key -/// does not set one: long enough for a pod restart or a brief network -/// interruption, short enough that leaked benchmark sandboxes disappear. -pub const DEFAULT_EPHEMERAL_DISCONNECT_GRACE_MS: u64 = 5 * 60 * 1_000; - /// Prefix of every environment registration key secret. pub const REGISTRATION_KEY_SECRET_PREFIX: &str = "lsrk_"; @@ -732,22 +445,30 @@ pub enum EnvironmentRegistrationKeyStatus { /// One reusable, universe-scoped admission policy for outbound `envd` /// registration, and the group of the environments it admitted. The row /// stores no counters: registration and active counts derive from -/// environment rows carrying the key id. +/// environment rows carrying the key id. Its secrets live beside it, so a +/// rotation keeps the group, the allowlists naming the key and its identity +/// mode. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct EnvironmentRegistrationKeyRecord { pub registration_key_id: EnvironmentRegistrationKeyId, /// Required: the group name shown wherever registered environments are /// listed. pub display_name: String, + /// Display prefix of the key's live secret. pub key_prefix: String, + /// Immutable. pub identity_mode: RegisteredIdentityMode, /// Non-closed environments this key may have at once; absent means /// unlimited. pub max_active_environments: Option, /// Absent means [`DEFAULT_EPHEMERAL_DISCONNECT_GRACE_MS`]. pub ephemeral_disconnect_grace_ms: Option, + /// Persistent keys only: close an environment whose daemon has been + /// away longer than this. + pub offline_close_after_ms: Option, pub expires_at_ms: Option, pub created_at_ms: i64, + pub updated_at_ms: i64, pub revoked_at_ms: Option, } @@ -766,18 +487,15 @@ impl EnvironmentRegistrationKeyRecord { pub fn check_admits(&self, now_ms: i64) -> Result<(), EnvironmentRegistryError> { match self.status(now_ms) { EnvironmentRegistrationKeyStatus::Active => Ok(()), - EnvironmentRegistrationKeyStatus::Revoked => { - Err(EnvironmentRegistryError::RegistrationKeyUnavailable { - registration_key_id: self.registration_key_id.to_string(), - reason: "revoked", - }) - } - EnvironmentRegistrationKeyStatus::Expired => { - Err(EnvironmentRegistryError::RegistrationKeyUnavailable { - registration_key_id: self.registration_key_id.to_string(), - reason: "expired", - }) - } + EnvironmentRegistrationKeyStatus::Revoked => Err(self.unavailable("revoked")), + EnvironmentRegistrationKeyStatus::Expired => Err(self.unavailable("expired")), + } + } + + fn unavailable(&self, reason: &'static str) -> EnvironmentRegistryError { + EnvironmentRegistryError::RegistrationKeyUnavailable { + registration_key_id: self.registration_key_id.to_string(), + reason, } } @@ -795,15 +513,22 @@ impl EnvironmentRegistrationKeyRecord { if self.ephemeral_disconnect_grace_ms == Some(0) { return invalid("ephemeral_disconnect_grace_ms must be positive when set"); } - validate_nonnegative_i64(self.created_at_ms, "created_at_ms")?; + if self.offline_close_after_ms == Some(0) { + return invalid("offline_close_after_ms must be positive when set"); + } + if self.offline_close_after_ms.is_some() + && self.identity_mode != RegisteredIdentityMode::Persistent + { + return invalid("offline_close_after_ms applies only to persistent keys"); + } + validate_timestamps(self.created_at_ms, self.updated_at_ms)?; if let Some(expires) = self.expires_at_ms { validate_nonnegative_i64(expires, "expires_at_ms")?; } - if let Some(revoked) = self.revoked_at_ms { - validate_nonnegative_i64(revoked, "revoked_at_ms")?; - if revoked < self.created_at_ms { - return invalid("revoked_at_ms must be >= created_at_ms"); - } + if let Some(revoked) = self.revoked_at_ms + && revoked < self.created_at_ms + { + return invalid("revoked_at_ms must be >= created_at_ms"); } Ok(()) } @@ -816,43 +541,72 @@ pub struct RegistrationKeyPolicy { pub identity_mode: RegisteredIdentityMode, pub max_active_environments: Option, pub ephemeral_disconnect_grace_ms: Option, + pub offline_close_after_ms: Option, + pub expires_at_ms: Option, +} + +/// The editable part of a key's policy; a full replace (absent means +/// unset). The identity mode is not part of it: it never changes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RegistrationKeySettings { + pub display_name: String, + pub max_active_environments: Option, + pub ephemeral_disconnect_grace_ms: Option, + pub offline_close_after_ms: Option, pub expires_at_ms: Option, } -/// A freshly minted registration key: the one-time plaintext secret plus the -/// record and hash to persist. The secret is structurally not persistable. +/// A freshly minted registration secret: the one-time plaintext plus the +/// hash and display prefix to persist. The secret is structurally not +/// persistable. #[derive(Clone, Debug)] -pub struct MintedRegistrationKey { +pub struct MintedRegistrationSecret { pub secret: SecretValue, pub secret_hash: String, - pub record: EnvironmentRegistrationKeyRecord, + pub key_prefix: String, } -/// Mint a registration key. The secret is returned exactly once; only its -/// SHA-256 hash and display prefix are persisted. -pub fn mint_registration_key( - registration_key_id: EnvironmentRegistrationKeyId, - policy: RegistrationKeyPolicy, - created_at_ms: i64, -) -> Result { +/// Mint a registration secret. Only its SHA-256 hash and display prefix are +/// ever persisted. +pub fn mint_registration_secret() -> MintedRegistrationSecret { let secret = auth::generate_prefixed_secret(REGISTRATION_KEY_SECRET_PREFIX); - let record = EnvironmentRegistrationKeyRecord { - registration_key_id, - display_name: policy.display_name, + MintedRegistrationSecret { + secret_hash: auth::secret_sha256_hex(&secret), key_prefix: auth::secret_display_prefix(&secret, REGISTRATION_KEY_DISPLAY_PREFIX_LEN), + secret: SecretValue::new(secret), + } +} + +/// A freshly minted registration key: its first secret plus the record to +/// persist. +#[derive(Clone, Debug)] +pub struct MintedRegistrationKey { + pub secret: MintedRegistrationSecret, + pub record: EnvironmentRegistrationKeyRecord, +} + +/// Mint a registration key. The secret is returned exactly once. +pub fn mint_registration_key( + registration_key_id: EnvironmentRegistrationKeyId, + policy: RegistrationKeyPolicy, + created_at_ms: i64, +) -> Result { + let secret = mint_registration_secret(); + let record = EnvironmentRegistrationKeyRecord { + registration_key_id, + display_name: policy.display_name, + key_prefix: secret.key_prefix.clone(), identity_mode: policy.identity_mode, max_active_environments: policy.max_active_environments, ephemeral_disconnect_grace_ms: policy.ephemeral_disconnect_grace_ms, + offline_close_after_ms: policy.offline_close_after_ms, expires_at_ms: policy.expires_at_ms, created_at_ms, + updated_at_ms: created_at_ms, revoked_at_ms: None, }; record.validate()?; - Ok(MintedRegistrationKey { - secret_hash: auth::secret_sha256_hex(&secret), - secret: SecretValue::new(secret), - record, - }) + Ok(MintedRegistrationKey { secret, record }) } /// Lowercase hex SHA-256 of a presented registration secret: the lookup key. @@ -860,12 +614,77 @@ pub fn registration_key_hash(secret: &str) -> String { auth::secret_sha256_hex(secret) } +/// Create a key with its first live secret. #[derive(Clone, Debug)] pub struct CreateEnvironmentRegistrationKey { pub secret_hash: String, pub record: EnvironmentRegistrationKeyRecord, } +/// Replace the editable policy of a key. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UpdateEnvironmentRegistrationKey { + pub registration_key_id: EnvironmentRegistrationKeyId, + pub settings: RegistrationKeySettings, + pub updated_at_ms: i64, +} + +impl UpdateEnvironmentRegistrationKey { + /// Apply the settings to `record`; the result is validated, so an + /// `offline_close_after_ms` on an ephemeral key is refused. + pub fn apply( + &self, + record: &mut EnvironmentRegistrationKeyRecord, + ) -> Result<(), EnvironmentRegistryError> { + validate_nonnegative_i64(self.updated_at_ms, "updated_at_ms")?; + let settings = self.settings.clone(); + let mut candidate = EnvironmentRegistrationKeyRecord { + display_name: settings.display_name, + max_active_environments: settings.max_active_environments, + ephemeral_disconnect_grace_ms: settings.ephemeral_disconnect_grace_ms, + offline_close_after_ms: settings.offline_close_after_ms, + expires_at_ms: settings.expires_at_ms, + ..record.clone() + }; + candidate.updated_at_ms = candidate.updated_at_ms.max(self.updated_at_ms); + candidate.validate()?; + *record = candidate; + Ok(()) + } +} + +/// Retire every live secret of the key and make this one its only live +/// secret. Daemons the key already admitted are unaffected: they never +/// present the key again. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RotateEnvironmentRegistrationKey { + pub registration_key_id: EnvironmentRegistrationKeyId, + pub secret_hash: String, + pub key_prefix: String, + pub rotated_at_ms: i64, +} + +impl RotateEnvironmentRegistrationKey { + /// Apply the rotation to the key record: the new display prefix and + /// update time. A revoked key refuses with `RegistrationKeyUnavailable`. + pub fn apply( + &self, + record: &mut EnvironmentRegistrationKeyRecord, + ) -> Result<(), EnvironmentRegistryError> { + validate_nonnegative_i64(self.rotated_at_ms, "rotated_at_ms")?; + validate_secret_hash(&self.secret_hash)?; + if record.revoked_at_ms.is_some() { + return Err(record.unavailable("revoked")); + } + let mut candidate = record.clone(); + candidate.key_prefix = self.key_prefix.clone(); + candidate.updated_at_ms = candidate.updated_at_ms.max(self.rotated_at_ms); + candidate.validate()?; + *record = candidate; + Ok(()) + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct RevokeEnvironmentRegistrationKey { pub registration_key_id: EnvironmentRegistrationKeyId, @@ -895,13 +714,24 @@ pub trait EnvironmentRegistrationKeyStore: Send + Sync { async fn list_registration_keys( &self, ) -> Result, EnvironmentRegistryError>; + async fn update_registration_key( + &self, + request: UpdateEnvironmentRegistrationKey, + ) -> Result; + /// A revoked key refuses with `RegistrationKeyUnavailable { reason: + /// "revoked" }`. + async fn rotate_registration_key( + &self, + request: RotateEnvironmentRegistrationKey, + ) -> Result; /// Idempotent: revoking a revoked key keeps the original revocation time. async fn revoke_registration_key( &self, request: RevokeEnvironmentRegistrationKey, ) -> Result; - /// Resolve a presented secret by hash regardless of status; the caller - /// turns revoked/expired into typed refusals. + /// Resolve a presented secret by hash among live (not retired) secrets, + /// regardless of the key's status; the caller turns revoked or expired + /// into typed refusals. async fn resolve_registration_key( &self, secret_hash: &str, @@ -912,207 +742,1412 @@ pub trait EnvironmentRegistrationKeyStore: Send + Sync { ) -> Result; } -/// Admission of a first-seen daemon identity. The store locks the key row, -/// re-checks that the key still admits, counts the key's non-closed -/// environments against its limit, and inserts the environment `Ready`, all -/// in one transaction. A retry with the same daemon converges through the -/// derived request id. +// --------------------------------------------------------------------------- +// Environment records +// --------------------------------------------------------------------------- + +/// The lifecycle Lightspeed owns. External and registered environments are +/// only ever `open` or `closed`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentState { + /// A provisioned environment whose target is being created and has not + /// passed its open handshake yet. + Provisioning, + Open, + /// A provisioned environment whose target is being closed. + Closing, + Closed, + /// A provisioned environment broken beyond recovery; see + /// `state_message`. + Failed, +} + +impl EnvironmentState { + pub fn as_str(self) -> &'static str { + match self { + Self::Provisioning => "provisioning", + Self::Open => "open", + Self::Closing => "closing", + Self::Closed => "closed", + Self::Failed => "failed", + } + } + + pub fn parse(value: &str) -> Option { + match value { + "provisioning" => Some(Self::Provisioning), + "open" => Some(Self::Open), + "closing" => Some(Self::Closing), + "closed" => Some(Self::Closed), + "failed" => Some(Self::Failed), + _ => None, + } + } +} + +impl fmt::Display for EnvironmentState { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Why an environment is closing or closed. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentCloseReason { + /// A caller closed it. + Requested, + /// Its registration key was revoked with its environments. + KeyRevoked, + /// Its ephemeral daemon stayed away past the key's disconnect grace. + EphemeralGrace, + /// Its ephemeral daemon said goodbye. + Departed, + /// Its persistent daemon stayed away past the key's + /// `offline_close_after_ms`. + OfflineTimeout, + /// The provider reported its target closed or gone. + ProviderEnded, +} + +impl EnvironmentCloseReason { + pub fn as_str(self) -> &'static str { + match self { + Self::Requested => "requested", + Self::KeyRevoked => "key_revoked", + Self::EphemeralGrace => "ephemeral_grace", + Self::Departed => "departed", + Self::OfflineTimeout => "offline_timeout", + Self::ProviderEnded => "provider_ended", + } + } + + pub fn parse(value: &str) -> Option { + match value { + "requested" => Some(Self::Requested), + "key_revoked" => Some(Self::KeyRevoked), + "ephemeral_grace" => Some(Self::EphemeralGrace), + "departed" => Some(Self::Departed), + "offline_timeout" => Some(Self::OfflineTimeout), + "provider_ended" => Some(Self::ProviderEnded), + _ => None, + } + } +} + +impl fmt::Display for EnvironmentCloseReason { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Display status derived from the state and the per-kind observations; +/// never stored. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentDisplayStatus { + Provisioning, + Ready, + Paused, + Suspended, + Stopped, + /// The provider's last report gives no usable phase. + Unknown, + /// A registered daemon is disconnected. + Away, + Closing, + Closed, + Failed, +} + +impl EnvironmentDisplayStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Provisioning => "provisioning", + Self::Ready => "ready", + Self::Paused => "paused", + Self::Suspended => "suspended", + Self::Stopped => "stopped", + Self::Unknown => "unknown", + Self::Away => "away", + Self::Closing => "closing", + Self::Closed => "closed", + Self::Failed => "failed", + } + } +} + +impl fmt::Display for EnvironmentDisplayStatus { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// What Lightspeed learned about the daemon serving an environment: from +/// `registration/register` for a registered daemon, from the data-plane +/// `initialize` of the open handshake for a provisioned one. Recorded so an +/// operator sees which build serves it, and which protocol version a +/// mismatched daemon spoke. Never consulted for admission: the protocol +/// version alone decides that. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DaemonBuild { + pub version: Option, + pub git_sha: Option, + pub protocol_version: u32, +} + +impl DaemonBuild { + pub fn from_implementation(implementation: &ImplementationInfo, protocol_version: u32) -> Self { + Self { + version: implementation.version().map(str::to_owned), + git_sha: implementation.git_sha().map(str::to_owned), + protocol_version, + } + } + + fn validate(&self) -> Result<(), EnvironmentRegistryError> { + validate_nonempty_optional("daemon version", self.version.as_deref())?; + validate_nonempty_optional("daemon git sha", self.git_sha.as_deref()) + } +} + +/// What the provider last reported about a provisioned environment's +/// target. Display only: no loop selects on it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ObservedTarget { + pub phase: TargetPhase, + /// Present only when `phase` is `active`. + pub power: Option, + pub message: Option, + pub observed_at_ms: i64, +} + +impl ObservedTarget { + fn validate(&self) -> Result<(), EnvironmentRegistryError> { + validate_nonnegative_i64(self.observed_at_ms, "observed_at_ms")?; + if self.power.is_some() && self.phase != TargetPhase::Active { + return invalid("an observed power state requires the active phase"); + } + if let Some(message) = &self.message { + validate_descriptive_text(message) + .map_err(|error| invalid_error(format!("observed message: {error}")))?; + } + Ok(()) + } +} + +/// The per-kind facts of a provisioned environment. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProvisionedSource { + pub provider_id: EnvironmentProviderId, + pub template_id: EnvironmentTemplateId, + /// Create-time idle override, sent with every `target/create` retry. + /// Never interpreted by Lightspeed. + pub idle: Option, + /// The provider's handle, once `target/create` answered with one. + pub provider_target_id: Option, + /// Display only; no loop selects on it. + pub observed: Option, + /// The public endpoint the provider last reported. + pub public_endpoint: Option, + /// Work schedule; `Some` exactly while `provisioning` or `closing`. + pub next_attempt_at_ms: Option, + /// Consecutive failed attempts of the current work; drives backoff. + pub attempts: u32, +} + +/// The per-kind facts of a registered environment. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RegisteredSource { + pub registration_key_id: EnvironmentRegistrationKeyId, + /// Lowercase hex of the daemon's raw Ed25519 public key: the identity. + /// One key maps to at most one environment in the deployment, ever, so + /// a closed environment's daemon can never register again. + pub daemon_public_key: String, + /// Read from the key; not stored on the row. + pub identity_mode: RegisteredIdentityMode, + /// The gateway holds a control connection, as far as durable state + /// knows. + pub connected: bool, + /// When `connected` last changed (or the gateway restarted). + pub connection_changed_at_ms: i64, + /// Coarse liveness: written at connect and disconnect, and at most every + /// [`REGISTERED_LIVENESS_STAMP_MS`] while connected. + pub last_seen_at_ms: i64, +} + +impl RegisteredSource { + /// The daemon id derived from the public key. + pub fn daemon_id(&self) -> EnvironmentDaemonId { + match DaemonPublicKey::parse(&self.daemon_public_key) { + Ok(key) => EnvironmentDaemonId::from_public_key(key.as_bytes()), + // A record that passed validation always parses; an unvalidated + // one still gets a stable id. + Err(_) => EnvironmentDaemonId::from_public_key(self.daemon_public_key.as_bytes()), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EnvironmentSource { + /// A provider target Lightspeed asked a provider to create. + Provisioned(ProvisionedSource), + /// An `envd` Lightspeed dials directly at a fixed endpoint. + External { endpoint: EnvironmentEndpoint }, + /// An `envd` that dialed the gateway outbound and was admitted by a + /// registration key. + Registered(RegisteredSource), +} + +impl EnvironmentSource { + pub fn provider_id(&self) -> Option<&EnvironmentProviderId> { + match self { + Self::Provisioned(source) => Some(&source.provider_id), + Self::External { .. } | Self::Registered(_) => None, + } + } + + pub fn registration_key_id(&self) -> Option<&EnvironmentRegistrationKeyId> { + match self { + Self::Registered(source) => Some(&source.registration_key_id), + Self::Provisioned(_) | Self::External { .. } => None, + } + } + + /// The stored `source_kind`: `provisioned`, `external` or `registered`. + pub fn kind_str(&self) -> &'static str { + match self { + Self::Provisioned(_) => "provisioned", + Self::External { .. } => "external", + Self::Registered(_) => "registered", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentRecord { + pub environment_id: EnvironmentId, + /// Caller retry key; `None` for registered environments, whose daemon + /// public key makes a retried registration converge. + pub request_id: Option, + pub source: EnvironmentSource, + pub display_name: Option, + /// Caller metadata only. + pub metadata: BTreeMap, + pub state: EnvironmentState, + /// Why the row failed, or what provisioning or closing waits on. At most + /// [`STATE_MESSAGE_MAX_BYTES`]. + pub state_message: Option, + /// Set exactly while `closing` or `closed`. + pub close_reason: Option, + pub state_changed_at_ms: i64, + /// The daemon that serves it: from `registration/register`, or the open + /// handshake. + pub daemon_build: Option, + pub created_at_ms: i64, + pub updated_at_ms: i64, + pub closed_at_ms: Option, +} + +/// Why an environment cannot be used yet. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentWaitReason { + /// Still provisioning. + Provisioning, + /// The registered daemon is not connected. + Disconnected, + /// The target is resuming from a powered-down state. + Waking, + /// The target runs but `envd` is not answering yet. + Booting, +} + +impl EnvironmentWaitReason { + pub fn as_str(self) -> &'static str { + match self { + Self::Provisioning => "provisioning", + Self::Disconnected => "disconnected", + Self::Waking => "waking", + Self::Booting => "booting", + } + } +} + +impl fmt::Display for EnvironmentWaitReason { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Why an environment cannot be used at all. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentGoneReason { + Closing, + Closed, + Failed, + /// The registered daemon has been away past its reconnect bound. + Away, +} + +impl EnvironmentGoneReason { + pub fn as_str(self) -> &'static str { + match self { + Self::Closing => "closing", + Self::Closed => "closed", + Self::Failed => "failed", + Self::Away => "away", + } + } +} + +impl fmt::Display for EnvironmentGoneReason { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// The single answer to "may I use this environment now". +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EnvironmentUse { + /// Dial it. + Use, + /// Not usable yet: re-check after `retry_after_ms`; `until_ms` bounds + /// the wait. + Wait { + reason: EnvironmentWaitReason, + retry_after_ms: u64, + until_ms: Option, + }, + /// Not usable: closing, closed, failed, or a daemon away past its bound. + Gone { + reason: EnvironmentGoneReason, + message: Option, + }, +} + +impl EnvironmentUse { + /// What a typed data-route refusal means for the caller. Callers that + /// hold the record use [`EnvironmentRecord::bound_route_refusal`]. + pub fn from_route_refusal(refusal: &DataRouteRefusal, retry_after_ms: Option) -> Self { + let wait = |reason| Self::Wait { + reason, + retry_after_ms: retry_after_ms.unwrap_or(WAIT_RETRY_AFTER_MS), + until_ms: None, + }; + match refusal { + DataRouteRefusal::Waking {} => wait(EnvironmentWaitReason::Waking), + DataRouteRefusal::Booting {} => wait(EnvironmentWaitReason::Booting), + DataRouteRefusal::Provisioning {} => wait(EnvironmentWaitReason::Provisioning), + DataRouteRefusal::Disconnected {} => wait(EnvironmentWaitReason::Disconnected), + DataRouteRefusal::Away {} => Self::Gone { + reason: EnvironmentGoneReason::Away, + message: None, + }, + DataRouteRefusal::Gone {} => Self::Gone { + reason: EnvironmentGoneReason::Closed, + message: None, + }, + DataRouteRefusal::Failed { message } => Self::Gone { + reason: EnvironmentGoneReason::Failed, + message: Some(message.as_str().to_owned()), + }, + } + } +} + +impl EnvironmentRecord { + pub fn is_registered(&self) -> bool { + matches!(self.source, EnvironmentSource::Registered(_)) + } + + pub fn provider_id(&self) -> Option<&EnvironmentProviderId> { + self.source.provider_id() + } + + pub fn registration_key_id(&self) -> Option<&EnvironmentRegistrationKeyId> { + self.source.registration_key_id() + } + + pub fn provisioned(&self) -> Option<&ProvisionedSource> { + match &self.source { + EnvironmentSource::Provisioned(source) => Some(source), + EnvironmentSource::External { .. } | EnvironmentSource::Registered(_) => None, + } + } + + pub fn registered(&self) -> Option<&RegisteredSource> { + match &self.source { + EnvironmentSource::Registered(source) => Some(source), + EnvironmentSource::Provisioned(_) | EnvironmentSource::External { .. } => None, + } + } + + /// The display status: the state, refined on an open row by the + /// registered connection or the provider's last observation. + pub fn display_status(&self) -> EnvironmentDisplayStatus { + match self.state { + EnvironmentState::Provisioning => EnvironmentDisplayStatus::Provisioning, + EnvironmentState::Closing => EnvironmentDisplayStatus::Closing, + EnvironmentState::Closed => EnvironmentDisplayStatus::Closed, + EnvironmentState::Failed => EnvironmentDisplayStatus::Failed, + EnvironmentState::Open => match &self.source { + EnvironmentSource::External { .. } => EnvironmentDisplayStatus::Ready, + EnvironmentSource::Registered(source) => { + if source.connected { + EnvironmentDisplayStatus::Ready + } else { + EnvironmentDisplayStatus::Away + } + } + EnvironmentSource::Provisioned(source) => match &source.observed { + None => EnvironmentDisplayStatus::Ready, + Some(observed) => match observed.phase { + TargetPhase::Active => match observed.power { + None | Some(PowerState::Running) => EnvironmentDisplayStatus::Ready, + Some(PowerState::Paused) => EnvironmentDisplayStatus::Paused, + Some(PowerState::Suspended) => EnvironmentDisplayStatus::Suspended, + Some(PowerState::Stopped) => EnvironmentDisplayStatus::Stopped, + }, + TargetPhase::Provisioning => EnvironmentDisplayStatus::Provisioning, + TargetPhase::Closing => EnvironmentDisplayStatus::Closing, + TargetPhase::Unknown | TargetPhase::Failed | TargetPhase::Closed => { + EnvironmentDisplayStatus::Unknown + } + }, + }, + }, + } + } + + /// The use gate every consumer shares: selection, the resolver, job + /// activities, discovery and the gateway route. It only reads. + pub fn use_gate(&self, now_ms: i64) -> EnvironmentUse { + match self.state { + EnvironmentState::Closing => gone(EnvironmentGoneReason::Closing, None), + EnvironmentState::Closed => gone(EnvironmentGoneReason::Closed, None), + EnvironmentState::Failed => { + gone(EnvironmentGoneReason::Failed, self.state_message.clone()) + } + EnvironmentState::Provisioning => EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Provisioning, + retry_after_ms: WAIT_RETRY_AFTER_MS, + until_ms: None, + }, + EnvironmentState::Open => match &self.source { + EnvironmentSource::Registered(source) if !source.connected => { + let away = || { + gone( + EnvironmentGoneReason::Away, + Some(format!( + "daemon away for {}", + compact_duration( + now_ms.saturating_sub(source.connection_changed_at_ms) + ) + )), + ) + }; + match source.identity_mode { + RegisteredIdentityMode::Ephemeral => away(), + RegisteredIdentityMode::Persistent => { + let until = source + .connection_changed_at_ms + .saturating_add(REGISTERED_RECONNECT_WAIT_MS); + if now_ms < until { + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Disconnected, + retry_after_ms: WAIT_RETRY_AFTER_MS, + until_ms: Some(until), + } + } else { + away() + } + } + } + } + _ => EnvironmentUse::Use, + }, + } + } + + /// What a typed data-route refusal means for this environment: + /// [`EnvironmentUse::from_route_refusal`], except that a `disconnected` + /// refusal of a registered environment follows its identity mode. The + /// row read connected, but the gateway held no control handle: the + /// moment between a dropped connection and its disconnect write. An + /// ephemeral daemon is then away at once; a persistent one is waited + /// for, unbounded here, so the caller bounds consecutive waits by + /// [`REGISTERED_RECONNECT_WAIT_MS`] until the row records the + /// disconnect and the gate's own bound applies. + pub fn bound_route_refusal( + &self, + refusal: &DataRouteRefusal, + retry_after_ms: Option, + ) -> EnvironmentUse { + let answer = EnvironmentUse::from_route_refusal(refusal, retry_after_ms); + match (&self.source, refusal) { + (EnvironmentSource::Registered(source), DataRouteRefusal::Disconnected {}) + if source.identity_mode == RegisteredIdentityMode::Ephemeral => + { + gone( + EnvironmentGoneReason::Away, + Some("daemon is not connected".to_owned()), + ) + } + _ => answer, + } + } + + pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { + validate_nonempty_optional("display_name", self.display_name.as_deref())?; + validate_caller_metadata(&self.metadata) + .map_err(|error| invalid_error(error.to_string()))?; + validate_timestamps(self.created_at_ms, self.updated_at_ms)?; + if self.state_changed_at_ms < self.created_at_ms { + return invalid("state_changed_at_ms must be >= created_at_ms"); + } + if let Some(message) = &self.state_message { + validate_descriptive_text(message) + .map_err(|error| invalid_error(format!("state message: {error}")))?; + } + if let Some(build) = &self.daemon_build { + build.validate()?; + } + let closing_or_closed = matches!( + self.state, + EnvironmentState::Closing | EnvironmentState::Closed + ); + if self.close_reason.is_some() != closing_or_closed { + return invalid("a close reason is recorded exactly while closing or closed"); + } + match self.closed_at_ms { + Some(closed_at) if self.state == EnvironmentState::Closed => { + if closed_at < self.created_at_ms { + return invalid("closed_at_ms must be >= created_at_ms"); + } + } + None if self.state != EnvironmentState::Closed => {} + _ => return invalid("closed_at_ms is recorded exactly while closed"), + } + if self.request_id.is_some() == self.is_registered() { + return invalid( + "provisioned and external environments carry a request id; registered ones do not", + ); + } + match &self.source { + EnvironmentSource::Provisioned(source) => { + let scheduled = matches!( + self.state, + EnvironmentState::Provisioning | EnvironmentState::Closing + ); + if source.next_attempt_at_ms.is_some() != scheduled { + return invalid( + "a provisioned environment is scheduled exactly while provisioning or closing", + ); + } + if let Some(policy) = &source.idle { + policy + .validate() + .map_err(|error| invalid_error(error.to_string()))?; + } + if let Some(observed) = &source.observed { + observed.validate()?; + } + if let Some(endpoint) = &source.public_endpoint { + if endpoint.is_empty() || endpoint.len() > MAX_ENDPOINT_BYTES { + return invalid(format!( + "public_endpoint must be 1..={MAX_ENDPOINT_BYTES} bytes" + )); + } + if self.state == EnvironmentState::Closed { + return invalid("a closed environment has no public_endpoint"); + } + } + } + EnvironmentSource::External { .. } => self.validate_open_or_closed("external")?, + EnvironmentSource::Registered(source) => { + self.validate_open_or_closed("registered")?; + parse_daemon_public_key(&source.daemon_public_key)?; + validate_nonnegative_i64( + source.connection_changed_at_ms, + "connection_changed_at_ms", + )?; + validate_nonnegative_i64(source.last_seen_at_ms, "last_seen_at_ms")?; + } + } + Ok(()) + } + + fn validate_open_or_closed(&self, kind: &str) -> Result<(), EnvironmentRegistryError> { + if !matches!( + self.state, + EnvironmentState::Open | EnvironmentState::Closed + ) { + return invalid(format!("{kind} environments are only open or closed")); + } + Ok(()) + } + + fn provisioned_mut(&mut self) -> Option<&mut ProvisionedSource> { + match &mut self.source { + EnvironmentSource::Provisioned(source) => Some(source), + EnvironmentSource::External { .. } | EnvironmentSource::Registered(_) => None, + } + } + + fn registered_mut(&mut self) -> Option<&mut RegisteredSource> { + match &mut self.source { + EnvironmentSource::Registered(source) => Some(source), + EnvironmentSource::Provisioned(_) | EnvironmentSource::External { .. } => None, + } + } + + /// Move to `state` at `at_ms`, clamped so the row's times stay ordered. + /// Returns the clamped time. + fn transition(&mut self, state: EnvironmentState, at_ms: i64) -> i64 { + let at = at_ms.max(self.created_at_ms).max(self.state_changed_at_ms); + self.state = state; + self.state_changed_at_ms = at; + self.touch(at); + at + } + + fn touch(&mut self, at_ms: i64) { + self.updated_at_ms = self.updated_at_ms.max(at_ms); + } + + /// Clear the work schedule of a provisioned row. + fn clear_schedule(&mut self) { + if let Some(source) = self.provisioned_mut() { + source.next_attempt_at_ms = None; + source.attempts = 0; + } + } + + fn close(&mut self, reason: EnvironmentCloseReason, at_ms: i64) { + let at = self.transition(EnvironmentState::Closed, at_ms); + self.close_reason = Some(reason); + self.closed_at_ms = Some(at); + self.state_message = None; + self.clear_schedule(); + // A closed target serves nothing. + if let Some(source) = self.provisioned_mut() { + source.public_endpoint = None; + } + } +} + +fn gone(reason: EnvironmentGoneReason, message: Option) -> EnvironmentUse { + EnvironmentUse::Gone { reason, message } +} + +/// A `state_message` built from any text: control characters cleaned and +/// cut to [`STATE_MESSAGE_MAX_BYTES`]. `None` when nothing is left. +pub fn clean_state_message(text: &str) -> Option { + DescriptiveText::lossy(text).map(DescriptiveText::into_string) +} + +// --------------------------------------------------------------------------- +// Creates +// --------------------------------------------------------------------------- + +/// Create a provisioned environment. The store checks that the universe's +/// binding of `provider_id` exists (`NotFound`) and is enabled +/// (`InvalidInput`). The same `request_id` with identical inputs returns the +/// stored row; with different inputs it is `AlreadyExists`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CreateEnvironment { + pub request_id: EnvironmentProvisionRequestId, + pub environment_id: EnvironmentId, + pub provider_id: EnvironmentProviderId, + pub template_id: EnvironmentTemplateId, + pub idle: Option, + pub display_name: Option, + pub metadata: BTreeMap, + pub created_at_ms: i64, +} + +impl CreateEnvironment { + /// The validated record this create inserts: `provisioning`, due at + /// once. + pub fn record(&self) -> Result { + let record = new_record( + self.environment_id.clone(), + Some(self.request_id.clone()), + EnvironmentSource::Provisioned(ProvisionedSource { + provider_id: self.provider_id.clone(), + template_id: self.template_id.clone(), + idle: self.idle.clone(), + provider_target_id: None, + observed: None, + public_endpoint: None, + next_attempt_at_ms: Some(self.created_at_ms), + attempts: 0, + }), + EnvironmentState::Provisioning, + self.display_name.clone(), + self.metadata.clone(), + None, + self.created_at_ms, + ); + record.validate()?; + Ok(record) + } + + /// True when `record` is what this create made: a retry with the same + /// inputs. The environment id and the time of a retry are not inputs. + pub fn matches(&self, record: &EnvironmentRecord) -> bool { + record.request_id.as_ref() == Some(&self.request_id) + && record.display_name == self.display_name + && record.metadata == self.metadata + && record.provisioned().is_some_and(|source| { + source.provider_id == self.provider_id + && source.template_id == self.template_id + && source.idle == self.idle + }) + } +} + +/// Create an external environment, `open` at once. The same `request_id` +/// with identical inputs returns the stored row; with different inputs it is +/// `AlreadyExists`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CreateExternalEnvironment { + pub request_id: EnvironmentProvisionRequestId, + pub environment_id: EnvironmentId, + pub endpoint: EnvironmentEndpoint, + pub display_name: Option, + pub metadata: BTreeMap, + pub created_at_ms: i64, +} + +impl CreateExternalEnvironment { + /// The validated record this create inserts. + pub fn record(&self) -> Result { + let record = new_record( + self.environment_id.clone(), + Some(self.request_id.clone()), + EnvironmentSource::External { + endpoint: self.endpoint.clone(), + }, + EnvironmentState::Open, + self.display_name.clone(), + self.metadata.clone(), + None, + self.created_at_ms, + ); + record.validate()?; + Ok(record) + } + + /// True when `record` is what this create made. + pub fn matches(&self, record: &EnvironmentRecord) -> bool { + record.request_id.as_ref() == Some(&self.request_id) + && record.display_name == self.display_name + && record.metadata == self.metadata + && matches!(&record.source, EnvironmentSource::External { endpoint } if endpoint == &self.endpoint) + } +} + +/// Admission of a first-seen daemon identity. The store locks the key, +/// re-checks that it still admits and that the presented secret is still +/// its live one, counts the key's non-closed environments against its +/// limit, and inserts the environment `open` and connected, all in one +/// transaction. Rotation takes the same lock, so a secret it retired admits +/// nothing once the rotation commits. A retry with the same daemon public +/// key returns the row it admitted (closed included: the identity is +/// spent). #[derive(Clone, Debug, PartialEq, Eq)] pub struct CreateRegisteredEnvironment { pub registration_key_id: EnvironmentRegistrationKeyId, + /// [`registration_key_hash`] of the secret the daemon presented. + pub secret_hash: String, pub environment_id: EnvironmentId, - pub incarnation_id: EnvironmentIncarnationId, /// Lowercase hex of the raw Ed25519 public key. pub daemon_public_key: String, pub display_name: Option, pub metadata: BTreeMap, + pub daemon_build: DaemonBuild, pub created_at_ms: i64, } impl CreateRegisteredEnvironment { + /// The refusal when the presented secret is no longer the key's live + /// one: a rotation retired it after the caller resolved the key. + pub fn secret_retired(&self) -> EnvironmentRegistryError { + EnvironmentRegistryError::RegistrationKeyUnavailable { + registration_key_id: self.registration_key_id.to_string(), + reason: "rotated", + } + } + pub fn daemon_id(&self) -> Result { - validate_daemon_public_key(&self.daemon_public_key)?; - Ok(EnvironmentDaemonId::from_public_key(&decode_hex( - &self.daemon_public_key, - )?)) + let public_key = parse_daemon_public_key(&self.daemon_public_key)?; + Ok(EnvironmentDaemonId::from_public_key(public_key.as_bytes())) + } + + /// The validated record this admission inserts under `key`. + pub fn record( + &self, + key: &EnvironmentRegistrationKeyRecord, + ) -> Result { + if key.registration_key_id != self.registration_key_id { + return invalid("registered environment admitted under another key"); + } + let record = new_record( + self.environment_id.clone(), + None, + EnvironmentSource::Registered(RegisteredSource { + registration_key_id: key.registration_key_id.clone(), + daemon_public_key: self.daemon_public_key.clone(), + identity_mode: key.identity_mode, + connected: true, + connection_changed_at_ms: self.created_at_ms, + last_seen_at_ms: self.created_at_ms, + }), + EnvironmentState::Open, + self.display_name.clone(), + self.metadata.clone(), + Some(self.daemon_build.clone()), + self.created_at_ms, + ); + record.validate()?; + Ok(record) } } -/// A gateway observation of a registered daemon's control connection. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RegisteredConnectionObservation { - /// A control connection was admitted: `Ready` plus a fresh stamp. - Connected, - /// The connection is still alive: fresh stamp only. - Heartbeat, - /// The connection ended: `Offline`; the stamp keeps its last value. - Disconnected, +#[allow(clippy::too_many_arguments)] +fn new_record( + environment_id: EnvironmentId, + request_id: Option, + source: EnvironmentSource, + state: EnvironmentState, + display_name: Option, + metadata: BTreeMap, + daemon_build: Option, + created_at_ms: i64, +) -> EnvironmentRecord { + EnvironmentRecord { + environment_id, + request_id, + source, + display_name, + metadata, + state, + state_message: None, + close_reason: None, + state_changed_at_ms: created_at_ms, + daemon_build, + created_at_ms, + updated_at_ms: created_at_ms, + closed_at_ms: None, + } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ObserveRegisteredEnvironment { +// --------------------------------------------------------------------------- +// Writes +// +// Each `apply` changes `record` in place and returns whether it changed +// anything; a write whose expected state no longer holds returns `false` and +// leaves the record as it was. The stores validate the result before they +// keep it. +// --------------------------------------------------------------------------- + +/// Replace the display name and caller metadata. Refused for a closed +/// environment. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UpdateEnvironment { pub environment_id: EnvironmentId, - pub observation: RegisteredConnectionObservation, - pub observed_at_ms: i64, - /// Reserved-prefix entries the gateway learned about the daemon at this - /// admission, merged into the row so a replaced daemon shows its new - /// build. Empty for heartbeats and disconnects. Ignored once the - /// environment is closing or closed. - #[serde(default)] + pub display_name: Option, pub metadata: BTreeMap, + pub updated_at_ms: i64, } -/// Metadata keys the gateway writes about the daemon serving a registered -/// environment. They carry the reserved prefix, so a daemon cannot set them. -pub const ENVD_VERSION_METADATA_KEY: &str = "lightspeed.envd.version"; -pub const ENVD_GIT_SHA_METADATA_KEY: &str = "lightspeed.envd.gitSha"; -pub const ENVD_PROTOCOL_VERSION_METADATA_KEY: &str = "lightspeed.envd.protocolVersion"; +impl UpdateEnvironment { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.updated_at_ms, "updated_at_ms")?; + if record.state == EnvironmentState::Closed { + return invalid("environment closed"); + } + record.display_name = self.display_name.clone(); + record.metadata = self.metadata.clone(); + record.touch(self.updated_at_ms); + Ok(true) + } +} -/// What the gateway learned about a daemon's build when it registered or -/// reconnected. Recorded on the environment row at every admission so an -/// operator sees which build serves the environment. Never consulted for -/// admission: the protocol version alone decides that, and this only -/// records which number the admitted daemon spoke. +/// A known registered daemon reconnected: connected, both connection stamps +/// set, and its build replaced (it may be a newer one). Refused with +/// `InvalidInput` when the environment is closed: the identity is spent. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct RegisteredDaemonBuild { - pub version: Option, - pub git_sha: Option, - pub protocol_version: u32, +pub struct MarkRegisteredConnected { + pub environment_id: EnvironmentId, + pub daemon_build: DaemonBuild, + pub at_ms: i64, } -impl RegisteredDaemonBuild { - pub fn from_registration( - implementation: &environment_protocol::shared::ImplementationInfo, - protocol_version: u32, - ) -> Self { - Self { - version: implementation.version.clone(), - git_sha: implementation.git_sha.clone(), - protocol_version, +impl MarkRegisteredConnected { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + if record.state == EnvironmentState::Closed { + return invalid("environment closed"); } + let Some(source) = record.registered_mut() else { + return invalid("connection writes apply only to registered environments"); + }; + source.connected = true; + source.connection_changed_at_ms = self.at_ms; + source.last_seen_at_ms = self.at_ms; + record.daemon_build = Some(self.daemon_build.clone()); + record.touch(self.at_ms); + Ok(true) } +} - /// The reserved entries for this build. A fact the daemon did not report - /// is left out, so the row keeps that key's last value. - pub fn metadata(&self) -> BTreeMap { - let mut metadata = BTreeMap::new(); - if let Some(version) = &self.version { - metadata.insert(ENVD_VERSION_METADATA_KEY.to_owned(), version.clone()); - } - if let Some(git_sha) = &self.git_sha { - metadata.insert(ENVD_GIT_SHA_METADATA_KEY.to_owned(), git_sha.clone()); +/// The current control connection of a registered daemon ended. Written +/// only while the row reads connected. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MarkRegisteredDisconnected { + pub environment_id: EnvironmentId, + pub at_ms: i64, +} + +impl MarkRegisteredDisconnected { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + let Some(source) = record.registered_mut() else { + return invalid("connection writes apply only to registered environments"); + }; + if !source.connected { + return Ok(false); } - metadata.insert( - ENVD_PROTOCOL_VERSION_METADATA_KEY.to_owned(), - self.protocol_version.to_string(), - ); - metadata + source.connected = false; + source.connection_changed_at_ms = self.at_ms; + record.touch(self.at_ms); + Ok(true) } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct CreateEnvironment { - pub request_id: EnvironmentProvisionRequestId, +/// The coarse liveness stamp of a connected registered daemon: written only +/// while open and connected, and only once the last stamp is at least +/// [`REGISTERED_LIVENESS_STAMP_MS`] old. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StampRegisteredSeen { pub environment_id: EnvironmentId, - pub incarnation_id: EnvironmentIncarnationId, - pub binding_id: EnvironmentProviderBindingId, - pub template_id: EnvironmentTemplateId, - pub display_name: Option, - pub metadata: BTreeMap, - pub idle_policy: Option, - pub created_at_ms: i64, + pub at_ms: i64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AdoptEnvironment { - pub request_id: EnvironmentProvisionRequestId, - pub environment_id: EnvironmentId, - pub incarnation_id: EnvironmentIncarnationId, - pub binding_id: EnvironmentProviderBindingId, - pub source_target: String, - pub display_name: Option, - pub metadata: BTreeMap, - pub created_at_ms: i64, +impl StampRegisteredSeen { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + let open = record.state == EnvironmentState::Open; + let Some(source) = record.registered_mut() else { + return invalid("connection writes apply only to registered environments"); + }; + if !open + || !source.connected + || source.last_seen_at_ms > self.at_ms.saturating_sub(REGISTERED_LIVENESS_STAMP_MS) + { + return Ok(false); + } + source.last_seen_at_ms = self.at_ms; + record.touch(self.at_ms); + Ok(true) + } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct CreateExternalEnvironment { - pub request_id: EnvironmentProvisionRequestId, - pub environment_id: EnvironmentId, - pub incarnation_id: EnvironmentIncarnationId, - pub connection: EnvironmentConnectionSpec, - pub display_name: Option, - pub metadata: BTreeMap, - pub created_at_ms: i64, +/// What one registered sweep changed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RegisteredSweep { + /// Rows that read connected with a stale liveness stamp, now + /// disconnected as of that stamp. + pub disconnected: Vec, + /// Rows the sweep closed, with the reason. + pub closed: Vec<(EnvironmentId, EnvironmentCloseReason)>, +} + +/// One registered sweep step for an open registered row under its key, at +/// `now_ms`. In order: +/// +/// 1. connected with `last_seen_at_ms < now - REGISTERED_STALE_AFTER_MS` → +/// disconnected, `connection_changed_at_ms = last_seen_at_ms`; +/// 2. disconnected, ephemeral, away longer than the key's grace → closed +/// (`ephemeral_grace`); +/// 3. disconnected, persistent, the key sets `offline_close_after_ms` and +/// the daemon is away longer → closed (`offline_timeout`). +/// +/// Returns whether step 1 applied and the close reason of step 2 or 3. +pub fn sweep_registered_environment( + record: &mut EnvironmentRecord, + key: &EnvironmentRegistrationKeyRecord, + now_ms: i64, +) -> (bool, Option) { + if record.state != EnvironmentState::Open { + return (false, None); + } + let Some(source) = record.registered_mut() else { + return (false, None); + }; + let mut disconnected = false; + if source.connected && source.last_seen_at_ms < now_ms.saturating_sub(REGISTERED_STALE_AFTER_MS) + { + source.connected = false; + source.connection_changed_at_ms = source.last_seen_at_ms; + disconnected = true; + } + let close = if source.connected { + None + } else { + let away_ms = now_ms.saturating_sub(source.connection_changed_at_ms); + match source.identity_mode { + RegisteredIdentityMode::Ephemeral + if away_ms > duration_ms(key.ephemeral_disconnect_grace_ms()) => + { + Some(EnvironmentCloseReason::EphemeralGrace) + } + RegisteredIdentityMode::Persistent + if key + .offline_close_after_ms + .is_some_and(|after| away_ms > duration_ms(after)) => + { + Some(EnvironmentCloseReason::OfflineTimeout) + } + _ => None, + } + }; + if disconnected { + record.touch(now_ms); + } + if let Some(reason) = close { + record.close(reason, now_ms); + } + (disconnected, close) } -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct ListEnvironments { - pub provider_id: Option, - pub binding_id: Option, - pub status: Option, - /// Only environments admitted by this registration key. - pub registration_key_id: Option, - /// Only environments carrying every listed metadata pair (containment). - #[serde(default)] - pub metadata: BTreeMap, +fn duration_ms(value: u64) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ObserveProvisionedEnvironment { +/// A span for people to read, such as `45s`, `3m12s` or `1h5m`. Negative +/// spans read as `0s`. +fn compact_duration(ms: i64) -> String { + let seconds = ms.max(0) / 1_000; + let (hours, minutes, seconds) = (seconds / 3_600, seconds / 60 % 60, seconds % 60); + match (hours, minutes, seconds) { + (0, 0, seconds) => format!("{seconds}s"), + (0, minutes, 0) => format!("{minutes}m"), + (0, minutes, seconds) => format!("{minutes}m{seconds}s"), + (hours, 0, _) => format!("{hours}h"), + (hours, minutes, _) => format!("{hours}h{minutes}m"), + } +} + +/// What the provider reported about a provisioned environment's target, in +/// any state. The target id is recorded (a different id than the recorded +/// one is refused); the observation and public endpoint replace the stored +/// ones unless an observation stamped later is already stored. The stored +/// observation is stamped with `observed_at_ms`. A closed row keeps no +/// public endpoint. Never changes the state. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordTargetObservation { pub environment_id: EnvironmentId, - pub provider_target_id: ProviderTargetId, - pub status: EnvironmentStatus, - /// Provider-reported supported power states for this target. - pub power_states: Vec, + /// Absent while an asynchronous provider has not assigned one yet; the + /// recorded id is kept. + pub provider_target_id: Option, + pub observed: ObservedTarget, + pub public_endpoint: Option, pub observed_at_ms: i64, } -/// Set the Lightspeed-owned power intent. Rejected for external, closing, -/// closed, and failed environments; the caller validates provider support. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SetEnvironmentPower { +impl RecordTargetObservation { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.observed_at_ms, "observed_at_ms")?; + let closed = record.state == EnvironmentState::Closed; + let Some(source) = record.provisioned_mut() else { + return invalid("provider observations apply only to provisioned environments"); + }; + if let (Some(recorded), Some(observed)) = ( + source.provider_target_id.as_ref(), + self.provider_target_id.as_ref(), + ) && recorded != observed + { + return invalid("provider target conflicts with the recorded target"); + } + let mut changed = false; + if source.provider_target_id.is_none() && self.provider_target_id.is_some() { + source.provider_target_id = self.provider_target_id.clone(); + changed = true; + } + let fresh = source + .observed + .as_ref() + .is_none_or(|previous| self.observed_at_ms >= previous.observed_at_ms); + if fresh { + source.observed = Some(ObservedTarget { + observed_at_ms: self.observed_at_ms, + ..self.observed.clone() + }); + if !closed { + source.public_endpoint = self.public_endpoint.clone(); + } + changed = true; + } + if changed { + record.touch(self.observed_at_ms); + } + Ok(changed) + } +} + +/// Reschedule the work of a provisioning or closing row: the next attempt, +/// the attempt count and what it waits on. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScheduleEnvironmentAttempt { pub environment_id: EnvironmentId, - pub desired_power: PowerState, - pub updated_at_ms: i64, + /// The state of the row the step read, `provisioning` or `closing`. The + /// write is skipped when the row has moved on, so a close that lands + /// while a provision step is in flight keeps its own schedule. + pub expected_state: EnvironmentState, + pub next_attempt_at_ms: i64, + pub attempts: u32, + /// Cleaned and cut by [`clean_state_message`]. + pub state_message: Option, + pub at_ms: i64, +} + +impl ScheduleEnvironmentAttempt { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + validate_nonnegative_i64(self.next_attempt_at_ms, "next_attempt_at_ms")?; + if record.state != self.expected_state + || !matches!( + record.state, + EnvironmentState::Provisioning | EnvironmentState::Closing + ) + { + return Ok(false); + } + let Some(source) = record.provisioned_mut() else { + return Ok(false); + }; + source.next_attempt_at_ms = Some(self.next_attempt_at_ms); + source.attempts = self.attempts; + record.state_message = self.state_message.as_deref().and_then(clean_state_message); + record.touch(self.at_ms); + Ok(true) + } } -/// Replace (or clear) the idle policy of a provisioned environment. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SetEnvironmentIdlePolicy { +/// The open handshake of a provisioning row succeeded: `open`, schedule and +/// message cleared, the daemon's build recorded. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OpenEnvironment { pub environment_id: EnvironmentId, - pub idle_policy: Option, - pub updated_at_ms: i64, + pub daemon_build: DaemonBuild, + pub at_ms: i64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct FailEnvironmentLifecycle { +impl OpenEnvironment { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + if record.provisioned().is_none() || record.state != EnvironmentState::Provisioning { + return Ok(false); + } + record.transition(EnvironmentState::Open, self.at_ms); + record.clear_schedule(); + record.state_message = None; + record.daemon_build = Some(self.daemon_build.clone()); + Ok(true) + } +} + +/// A provisioning or open provisioned row is broken: `failed` with +/// `message`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FailEnvironment { pub environment_id: EnvironmentId, pub message: String, - pub observed_at_ms: i64, + /// The daemon whose build made the environment unusable, such as one + /// that speaks another protocol version. + pub daemon_build: Option, + pub at_ms: i64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +impl FailEnvironment { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + if record.provisioned().is_none() { + return invalid("only provisioned environments fail"); + } + if !matches!( + record.state, + EnvironmentState::Provisioning | EnvironmentState::Open + ) { + return Ok(false); + } + record.transition(EnvironmentState::Failed, self.at_ms); + record.clear_schedule(); + record.state_message = Some( + clean_state_message(&self.message).unwrap_or_else(|| "unspecified failure".to_owned()), + ); + if let Some(build) = &self.daemon_build { + record.daemon_build = Some(build.clone()); + } + Ok(true) + } +} + +/// Begin closing. A provisioned row moves from provisioning, open or failed +/// to `closing`, due at once; an external or registered row moves from open +/// straight to `closed`. A closing or closed row is left as it is. +#[derive(Clone, Debug, PartialEq, Eq)] pub struct BeginCloseEnvironment { pub environment_id: EnvironmentId, - pub updated_at_ms: i64, + pub reason: EnvironmentCloseReason, + pub at_ms: i64, +} + +impl BeginCloseEnvironment { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + if matches!( + record.state, + EnvironmentState::Closing | EnvironmentState::Closed + ) { + return Ok(false); + } + if record.provisioned().is_some() { + let at = record.transition(EnvironmentState::Closing, self.at_ms); + record.close_reason = Some(self.reason); + record.state_message = None; + if let Some(source) = record.provisioned_mut() { + source.next_attempt_at_ms = Some(at); + source.attempts = 0; + } + } else { + record.close(self.reason, self.at_ms); + } + Ok(true) + } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +/// The provider confirmed a closing row's target gone: `closed`. +#[derive(Clone, Debug, PartialEq, Eq)] pub struct FinishCloseEnvironment { pub environment_id: EnvironmentId, - pub observed_at_ms: i64, + pub at_ms: i64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SetEnvironmentIngress { +impl FinishCloseEnvironment { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + if record.state != EnvironmentState::Closing { + return Ok(false); + } + let reason = record + .close_reason + .unwrap_or(EnvironmentCloseReason::Requested); + record.close(reason, self.at_ms); + Ok(true) + } +} + +/// The provider reported an open or failed row's target closed or not +/// found: `closed` with `provider_ended`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EndProvisionedEnvironment { pub environment_id: EnvironmentId, - pub enabled: bool, - pub public_endpoint: Option, - pub updated_at_ms: i64, + pub at_ms: i64, +} + +impl EndProvisionedEnvironment { + pub fn apply(&self, record: &mut EnvironmentRecord) -> Result { + validate_nonnegative_i64(self.at_ms, "at_ms")?; + if record.provisioned().is_none() + || !matches!( + record.state, + EnvironmentState::Open | EnvironmentState::Failed + ) + { + return Ok(false); + } + record.close(EnvironmentCloseReason::ProviderEnded, self.at_ms); + Ok(true) + } +} + +/// One page of environments, ordered by environment id. Closed rows are +/// listed only with `include_closed` or `state == Closed`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ListEnvironments { + pub provider_id: Option, + /// Only environments admitted by this registration key. + pub registration_key_id: Option, + pub state: Option, + pub include_closed: bool, + /// Only environments carrying every listed metadata pair (containment). + pub metadata: BTreeMap, + /// 1 to [`LIST_ENVIRONMENTS_MAX_LIMIT`]. + pub limit: u32, + /// Continue after this environment id (exclusive). + pub after: Option, +} + +impl Default for ListEnvironments { + fn default() -> Self { + Self { + provider_id: None, + registration_key_id: None, + state: None, + include_closed: false, + metadata: BTreeMap::new(), + limit: LIST_ENVIRONMENTS_DEFAULT_LIMIT, + after: None, + } + } +} + +impl ListEnvironments { + pub fn validate(&self) -> Result<(), EnvironmentRegistryError> { + if !(1..=LIST_ENVIRONMENTS_MAX_LIMIT).contains(&self.limit) { + return invalid(format!( + "list limit must be 1..={LIST_ENVIRONMENTS_MAX_LIMIT}" + )); + } + Ok(()) + } + + /// Whether `record` passes every filter except the page bounds. + pub fn matches(&self, record: &EnvironmentRecord) -> bool { + let closed_listed = self.include_closed || self.state == Some(EnvironmentState::Closed); + (record.state != EnvironmentState::Closed || closed_listed) + && self.state.is_none_or(|state| state == record.state) + && self + .provider_id + .as_ref() + .is_none_or(|id| record.provider_id() == Some(id)) + && self + .registration_key_id + .as_ref() + .is_none_or(|id| record.registration_key_id() == Some(id)) + && harness::storage::metadata_matches(&record.metadata, &self.metadata) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct EnvironmentPage { + pub environments: Vec, + /// Pass as `after` for the next page; `None` on the last page. + pub next_after: Option, } +// --------------------------------------------------------------------------- +// Environment credentials +// --------------------------------------------------------------------------- + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct EnvironmentCredentialRecord { pub environment_id: EnvironmentId, @@ -1162,28 +2197,40 @@ pub struct ListEnvironmentCredentials { pub environment_id: EnvironmentId, } +// --------------------------------------------------------------------------- +// Store traits +// --------------------------------------------------------------------------- + +/// Deployment-scoped providers. #[async_trait] pub trait EnvironmentProviderStore: Send + Sync { async fn put_provider( &self, - record: PutEnvironmentProvider, + request: PutEnvironmentProvider, ) -> Result; async fn read_provider( &self, provider_id: &EnvironmentProviderId, ) -> Result; + /// The provider's bearer credential, unsealed; `None` when it has none. + async fn read_provider_credential( + &self, + provider_id: &EnvironmentProviderId, + ) -> Result, EnvironmentRegistryError>; async fn list_providers( &self, - request: ListEnvironmentProviders, ) -> Result, EnvironmentRegistryError>; + /// Refused while bindings or non-closed environments reference it. async fn delete_provider( &self, provider_id: &EnvironmentProviderId, ) -> Result; } +/// A universe's provider bindings, one per provider. #[async_trait] pub trait EnvironmentProviderBindingStore: Send + Sync { + /// The provider must exist. async fn put_provider_binding( &self, request: PutEnvironmentProviderBinding, @@ -1191,29 +2238,29 @@ pub trait EnvironmentProviderBindingStore: Send + Sync { async fn read_provider_binding( &self, universe_id: Uuid, - binding_id: &EnvironmentProviderBindingId, + provider_id: &EnvironmentProviderId, ) -> Result; async fn list_provider_bindings( &self, universe_id: Uuid, ) -> Result, EnvironmentRegistryError>; + /// Refused while the universe has non-closed environments of the + /// provider. async fn delete_provider_binding( &self, universe_id: Uuid, - binding_id: &EnvironmentProviderBindingId, + provider_id: &EnvironmentProviderId, ) -> Result; } +/// A universe's environments. Lifecycle writes return the current record, +/// whether they applied or were skipped. #[async_trait] pub trait EnvironmentStore: Send + Sync { async fn create_environment( &self, request: CreateEnvironment, ) -> Result; - async fn adopt_environment( - &self, - request: AdoptEnvironment, - ) -> Result; async fn create_external_environment( &self, request: CreateExternalEnvironment, @@ -1222,70 +2269,94 @@ pub trait EnvironmentStore: Send + Sync { &self, request: CreateRegisteredEnvironment, ) -> Result; + async fn read_environment( + &self, + environment_id: &EnvironmentId, + ) -> Result; /// The one environment (closed included) bound to a daemon public key in /// this universe. async fn read_environment_by_daemon_public_key( &self, daemon_public_key: &str, ) -> Result, EnvironmentRegistryError>; - async fn observe_registered_environment( + /// Rows that no longer decode are left out and logged by the store, so + /// one bad row never hides the others; every list method does the same. + async fn list_environments( &self, - request: ObserveRegisteredEnvironment, + request: ListEnvironments, + ) -> Result; + async fn update_environment( + &self, + request: UpdateEnvironment, ) -> Result; - /// Registered environments that are not closing or closed: the - /// lifecycle reconciler's candidates for stale repair and ephemeral - /// cleanup. - async fn list_open_registered_environments( + async fn begin_close_environment( &self, - ) -> Result, EnvironmentRegistryError>; - async fn read_environment( + request: BeginCloseEnvironment, + ) -> Result; + async fn finish_close_environment( &self, - environment_id: &EnvironmentId, + request: FinishCloseEnvironment, ) -> Result; - async fn read_environment_by_request_id( + + async fn mark_registered_connected( &self, - request_id: &EnvironmentProvisionRequestId, + request: MarkRegisteredConnected, ) -> Result; - async fn list_environments( + async fn mark_registered_disconnected( &self, - request: ListEnvironments, - ) -> Result, EnvironmentRegistryError>; - async fn list_environments_needing_reconcile( + request: MarkRegisteredDisconnected, + ) -> Result; + /// Whether the stamp was written. + async fn stamp_registered_seen( &self, - ) -> Result, EnvironmentRegistryError>; - async fn observe_provisioned_environment( + request: StampRegisteredSeen, + ) -> Result; + /// Apply [`sweep_registered_environment`] to every open registered row + /// under its key. + async fn sweep_registered_environments( &self, - request: ObserveProvisionedEnvironment, - ) -> Result; - async fn fail_environment_lifecycle( + now_ms: i64, + ) -> Result; + /// After a gateway restart: every connected row becomes disconnected + /// with `connection_changed_at_ms = now_ms`, since the connections the + /// gateway held ended now, and persistent rows get their full reconnect + /// wait. `last_seen_at_ms` is kept. Returns the rows it changed. + async fn repair_registered_connections( &self, - request: FailEnvironmentLifecycle, - ) -> Result; - async fn begin_close_environment( + now_ms: i64, + ) -> Result; + + /// Provisioned rows whose next attempt is due at `now_ms`, earliest + /// first. + async fn list_environment_work( &self, - request: BeginCloseEnvironment, - ) -> Result; - async fn finish_close_environment( + now_ms: i64, + limit: u32, + ) -> Result, EnvironmentRegistryError>; + /// Open provisioned rows: the display sweep's candidates. + async fn list_open_provisioned_environments( &self, - request: FinishCloseEnvironment, + ) -> Result, EnvironmentRegistryError>; + async fn record_target_observation( + &self, + request: RecordTargetObservation, ) -> Result; - async fn set_environment_ingress( + async fn schedule_environment_attempt( &self, - request: SetEnvironmentIngress, + request: ScheduleEnvironmentAttempt, ) -> Result; - async fn set_environment_power( + async fn open_environment( &self, - request: SetEnvironmentPower, + request: OpenEnvironment, ) -> Result; - async fn set_environment_idle_policy( + async fn fail_environment( &self, - request: SetEnvironmentIdlePolicy, + request: FailEnvironment, ) -> Result; - /// Ready provisioned environments carrying an idle policy: the power - /// reaper's candidates. - async fn list_environments_with_idle_policy( + async fn end_provisioned_environment( &self, - ) -> Result, EnvironmentRegistryError>; + request: EndProvisionedEnvironment, + ) -> Result; } #[async_trait] @@ -1309,7 +2380,7 @@ pub fn template_record( value: &EnvironmentTemplate, ) -> Result { let template_id = - EnvironmentTemplateId::try_new(value.template_id.clone()).map_err(|error| { + EnvironmentTemplateId::try_new(value.template_id.as_str()).map_err(|error| { EnvironmentRegistryError::InvalidInput { message: format!("invalid template id: {error}"), } @@ -1317,13 +2388,19 @@ pub fn template_record( Ok(template_id) } +pub mod endpoint; +pub use endpoint::{EndpointError, EndpointScheme, EnvironmentEndpoint, MAX_ENDPOINT_BYTES}; mod memory; -pub use memory::{InMemoryEnvironmentRegistryStore, apply_registered_observation}; +pub use memory::InMemoryEnvironmentRegistryStore; fn invalid(message: impl Into) -> Result { - Err(EnvironmentRegistryError::InvalidInput { + Err(invalid_error(message)) +} + +fn invalid_error(message: impl Into) -> EnvironmentRegistryError { + EnvironmentRegistryError::InvalidInput { message: message.into(), - }) + } } fn validate_timestamps( @@ -1338,22 +2415,6 @@ fn validate_timestamps( Ok(()) } -fn validate_endpoint(name: &'static str, value: &str) -> Result<(), EnvironmentRegistryError> { - validate_nonempty_string(name, value)?; - if value.chars().any(char::is_whitespace) { - return invalid(format!("{name} must not contain whitespace")); - } - Ok(()) -} - -fn validate_provider_target_id(value: &ProviderTargetId) -> Result<(), EnvironmentRegistryError> { - validate_general_string_id("target_id", value.as_str()).map_err(|error| { - EnvironmentRegistryError::InvalidInput { - message: error.to_string(), - } - }) -} - fn validate_registration_display_name(value: &str) -> Result<(), EnvironmentRegistryError> { validate_nonempty_string("registration key display_name", value)?; if value.len() > REGISTRATION_KEY_DISPLAY_NAME_MAX_BYTES @@ -1367,60 +2428,26 @@ fn validate_registration_display_name(value: &str) -> Result<(), EnvironmentRegi Ok(()) } -/// A daemon public key is the raw 32-byte Ed25519 key as lowercase hex. -pub fn validate_daemon_public_key(value: &str) -> Result<(), EnvironmentRegistryError> { +/// A registration secret hash is lowercase hex SHA-256. +fn validate_secret_hash(value: &str) -> Result<(), EnvironmentRegistryError> { if value.len() != 64 || !value .bytes() - .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) { - return invalid("daemon public key must be 64 lowercase hex characters"); + return invalid("registration secret hash must be 64 lowercase hex characters"); } Ok(()) } -fn decode_hex(value: &str) -> Result, EnvironmentRegistryError> { - if !value.len().is_multiple_of(2) { - return invalid("hex value has odd length"); - } - (0..value.len()) - .step_by(2) - .map(|index| { - u8::from_str_radix(&value[index..index + 2], 16) - .map_err(|_| invalid_error("hex value contains a non-hex character")) - }) - .collect() -} - -fn invalid_error(message: impl Into) -> EnvironmentRegistryError { - EnvironmentRegistryError::InvalidInput { - message: message.into(), - } +/// A daemon public key is the raw 32-byte Ed25519 key as lowercase hex. +pub fn validate_daemon_public_key(value: &str) -> Result<(), EnvironmentRegistryError> { + parse_daemon_public_key(value).map(|_| ()) } -/// Stored records obey the registration handshake's bounds (entry count, -/// key and value bytes, no control characters). The reserved-prefix check -/// applies to caller input at the API boundary, not here: Lightspeed itself -/// annotates records under that prefix, and those entries sit on top of a -/// caller's full allowance rather than eating into it, so the entry count -/// applies to the caller's keys alone. -fn validate_metadata(value: &BTreeMap) -> Result<(), EnvironmentRegistryError> { - use environment_protocol::registration::{ - MAX_METADATA_ENTRIES, RESERVED_METADATA_PREFIX, validate_metadata_entry, - }; - let caller_entries = value - .keys() - .filter(|key| !key.starts_with(RESERVED_METADATA_PREFIX)) - .count(); - if caller_entries > MAX_METADATA_ENTRIES { - return Err(invalid_error(format!( - "metadata has more than {MAX_METADATA_ENTRIES} caller entries" - ))); - } - for (key, entry) in value { - validate_metadata_entry(key, entry).map_err(invalid_error)?; - } - Ok(()) +fn parse_daemon_public_key(value: &str) -> Result { + DaemonPublicKey::parse(value) + .map_err(|_| invalid_error("daemon public key must be 64 lowercase hex characters")) } fn validate_nonempty_optional( diff --git a/crates/environments/src/memory.rs b/crates/environments/src/memory.rs index fe9693f2e..1c908a054 100644 --- a/crates/environments/src/memory.rs +++ b/crates/environments/src/memory.rs @@ -6,23 +6,74 @@ use uuid::Uuid; use super::*; type CredentialKey = (EnvironmentId, String); -type BindingKey = (Uuid, EnvironmentProviderBindingId); +type BindingKey = (Uuid, EnvironmentProviderId); -struct StoredRegistrationKey { +struct StoredSecret { secret_hash: String, + retired_at_ms: Option, +} + +struct StoredRegistrationKey { record: EnvironmentRegistrationKeyRecord, + secrets: Vec, } #[derive(Default)] struct RegistryState { providers: BTreeMap, + provider_credentials: BTreeMap, bindings: BTreeMap, environments: BTreeMap, - requests: BTreeMap, credentials: BTreeMap, registration_keys: BTreeMap, } +impl RegistryState { + fn by_request_id( + &self, + request_id: &EnvironmentProvisionRequestId, + ) -> Option<&EnvironmentRecord> { + self.environments + .values() + .find(|record| record.request_id.as_ref() == Some(request_id)) + } + + fn by_daemon_public_key(&self, daemon_public_key: &str) -> Option<&EnvironmentRecord> { + self.environments.values().find(|record| { + record + .registered() + .is_some_and(|source| source.daemon_public_key == daemon_public_key) + }) + } + + fn insert_new( + &mut self, + record: EnvironmentRecord, + ) -> Result { + if self.environments.contains_key(&record.environment_id) { + return Err(EnvironmentRegistryError::AlreadyExists { + kind: "environment", + id: record.environment_id.to_string(), + }); + } + self.environments + .insert(record.environment_id.clone(), record.clone()); + Ok(record) + } + + fn registration_key( + &self, + registration_key_id: &EnvironmentRegistrationKeyId, + ) -> Result<&EnvironmentRegistrationKeyRecord, EnvironmentRegistryError> { + self.registration_keys + .get(registration_key_id) + .map(|stored| &stored.record) + .ok_or_else(|| not_found("environment_registration_key", registration_key_id)) + } +} + +/// An in-memory registry for one universe, following the same rules as the +/// PostgreSQL store through the shared `apply` functions. pub struct InMemoryEnvironmentRegistryStore { universe_id: Uuid, state: RwLock, @@ -69,6 +120,36 @@ impl InMemoryEnvironmentRegistryStore { message: "environment registry write lock poisoned".to_owned(), }) } + + /// Apply one write to one environment: on a copy, validated, stored + /// only when valid. A refused change leaves the stored record untouched. + fn change( + &self, + environment_id: &EnvironmentId, + change: impl FnOnce(&mut EnvironmentRecord) -> Result, + ) -> Result<(EnvironmentRecord, bool), EnvironmentRegistryError> { + let mut state = self.write_state()?; + change_environment(&mut state, environment_id, change) + } +} + +fn change_environment( + state: &mut RegistryState, + environment_id: &EnvironmentId, + change: impl FnOnce(&mut EnvironmentRecord) -> Result, +) -> Result<(EnvironmentRecord, bool), EnvironmentRegistryError> { + let stored = state + .environments + .get_mut(environment_id) + .ok_or_else(|| not_found("environment", environment_id))?; + let mut candidate = stored.clone(); + let changed = change(&mut candidate)?; + if !changed { + return Ok((stored.clone(), false)); + } + candidate.validate()?; + *stored = candidate.clone(); + Ok((candidate, true)) } #[async_trait] @@ -77,10 +158,20 @@ impl EnvironmentProviderStore for InMemoryEnvironmentRegistryStore { &self, request: PutEnvironmentProvider, ) -> Result { - let mut record = request.into_record()?; let mut state = self.write_state()?; - if let Some(existing) = state.providers.get(&record.provider_id) { - record.created_at_ms = existing.created_at_ms; + let credential = request.credential.clone(); + let existing = state.providers.get(&request.provider_id).cloned(); + let record = request.into_record(existing.as_ref())?; + match credential { + ProviderCredentialChange::Keep => {} + ProviderCredentialChange::Set(credential) => { + state + .provider_credentials + .insert(record.provider_id.clone(), credential); + } + ProviderCredentialChange::Clear => { + state.provider_credentials.remove(&record.provider_id); + } } state .providers @@ -99,9 +190,19 @@ impl EnvironmentProviderStore for InMemoryEnvironmentRegistryStore { .ok_or_else(|| not_found("environment_provider", provider_id)) } + async fn read_provider_credential( + &self, + provider_id: &EnvironmentProviderId, + ) -> Result, EnvironmentRegistryError> { + let state = self.read_state()?; + if !state.providers.contains_key(provider_id) { + return Err(not_found("environment_provider", provider_id)); + } + Ok(state.provider_credentials.get(provider_id).cloned()) + } + async fn list_providers( &self, - _request: ListEnvironmentProviders, ) -> Result, EnvironmentRegistryError> { Ok(self.read_state()?.providers.values().cloned().collect()) } @@ -118,10 +219,17 @@ impl EnvironmentProviderStore for InMemoryEnvironmentRegistryStore { { return invalid("environment provider is referenced by a universe binding"); } - state + if state.environments.values().any(|record| { + record.state != EnvironmentState::Closed && record.provider_id() == Some(provider_id) + }) { + return invalid("environment provider is referenced by a non-closed environment"); + } + let record = state .providers .remove(provider_id) - .ok_or_else(|| not_found("environment_provider", provider_id)) + .ok_or_else(|| not_found("environment_provider", provider_id))?; + state.provider_credentials.remove(provider_id); + Ok(record) } } @@ -131,48 +239,12 @@ impl EnvironmentProviderBindingStore for InMemoryEnvironmentRegistryStore { &self, request: PutEnvironmentProviderBinding, ) -> Result { - validate_nonnegative_i64(request.updated_at_ms, "updated_at_ms")?; let mut state = self.write_state()?; if !state.providers.contains_key(&request.provider_id) { return Err(not_found("environment_provider", &request.provider_id)); } - if state.bindings.values().any(|binding| { - binding.universe_id == request.universe_id - && binding.provider_id == request.provider_id - && binding.binding_id != request.binding_id - }) { - return Err(EnvironmentRegistryError::AlreadyExists { - kind: "environment_provider_binding", - id: format!("{}/{}", request.universe_id, request.provider_id), - }); - } - let key = (request.universe_id, request.binding_id.clone()); - let existing = state.bindings.get(&key); - if existing.is_some_and(|record| record.provider_id != request.provider_id) { - return invalid("provider_id is immutable for an existing binding"); - } - let actual = existing.map(|record| record.revision); - if request.expected_revision != actual { - return Err(EnvironmentRegistryError::RevisionConflict { - kind: "environment_provider_binding", - id: request.binding_id.to_string(), - expected: request.expected_revision, - actual, - }); - } - let record = EnvironmentProviderBindingRecord { - universe_id: request.universe_id, - binding_id: request.binding_id, - provider_id: request.provider_id, - status: request.status, - revision: actual.unwrap_or(0) + 1, - metadata: request.metadata, - created_at_ms: existing - .map(|record| record.created_at_ms) - .unwrap_or(request.updated_at_ms), - updated_at_ms: request.updated_at_ms, - }; - record.validate()?; + let key = (request.universe_id, request.provider_id.clone()); + let record = request.into_record(state.bindings.get(&key))?; state.bindings.insert(key, record.clone()); Ok(record) } @@ -180,13 +252,13 @@ impl EnvironmentProviderBindingStore for InMemoryEnvironmentRegistryStore { async fn read_provider_binding( &self, universe_id: Uuid, - binding_id: &EnvironmentProviderBindingId, + provider_id: &EnvironmentProviderId, ) -> Result { self.read_state()? .bindings - .get(&(universe_id, binding_id.clone())) + .get(&(universe_id, provider_id.clone())) .cloned() - .ok_or_else(|| not_found("environment_provider_binding", binding_id)) + .ok_or_else(|| not_found("environment_provider_binding", provider_id)) } async fn list_provider_bindings( @@ -205,19 +277,21 @@ impl EnvironmentProviderBindingStore for InMemoryEnvironmentRegistryStore { async fn delete_provider_binding( &self, universe_id: Uuid, - binding_id: &EnvironmentProviderBindingId, + provider_id: &EnvironmentProviderId, ) -> Result { let mut state = self.write_state()?; - if state.environments.values().any(|environment| { - environment.status != EnvironmentStatus::Closed - && environment.binding_id() == Some(binding_id) - }) { + if universe_id == self.universe_id + && state.environments.values().any(|record| { + record.state != EnvironmentState::Closed + && record.provider_id() == Some(provider_id) + }) + { return invalid("provider binding is referenced by a non-closed environment"); } state .bindings - .remove(&(universe_id, binding_id.clone())) - .ok_or_else(|| not_found("environment_provider_binding", binding_id)) + .remove(&(universe_id, provider_id.clone())) + .ok_or_else(|| not_found("environment_provider_binding", provider_id)) } } @@ -229,136 +303,25 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { ) -> Result { validate_nonnegative_i64(request.created_at_ms, "created_at_ms")?; let mut state = self.write_state()?; - if let Some(environment_id) = state.requests.get(&request.request_id) { - return state - .environments - .get(environment_id) - .cloned() - .ok_or_else(|| not_found("environment", environment_id)); - } - let binding = state - .bindings - .get(&(self.universe_id, request.binding_id.clone())) - .cloned() - .ok_or_else(|| not_found("environment_provider_binding", &request.binding_id))?; - if binding.status != EnvironmentProviderBindingStatus::Enabled { - return invalid("environment provider binding is disabled"); - } - if state.environments.contains_key(&request.environment_id) { - return Err(EnvironmentRegistryError::AlreadyExists { - kind: "environment", - id: request.environment_id.to_string(), - }); - } - let record = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request.request_id.clone(), - source: EnvironmentSource::Provisioned { - provider_id: binding.provider_id, - binding_id: request.binding_id, - }, - display_name: request.display_name, - status: EnvironmentStatus::Provisioning, - desired_power: PowerState::Running, - idle_policy: request.idle_policy, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id, - provision_request_id: Some(request.request_id.clone()), - provider_target_id: None, - template_id: Some(request.template_id), - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata, - last_seen_at_ms: None, - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - record.validate()?; - state - .requests - .insert(request.request_id, request.environment_id.clone()); - state - .environments - .insert(request.environment_id, record.clone()); - Ok(record) - } - - async fn adopt_environment( - &self, - request: AdoptEnvironment, - ) -> Result { - validate_nonnegative_i64(request.created_at_ms, "created_at_ms")?; - if request.source_target.is_empty() - || request.source_target.len() > 255 - || request.source_target.chars().any(char::is_control) - { - return invalid( - "adoption source must be non-empty, at most 255 bytes, and contain no control characters", - ); - } - let mut state = self.write_state()?; - if let Some(environment_id) = state.requests.get(&request.request_id) { - return state - .environments - .get(environment_id) - .cloned() - .ok_or_else(|| not_found("environment", environment_id)); + if let Some(existing) = state.by_request_id(&request.request_id) { + return if request.matches(existing) { + Ok(existing.clone()) + } else { + Err(EnvironmentRegistryError::AlreadyExists { + kind: "environment_request", + id: request.request_id.to_string(), + }) + }; } let binding = state .bindings - .get(&(self.universe_id, request.binding_id.clone())) - .cloned() - .ok_or_else(|| not_found("environment_provider_binding", &request.binding_id))?; - if binding.status != EnvironmentProviderBindingStatus::Enabled { + .get(&(self.universe_id, request.provider_id.clone())) + .ok_or_else(|| not_found("environment_provider_binding", &request.provider_id))?; + if !binding.enabled { return invalid("environment provider binding is disabled"); } - if state.environments.contains_key(&request.environment_id) { - return Err(EnvironmentRegistryError::AlreadyExists { - kind: "environment", - id: request.environment_id.to_string(), - }); - } - let record = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request.request_id.clone(), - source: EnvironmentSource::Provisioned { - provider_id: binding.provider_id, - binding_id: request.binding_id, - }, - display_name: request.display_name, - status: EnvironmentStatus::Provisioning, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id, - provision_request_id: Some(request.request_id.clone()), - provider_target_id: None, - template_id: None, - adoption_source_target: Some(request.source_target), - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata, - last_seen_at_ms: None, - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - record.validate()?; - state - .requests - .insert(request.request_id, request.environment_id.clone()); - state - .environments - .insert(request.environment_id, record.clone()); - Ok(record) + let record = request.record()?; + state.insert_new(record) } async fn create_external_environment( @@ -366,56 +329,19 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { request: CreateExternalEnvironment, ) -> Result { validate_nonnegative_i64(request.created_at_ms, "created_at_ms")?; - request.connection.validate()?; let mut state = self.write_state()?; - if let Some(environment_id) = state.requests.get(&request.request_id) { - return state - .environments - .get(environment_id) - .cloned() - .ok_or_else(|| not_found("environment", environment_id)); - } - if state.environments.contains_key(&request.environment_id) { - return Err(EnvironmentRegistryError::AlreadyExists { - kind: "environment", - id: request.environment_id.to_string(), - }); + if let Some(existing) = state.by_request_id(&request.request_id) { + return if request.matches(existing) { + Ok(existing.clone()) + } else { + Err(EnvironmentRegistryError::AlreadyExists { + kind: "environment_request", + id: request.request_id.to_string(), + }) + }; } - let record = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request.request_id.clone(), - source: EnvironmentSource::External { - connection: request.connection, - }, - display_name: request.display_name, - status: EnvironmentStatus::Ready, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id, - provision_request_id: None, - provider_target_id: None, - template_id: None, - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata, - last_seen_at_ms: None, - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - record.validate()?; - state - .requests - .insert(request.request_id, request.environment_id.clone()); - state - .environments - .insert(request.environment_id, record.clone()); - Ok(record) + let record = request.record()?; + state.insert_new(record) } async fn create_registered_environment( @@ -423,40 +349,34 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { request: CreateRegisteredEnvironment, ) -> Result { validate_nonnegative_i64(request.created_at_ms, "created_at_ms")?; - let daemon_id = request.daemon_id()?; - let request_id = EnvironmentProvisionRequestId::for_daemon(&daemon_id); + validate_daemon_public_key(&request.daemon_public_key)?; + validate_secret_hash(&request.secret_hash)?; let mut state = self.write_state()?; - if let Some(environment_id) = state.requests.get(&request_id) { - return state - .environments - .get(environment_id) - .cloned() - .ok_or_else(|| not_found("environment", environment_id)); - } - if state.environments.values().any(|record| { - matches!(&record.source, EnvironmentSource::Registered { daemon_public_key, .. } - if daemon_public_key == &request.daemon_public_key) - }) { - return Err(EnvironmentRegistryError::AlreadyExists { - kind: "daemon_identity", - id: daemon_id.to_string(), - }); + if let Some(existing) = state.by_daemon_public_key(&request.daemon_public_key) { + return Ok(existing.clone()); } let key = state + .registration_key(&request.registration_key_id)? + .clone(); + key.check_admits(request.created_at_ms)?; + let live = state .registration_keys .get(&request.registration_key_id) - .map(|stored| stored.record.clone()) - .ok_or_else(|| { - not_found("environment_registration_key", &request.registration_key_id) - })?; - key.check_admits(request.created_at_ms)?; + .is_some_and(|stored| { + stored.secrets.iter().any(|secret| { + secret.retired_at_ms.is_none() && secret.secret_hash == request.secret_hash + }) + }); + if !live { + return Err(request.secret_retired()); + } if let Some(limit) = key.max_active_environments { let active = state .environments .values() .filter(|record| { record.registration_key_id() == Some(&key.registration_key_id) - && record.status != EnvironmentStatus::Closed + && record.state != EnvironmentState::Closed }) .count(); if active >= limit as usize { @@ -466,50 +386,19 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { }); } } - if state.environments.contains_key(&request.environment_id) { - return Err(EnvironmentRegistryError::AlreadyExists { - kind: "environment", - id: request.environment_id.to_string(), - }); - } - let record = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request_id.clone(), - source: EnvironmentSource::Registered { - registration_key_id: key.registration_key_id, - daemon_id, - daemon_public_key: request.daemon_public_key, - identity_mode: key.identity_mode, - }, - display_name: request.display_name, - status: EnvironmentStatus::Ready, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id, - provision_request_id: None, - provider_target_id: None, - template_id: None, - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata, - last_seen_at_ms: Some(request.created_at_ms), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - record.validate()?; - state - .requests - .insert(request_id, request.environment_id.clone()); - state + let record = request.record(&key)?; + state.insert_new(record) + } + + async fn read_environment( + &self, + environment_id: &EnvironmentId, + ) -> Result { + self.read_state()? .environments - .insert(request.environment_id, record.clone()); - Ok(record) + .get(environment_id) + .cloned() + .ok_or_else(|| not_found("environment", environment_id)) } async fn read_environment_by_daemon_public_key( @@ -518,135 +407,171 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { ) -> Result, EnvironmentRegistryError> { Ok(self .read_state()? + .by_daemon_public_key(daemon_public_key) + .cloned()) + } + + async fn list_environments( + &self, + request: ListEnvironments, + ) -> Result { + request.validate()?; + let state = self.read_state()?; + let mut matching = state .environments .values() - .find(|record| { - matches!(&record.source, EnvironmentSource::Registered { daemon_public_key: key, .. } - if key == daemon_public_key) + .filter(|record| { + request + .after + .as_ref() + .is_none_or(|after| &record.environment_id > after) }) - .cloned()) + .filter(|record| request.matches(record)); + let environments: Vec = matching + .by_ref() + .take(request.limit as usize) + .cloned() + .collect(); + let next_after = if matching.next().is_some() { + environments + .last() + .map(|record| record.environment_id.clone()) + } else { + None + }; + Ok(EnvironmentPage { + environments, + next_after, + }) } - async fn observe_registered_environment( + async fn update_environment( &self, - request: ObserveRegisteredEnvironment, + request: UpdateEnvironment, ) -> Result { - validate_nonnegative_i64(request.observed_at_ms, "observed_at_ms")?; - let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - apply_registered_observation( - record, - request.observation, - request.observed_at_ms, - request.metadata, - )?; - record.validate()?; - Ok(record.clone()) + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) } - async fn list_open_registered_environments( + async fn begin_close_environment( &self, - ) -> Result, EnvironmentRegistryError> { - Ok(self - .read_state()? - .environments - .values() - .filter(|record| { - record.is_registered() - && !matches!( - record.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) - }) - .cloned() - .collect()) + request: BeginCloseEnvironment, + ) -> Result { + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) } - async fn read_environment( + async fn finish_close_environment( &self, - environment_id: &EnvironmentId, + request: FinishCloseEnvironment, ) -> Result { - self.read_state()? - .environments - .get(environment_id) - .cloned() - .ok_or_else(|| not_found("environment", environment_id)) + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) } - async fn read_environment_by_request_id( + async fn mark_registered_connected( &self, - request_id: &EnvironmentProvisionRequestId, + request: MarkRegisteredConnected, ) -> Result { - let state = self.read_state()?; - let environment_id = state - .requests - .get(request_id) - .ok_or_else(|| not_found("environment_request", request_id))?; - state - .environments - .get(environment_id) - .cloned() - .ok_or_else(|| not_found("environment", environment_id)) + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) } - async fn list_environments( + async fn mark_registered_disconnected( &self, - request: ListEnvironments, - ) -> Result, EnvironmentRegistryError> { - Ok(self - .read_state()? + request: MarkRegisteredDisconnected, + ) -> Result { + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) + } + + async fn stamp_registered_seen( + &self, + request: StampRegisteredSeen, + ) -> Result { + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(_, written)| written) + } + + async fn sweep_registered_environments( + &self, + now_ms: i64, + ) -> Result { + let mut state = self.write_state()?; + let candidates: Vec<(EnvironmentId, EnvironmentRegistrationKeyRecord)> = state .environments .values() - .filter(|record| { - request - .provider_id - .as_ref() - .is_none_or(|id| record.provider_id() == Some(id)) + .filter(|record| record.state == EnvironmentState::Open) + .filter_map(|record| { + let key = state.registration_keys.get(record.registration_key_id()?)?; + Some((record.environment_id.clone(), key.record.clone())) }) - .filter(|record| { - request - .binding_id - .as_ref() - .is_none_or(|id| record.binding_id() == Some(id)) - }) - .filter(|record| request.status.is_none_or(|status| status == record.status)) - .filter(|record| { - request - .registration_key_id - .as_ref() - .is_none_or(|id| record.registration_key_id() == Some(id)) - }) - .filter(|record| { - harness::storage::metadata_matches(&record.metadata, &request.metadata) - }) - .cloned() - .collect()) + .collect(); + let mut sweep = RegisteredSweep::default(); + for (environment_id, key) in candidates { + let mut outcome = (false, None); + change_environment(&mut state, &environment_id, |record| { + outcome = sweep_registered_environment(record, &key, now_ms); + Ok(outcome.0 || outcome.1.is_some()) + })?; + if outcome.0 { + sweep.disconnected.push(environment_id.clone()); + } + if let Some(reason) = outcome.1 { + sweep.closed.push((environment_id, reason)); + } + } + Ok(sweep) + } + + async fn repair_registered_connections( + &self, + now_ms: i64, + ) -> Result { + validate_nonnegative_i64(now_ms, "now_ms")?; + let mut state = self.write_state()?; + let mut repaired = 0; + for record in state.environments.values_mut() { + let Some(source) = record.registered_mut() else { + continue; + }; + if source.connected { + source.connected = false; + source.connection_changed_at_ms = now_ms; + record.touch(now_ms); + repaired += 1; + } + } + Ok(repaired) } - async fn list_environments_needing_reconcile( + async fn list_environment_work( &self, + now_ms: i64, + limit: u32, ) -> Result, EnvironmentRegistryError> { - Ok(self - .read_state()? + let state = self.read_state()?; + let mut due: Vec<(i64, &EnvironmentRecord)> = state .environments .values() - .filter(|record| { - matches!( - record.status, - EnvironmentStatus::Provisioning - | EnvironmentStatus::Booting - | EnvironmentStatus::Closing - | EnvironmentStatus::Unknown - ) || record.power_diverges() + .filter_map(|record| { + let next = record.provisioned()?.next_attempt_at_ms?; + (next <= now_ms).then_some((next, record)) }) - .cloned() + .collect(); + due.sort_by(|(left_at, left), (right_at, right)| { + left_at + .cmp(right_at) + .then_with(|| left.environment_id.cmp(&right.environment_id)) + }); + Ok(due + .into_iter() + .take(limit as usize) + .map(|(_, record)| record.clone()) .collect()) } - async fn list_environments_with_idle_policy( + async fn list_open_provisioned_environments( &self, ) -> Result, EnvironmentRegistryError> { Ok(self @@ -654,197 +579,76 @@ impl EnvironmentStore for InMemoryEnvironmentRegistryStore { .environments .values() .filter(|record| { - record.status == EnvironmentStatus::Ready && record.idle_policy.is_some() + record.state == EnvironmentState::Open && record.provisioned().is_some() }) .cloned() .collect()) } - async fn observe_provisioned_environment( + async fn record_target_observation( &self, - request: ObserveProvisionedEnvironment, + request: RecordTargetObservation, ) -> Result { let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - if request.observed_at_ms < record.incarnation.updated_at_ms { - return Ok(record.clone()); - } - if record - .incarnation - .provider_target_id - .as_ref() - .is_some_and(|existing| existing != &request.provider_target_id) + if let Some(target_id) = &request.provider_target_id + && let Some(provider_id) = state + .environments + .get(&request.environment_id) + .and_then(EnvironmentRecord::provider_id) + && state.environments.values().any(|other| { + other.environment_id != request.environment_id + && other.state != EnvironmentState::Closed + && other.provider_id() == Some(provider_id) + && other + .provisioned() + .and_then(|source| source.provider_target_id.as_ref()) + == Some(target_id) + }) { - return invalid("provider target conflicts with current incarnation"); - } - record.incarnation.provider_target_id = Some(request.provider_target_id); - record.incarnation.power_states = request.power_states; - record.incarnation.updated_at_ms = request.observed_at_ms; - record.updated_at_ms = request.observed_at_ms; - record.status = request.status; - record.validate()?; - Ok(record.clone()) - } - - async fn fail_environment_lifecycle( - &self, - request: FailEnvironmentLifecycle, - ) -> Result { - let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - record.status = EnvironmentStatus::Failed; - record - .metadata - .insert("lifecycleError".to_owned(), request.message); - record.incarnation.updated_at_ms = request.observed_at_ms; - record.updated_at_ms = request.observed_at_ms; - Ok(record.clone()) - } - - async fn begin_close_environment( - &self, - request: BeginCloseEnvironment, - ) -> Result { - let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - if record.status != EnvironmentStatus::Closed { - record.status = EnvironmentStatus::Closing; - record.updated_at_ms = request.updated_at_ms; + return Err(EnvironmentRegistryError::AlreadyExists { + kind: "provider_target", + id: target_id.to_string(), + }); } - Ok(record.clone()) + change_environment(&mut state, &request.environment_id, |record| { + request.apply(record) + }) + .map(|(record, _)| record) } - async fn finish_close_environment( + async fn schedule_environment_attempt( &self, - request: FinishCloseEnvironment, + request: ScheduleEnvironmentAttempt, ) -> Result { - let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - record.status = EnvironmentStatus::Closed; - record.public_ingress_enabled = false; - record.public_endpoint = None; - record.incarnation.updated_at_ms = request.observed_at_ms; - record.updated_at_ms = request.observed_at_ms; - Ok(record.clone()) + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) } - async fn set_environment_ingress( + async fn open_environment( &self, - request: SetEnvironmentIngress, + request: OpenEnvironment, ) -> Result { - validate_nonnegative_i64(request.updated_at_ms, "updated_at_ms")?; - let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - if !matches!(record.source, EnvironmentSource::Provisioned { .. }) { - return invalid("provider-managed ingress requires a provisioned environment"); - } - if matches!( - record.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) && request.enabled - { - return invalid("cannot enable ingress for a closing environment"); - } - if request.enabled != request.public_endpoint.is_some() { - return invalid( - "enabled ingress requires a public endpoint and disabled ingress forbids one", - ); - } - record.public_ingress_enabled = request.enabled; - record.public_endpoint = request.public_endpoint; - record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); - record.validate()?; - Ok(record.clone()) + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) } - async fn set_environment_power( + async fn fail_environment( &self, - request: SetEnvironmentPower, + request: FailEnvironment, ) -> Result { - validate_nonnegative_i64(request.updated_at_ms, "updated_at_ms")?; - let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - check_power_mutable(record)?; - record.desired_power = request.desired_power; - record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); - record.validate()?; - Ok(record.clone()) + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) } - async fn set_environment_idle_policy( + async fn end_provisioned_environment( &self, - request: SetEnvironmentIdlePolicy, + request: EndProvisionedEnvironment, ) -> Result { - validate_nonnegative_i64(request.updated_at_ms, "updated_at_ms")?; - if let Some(policy) = &request.idle_policy { - policy.validate()?; - } - let mut state = self.write_state()?; - let record = state - .environments - .get_mut(&request.environment_id) - .ok_or_else(|| not_found("environment", &request.environment_id))?; - check_power_mutable(record)?; - record.idle_policy = request.idle_policy; - record.updated_at_ms = record.updated_at_ms.max(request.updated_at_ms); - record.validate()?; - Ok(record.clone()) + self.change(&request.environment_id, |record| request.apply(record)) + .map(|(record, _)| record) } } -/// Apply a gateway connection observation to a registered environment. -/// Closing and closed environments keep their state: a late heartbeat must -/// not resurrect them. -pub fn apply_registered_observation( - record: &mut EnvironmentRecord, - observation: RegisteredConnectionObservation, - observed_at_ms: i64, - metadata: std::collections::BTreeMap, -) -> Result<(), EnvironmentRegistryError> { - if !record.is_registered() { - return invalid("connection observations apply only to registered environments"); - } - if matches!( - record.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) { - return Ok(()); - } - record.metadata.extend(metadata); - match observation { - RegisteredConnectionObservation::Connected => { - record.status = EnvironmentStatus::Ready; - record.last_seen_at_ms = Some(observed_at_ms); - } - RegisteredConnectionObservation::Heartbeat => { - record.last_seen_at_ms = Some(observed_at_ms); - } - RegisteredConnectionObservation::Disconnected => { - record.status = EnvironmentStatus::Offline; - } - } - record.updated_at_ms = record.updated_at_ms.max(observed_at_ms); - Ok(()) -} - #[async_trait] impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { async fn create_registration_key( @@ -852,14 +656,12 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { request: CreateEnvironmentRegistrationKey, ) -> Result { request.record.validate()?; + validate_secret_hash(&request.secret_hash)?; let mut state = self.write_state()?; if state .registration_keys .contains_key(&request.record.registration_key_id) - || state - .registration_keys - .values() - .any(|stored| stored.secret_hash == request.secret_hash) + || secret_known(&state, &request.secret_hash) { return Err(EnvironmentRegistryError::AlreadyExists { kind: "environment_registration_key", @@ -869,8 +671,11 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { state.registration_keys.insert( request.record.registration_key_id.clone(), StoredRegistrationKey { - secret_hash: request.secret_hash, record: request.record.clone(), + secrets: vec![StoredSecret { + secret_hash: request.secret_hash, + retired_at_ms: None, + }], }, ); Ok(request.record) @@ -881,10 +686,8 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { registration_key_id: &EnvironmentRegistrationKeyId, ) -> Result { self.read_state()? - .registration_keys - .get(registration_key_id) - .map(|stored| stored.record.clone()) - .ok_or_else(|| not_found("environment_registration_key", registration_key_id)) + .registration_key(registration_key_id) + .cloned() } async fn list_registration_keys( @@ -898,6 +701,51 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { .collect()) } + async fn update_registration_key( + &self, + request: UpdateEnvironmentRegistrationKey, + ) -> Result { + let mut state = self.write_state()?; + let stored = state + .registration_keys + .get_mut(&request.registration_key_id) + .ok_or_else(|| { + not_found("environment_registration_key", &request.registration_key_id) + })?; + request.apply(&mut stored.record)?; + Ok(stored.record.clone()) + } + + async fn rotate_registration_key( + &self, + request: RotateEnvironmentRegistrationKey, + ) -> Result { + let mut state = self.write_state()?; + if secret_known(&state, &request.secret_hash) { + return Err(EnvironmentRegistryError::AlreadyExists { + kind: "environment_registration_key_secret", + id: request.key_prefix.clone(), + }); + } + let stored = state + .registration_keys + .get_mut(&request.registration_key_id) + .ok_or_else(|| { + not_found("environment_registration_key", &request.registration_key_id) + })?; + request.apply(&mut stored.record)?; + for secret in &mut stored.secrets { + if secret.retired_at_ms.is_none() { + secret.retired_at_ms = Some(request.rotated_at_ms); + } + } + stored.secrets.push(StoredSecret { + secret_hash: request.secret_hash, + retired_at_ms: None, + }); + Ok(stored.record.clone()) + } + async fn revoke_registration_key( &self, request: RevokeEnvironmentRegistrationKey, @@ -911,8 +759,9 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { not_found("environment_registration_key", &request.registration_key_id) })?; if stored.record.revoked_at_ms.is_none() { - stored.record.revoked_at_ms = - Some(request.revoked_at_ms.max(stored.record.created_at_ms)); + let revoked_at_ms = request.revoked_at_ms.max(stored.record.created_at_ms); + stored.record.revoked_at_ms = Some(revoked_at_ms); + stored.record.updated_at_ms = stored.record.updated_at_ms.max(revoked_at_ms); } Ok(stored.record.clone()) } @@ -925,7 +774,11 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { .read_state()? .registration_keys .values() - .find(|stored| stored.secret_hash == secret_hash) + .find(|stored| { + stored.secrets.iter().any(|secret| { + secret.retired_at_ms.is_none() && secret.secret_hash == secret_hash + }) + }) .map(|stored| stored.record.clone())) } @@ -934,12 +787,7 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { registration_key_id: &EnvironmentRegistrationKeyId, ) -> Result { let state = self.read_state()?; - if !state.registration_keys.contains_key(registration_key_id) { - return Err(not_found( - "environment_registration_key", - registration_key_id, - )); - } + state.registration_key(registration_key_id)?; let mut usage = RegistrationKeyUsage::default(); for record in state .environments @@ -947,7 +795,7 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { .filter(|record| record.registration_key_id() == Some(registration_key_id)) { usage.registered += 1; - if record.status != EnvironmentStatus::Closed { + if record.state != EnvironmentState::Closed { usage.active += 1; } usage.last_registered_at_ms = Some( @@ -960,21 +808,14 @@ impl EnvironmentRegistrationKeyStore for InMemoryEnvironmentRegistryStore { } } -/// Power intent and idle policy exist only for provisioned environments that -/// are not on their way out. -pub(crate) fn check_power_mutable( - record: &EnvironmentRecord, -) -> Result<(), EnvironmentRegistryError> { - if !matches!(record.source, EnvironmentSource::Provisioned { .. }) { - return invalid("only provisioned environments have power control"); - } - if matches!( - record.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed | EnvironmentStatus::Failed - ) { - return invalid("cannot change power of a closing, closed, or failed environment"); - } - Ok(()) +/// Secret hashes are unique across every key, retired ones included. +fn secret_known(state: &RegistryState, secret_hash: &str) -> bool { + state.registration_keys.values().any(|stored| { + stored + .secrets + .iter() + .any(|secret| secret.secret_hash == secret_hash) + }) } #[async_trait] diff --git a/crates/environments/src/tests.rs b/crates/environments/src/tests.rs index 605df8215..2b89129a5 100644 --- a/crates/environments/src/tests.rs +++ b/crates/environments/src/tests.rs @@ -1,304 +1,590 @@ use std::collections::BTreeMap; -use environment_protocol::shared::{EnvironmentTransport, ProviderTargetId}; +use environment_protocol::{ + ids::ProviderTargetId, + metadata::{DescriptiveText, MAX_METADATA_ENTRIES}, +}; use uuid::Uuid; use super::*; +const PROVIDER: &str = "incus-local"; + +fn endpoint(value: &str) -> EnvironmentEndpoint { + EnvironmentEndpoint::parse(value).expect("endpoint") +} + +fn target(value: &str) -> ProviderTargetId { + ProviderTargetId::new(value).expect("target id") +} + fn provider() -> PutEnvironmentProvider { PutEnvironmentProvider { - provider_id: EnvironmentProviderId::new("incus-local"), + provider_id: EnvironmentProviderId::new(PROVIDER), display_name: Some("Local Incus".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "ws://127.0.0.1:19090/control", - EnvironmentTransport::WebSocket, - ), - metadata: BTreeMap::new(), + controller_endpoint: endpoint("ws://127.0.0.1:19090/control"), + data_route_base: None, + credential: ProviderCredentialChange::Set(SecretValue::new("provider-secret")), updated_at_ms: 1_000, } } -fn binding(universe_id: Uuid, expected_revision: Option) -> PutEnvironmentProviderBinding { +fn binding(universe_id: Uuid, enabled: bool, at: i64) -> PutEnvironmentProviderBinding { PutEnvironmentProviderBinding { universe_id, - binding_id: EnvironmentProviderBindingId::new("primary"), - provider_id: EnvironmentProviderId::new("incus-local"), - status: EnvironmentProviderBindingStatus::Enabled, - metadata: BTreeMap::new(), - expected_revision, - updated_at_ms: 1_000 + expected_revision.unwrap_or(0) as i64, + provider_id: EnvironmentProviderId::new(PROVIDER), + enabled, + updated_at_ms: at, } } -fn create(request: &str, environment: &str, incarnation: &str, at: i64) -> CreateEnvironment { +fn create(request: &str, environment: &str, at: i64) -> CreateEnvironment { CreateEnvironment { request_id: EnvironmentProvisionRequestId::new(request), environment_id: EnvironmentId::new(environment), - incarnation_id: EnvironmentIncarnationId::new(incarnation), - binding_id: EnvironmentProviderBindingId::new("primary"), + provider_id: EnvironmentProviderId::new(PROVIDER), template_id: EnvironmentTemplateId::new("rust-v1"), + idle: None, display_name: None, metadata: BTreeMap::new(), + created_at_ms: at, + } +} - idle_policy: None, +fn external(request: &str, environment: &str, at: i64) -> CreateExternalEnvironment { + CreateExternalEnvironment { + request_id: EnvironmentProvisionRequestId::new(request), + environment_id: EnvironmentId::new(environment), + endpoint: endpoint("ws://envd.example:9000"), + display_name: None, + metadata: BTreeMap::new(), created_at_ms: at, } } +fn build(version: &str) -> DaemonBuild { + DaemonBuild { + version: Some(version.to_owned()), + git_sha: Some("aaaa".to_owned()), + protocol_version: 4, + } +} + +fn observed(phase: TargetPhase, power: Option, at: i64) -> ObservedTarget { + ObservedTarget { + phase, + power, + message: None, + observed_at_ms: at, + } +} + +fn observation( + environment_id: &EnvironmentId, + target_id: Option<&str>, + target: ObservedTarget, +) -> RecordTargetObservation { + RecordTargetObservation { + environment_id: environment_id.clone(), + provider_target_id: target_id.map(self::target), + observed_at_ms: target.observed_at_ms, + observed: target, + public_endpoint: None, + } +} + async fn store() -> (Uuid, InMemoryEnvironmentRegistryStore) { let universe_id = Uuid::new_v4(); let store = InMemoryEnvironmentRegistryStore::for_universe(universe_id); store.put_provider(provider()).await.expect("provider"); store - .put_provider_binding(binding(universe_id, None)) + .put_provider_binding(binding(universe_id, true, 1_000)) .await .expect("binding"); (universe_id, store) } -#[tokio::test(flavor = "current_thread")] -async fn binding_put_is_revisioned_and_unique_per_provider() { - let (universe_id, store) = store().await; - let first = store - .read_provider_binding(universe_id, &EnvironmentProviderBindingId::new("primary")) +async fn open_provisioned( + store: &InMemoryEnvironmentRegistryStore, + environment: &str, +) -> EnvironmentRecord { + let created = store + .create_environment(create(&format!("req-{environment}"), environment, 2_000)) + .await + .expect("create"); + store + .record_target_observation(observation( + &created.environment_id, + Some(&format!("target-{environment}")), + observed(TargetPhase::Active, Some(PowerState::Running), 2_100), + )) + .await + .expect("observe"); + store + .open_environment(OpenEnvironment { + environment_id: created.environment_id, + daemon_build: build("0.5.0"), + at_ms: 2_200, + }) .await - .expect("read"); - assert_eq!(first.revision, 1); + .expect("open") +} + +// --- providers and bindings -------------------------------------------------- - let conflict = store - .put_provider_binding(binding(universe_id, None)) +#[tokio::test(flavor = "current_thread")] +async fn provider_credentials_are_optional_and_follow_the_change() { + let store = InMemoryEnvironmentRegistryStore::new(); + let provider_id = EnvironmentProviderId::new(PROVIDER); + let mut bare = provider(); + bare.credential = ProviderCredentialChange::Keep; + let created = store + .put_provider(bare.clone()) .await - .expect_err("stale write"); + .expect("no credential"); + assert!(!created.has_credential); + assert_eq!( + store + .read_provider_credential(&provider_id) + .await + .expect("read"), + None + ); + + let mut unprintable = provider(); + unprintable.credential = ProviderCredentialChange::Set(SecretValue::new("two words")); assert!(matches!( - conflict, - EnvironmentRegistryError::RevisionConflict { - actual: Some(1), - .. - } + store.put_provider(unprintable).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) )); - let second = store - .put_provider_binding(binding(universe_id, Some(1))) - .await - .expect("replace"); - assert_eq!(second.revision, 2); + let set = store.put_provider(provider()).await.expect("set"); + assert!(set.has_credential); + assert_eq!(set.created_at_ms, 1_000); + bare.updated_at_ms = 2_000; + let kept = store.put_provider(bare).await.expect("keep"); + assert!(kept.has_credential); + assert_eq!( + store + .read_provider_credential(&provider_id) + .await + .expect("read") + .expect("stored") + .expose(), + "provider-secret" + ); - let mut duplicate = binding(universe_id, None); - duplicate.binding_id = EnvironmentProviderBindingId::new("secondary"); + let mut clear = provider(); + clear.credential = ProviderCredentialChange::Clear; + clear.updated_at_ms = 3_000; + let cleared = store.put_provider(clear).await.expect("clear"); + assert!(!cleared.has_credential); + assert_eq!( + store + .read_provider_credential(&provider_id) + .await + .expect("read"), + None + ); assert!(matches!( - store.put_provider_binding(duplicate).await, - Err(EnvironmentRegistryError::AlreadyExists { .. }) + store + .read_provider_credential(&EnvironmentProviderId::new("missing")) + .await, + Err(EnvironmentRegistryError::NotFound { .. }) + )); + let debug = format!("{:?}", provider()); + assert!(!debug.contains("provider-secret")); +} + +#[test] +fn data_route_base_is_configured_or_derived_and_never_downgrades() { + let derived = provider().into_record(None).expect("derived"); + assert_eq!( + derived.effective_data_route_base().expect("base").as_str(), + "ws://127.0.0.1:19090/routes" + ); + + let mut configured = provider(); + configured.data_route_base = Some(DataRouteBase::new("ws://relay.example:7000/data").unwrap()); + let configured = configured.into_record(None).expect("configured"); + assert_eq!( + configured + .effective_data_route_base() + .expect("base") + .as_str(), + "ws://relay.example:7000/data" + ); + + let mut downgrade = provider(); + downgrade.controller_endpoint = endpoint("wss://incus.example.com/control"); + downgrade.data_route_base = Some(DataRouteBase::new("ws://incus.example.com/routes").unwrap()); + assert!(matches!( + downgrade.into_record(None), + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + + let mut unix = provider(); + unix.controller_endpoint = endpoint("unix:///run/incus-provider.sock"); + assert!(matches!( + unix.clone().into_record(None), + Err(EnvironmentRegistryError::InvalidInput { .. }) )); + unix.data_route_base = Some(DataRouteBase::new("ws://127.0.0.1:19091/routes").unwrap()); + unix.into_record(None).expect("unix with a configured base"); } #[tokio::test(flavor = "current_thread")] -async fn provider_delete_requires_all_bindings_to_be_removed() { +async fn bindings_are_one_per_provider_and_guard_deletes() { let (universe_id, store) = store().await; - let provider_id = EnvironmentProviderId::new("incus-local"); + let provider_id = EnvironmentProviderId::new(PROVIDER); + let replaced = store + .put_provider_binding(binding(universe_id, false, 1_500)) + .await + .expect("replace"); + assert!(!replaced.enabled); + assert_eq!(replaced.created_at_ms, 1_000); + assert_eq!( + store + .list_provider_bindings(universe_id) + .await + .unwrap() + .len(), + 1 + ); + assert!(matches!( + store + .put_provider_binding(PutEnvironmentProviderBinding { + provider_id: EnvironmentProviderId::new("missing"), + ..binding(universe_id, true, 1_600) + }) + .await, + Err(EnvironmentRegistryError::NotFound { .. }) + )); + + // A disabled binding refuses creates; a missing one is not found. + assert!(matches!( + store + .create_environment(create("r-1", "env-1", 2_000)) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + store + .put_provider_binding(binding(universe_id, true, 1_700)) + .await + .expect("enable"); + let environment = store + .create_environment(create("r-1", "env-1", 2_000)) + .await + .expect("create"); + + // Live environments guard the binding and the provider. + assert!(matches!( + store + .delete_provider_binding(universe_id, &provider_id) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); assert!(matches!( store.delete_provider(&provider_id).await, Err(EnvironmentRegistryError::InvalidInput { .. }) )); store - .delete_provider_binding(universe_id, &EnvironmentProviderBindingId::new("primary")) + .begin_close_environment(BeginCloseEnvironment { + environment_id: environment.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 3_000, + }) .await - .expect("delete binding"); - let deleted = store - .delete_provider(&provider_id) + .expect("begin close"); + store + .finish_close_environment(FinishCloseEnvironment { + environment_id: environment.environment_id, + at_ms: 3_100, + }) .await - .expect("delete provider"); - assert_eq!(deleted.provider_id, provider_id); + .expect("close"); + store + .delete_provider_binding(universe_id, &provider_id) + .await + .expect("delete binding"); assert!(matches!( - store.read_provider(&provider_id).await, + store + .create_environment(create("r-2", "env-2", 4_000)) + .await, Err(EnvironmentRegistryError::NotFound { .. }) )); + store + .delete_provider(&provider_id) + .await + .expect("delete provider"); } +// --- creates ----------------------------------------------------------------- + #[tokio::test(flavor = "current_thread")] -async fn stable_request_id_returns_the_original_environment() { +async fn request_ids_are_idempotent_only_for_identical_inputs() { let (_, store) = store().await; let first = store - .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) + .create_environment(create("request-1", "environment-1", 2_000)) .await .expect("first"); + assert_eq!(first.state, EnvironmentState::Provisioning); + let source = first.provisioned().expect("provisioned"); + assert_eq!(source.next_attempt_at_ms, Some(2_000)); + assert_eq!(source.attempts, 0); + + // A retry with a fresh environment id and time converges. let retry = store - .create_environment(create( - "request-1", - "different-environment", - "different-incarnation", - 3_000, - )) + .create_environment(create("request-1", "other-environment", 3_000)) .await .expect("retry"); - assert_eq!(retry.environment_id, first.environment_id); - assert_eq!( - retry.incarnation.incarnation_id, - first.incarnation.incarnation_id - ); + assert_eq!(retry, first); + + let mut changed = create("request-1", "other-environment", 3_000); + changed.idle = Some(IdlePolicy { + pause_after_ms: Some(60_000), + ..IdlePolicy::default() + }); + assert!(matches!( + store.create_environment(changed).await, + Err(EnvironmentRegistryError::AlreadyExists { .. }) + )); + // Another kind cannot claim the same request id. + assert!(matches!( + store + .create_external_environment(external("request-1", "ext", 3_000)) + .await, + Err(EnvironmentRegistryError::AlreadyExists { .. }) + )); + assert!(matches!( + store + .create_environment(create("request-2", "environment-1", 3_000)) + .await, + Err(EnvironmentRegistryError::AlreadyExists { .. }) + )); + + let external_row = store + .create_external_environment(external("ext-1", "ext-env", 4_000)) + .await + .expect("external"); + assert_eq!(external_row.state, EnvironmentState::Open); assert_eq!( store - .list_environments(ListEnvironments::default()) + .create_external_environment(external("ext-1", "ext-env-2", 5_000)) .await - .expect("list") - .len(), - 1 + .expect("external retry"), + external_row ); } #[tokio::test(flavor = "current_thread")] -async fn adoption_creates_a_provisioned_environment_without_a_template() { +async fn invalid_idle_overrides_are_refused() { let (_, store) = store().await; - let adopted = store - .adopt_environment(AdoptEnvironment { - request_id: EnvironmentProvisionRequestId::new("adopt-request-1"), - environment_id: EnvironmentId::new("environment-adopted"), - incarnation_id: EnvironmentIncarnationId::new("incarnation-adopted"), - binding_id: EnvironmentProviderBindingId::new("primary"), - source_target: "legacy/hand-built-vm".to_owned(), - display_name: Some("Hand-built VM".to_owned()), - metadata: BTreeMap::new(), - created_at_ms: 2_000, - }) - .await - .expect("adopt"); - - assert_eq!(adopted.status, EnvironmentStatus::Provisioning); + let mut request = create("request-1", "environment-1", 2_000); + request.idle = Some(IdlePolicy { + pause_after_ms: Some(30), + stop_after_ms: Some(10), + ..IdlePolicy::default() + }); assert!(matches!( - adopted.source, - EnvironmentSource::Provisioned { .. } + store.create_environment(request).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) )); - assert_eq!(adopted.incarnation.template_id, None); - assert_eq!( - adopted.incarnation.adoption_source_target.as_deref(), - Some("legacy/hand-built-vm") - ); +} - let retry = store - .adopt_environment(AdoptEnvironment { - request_id: EnvironmentProvisionRequestId::new("adopt-request-1"), - environment_id: EnvironmentId::new("ignored-on-retry"), - incarnation_id: EnvironmentIncarnationId::new("ignored-on-retry"), - binding_id: EnvironmentProviderBindingId::new("primary"), - source_target: "legacy/another-vm".to_owned(), - display_name: None, - metadata: BTreeMap::new(), - created_at_ms: 3_000, +#[tokio::test(flavor = "current_thread")] +async fn external_environment_persists_its_endpoint() { + let store = InMemoryEnvironmentRegistryStore::new(); + let environment = store + .create_external_environment(CreateExternalEnvironment { + endpoint: endpoint("unix:///run/lightspeed/envd.sock"), + ..external("external-request", "external-environment", 1_000) }) .await - .expect("idempotent retry"); - assert_eq!(retry.environment_id, adopted.environment_id); + .expect("create external environment"); + let EnvironmentSource::External { endpoint } = &environment.source else { + panic!("external source") + }; + assert_eq!(endpoint.as_str(), "unix:///run/lightspeed/envd.sock"); + assert_eq!( + environment.display_status(), + EnvironmentDisplayStatus::Ready + ); + assert_eq!(environment.use_gate(1_000), EnvironmentUse::Use); } +// --- provisioned lifecycle --------------------------------------------------- + #[tokio::test(flavor = "current_thread")] -async fn lightspeed_does_not_enforce_provider_quota() { +async fn provisioning_opens_after_the_handshake_and_records_the_build() { let (_, store) = store().await; - let first = store - .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) + let created = store + .create_environment(create("request-1", "environment-1", 2_000)) .await - .expect("first"); - store - .fail_environment_lifecycle(FailEnvironmentLifecycle { - environment_id: first.environment_id.clone(), - message: "provider failed".to_owned(), - observed_at_ms: 3_000, + .expect("create"); + let scheduled = store + .schedule_environment_attempt(ScheduleEnvironmentAttempt { + environment_id: created.environment_id.clone(), + expected_state: EnvironmentState::Provisioning, + next_attempt_at_ms: 4_000, + attempts: 1, + state_message: Some(format!("connect failed\u{7}{}", "x".repeat(2_000))), + at_ms: 2_000, }) .await - .expect("fail"); - store - .create_environment(create("request-2", "environment-2", "incarnation-2", 4_000)) + .expect("schedule"); + let message = scheduled.state_message.as_deref().expect("message"); + assert!(message.len() <= STATE_MESSAGE_MAX_BYTES); + assert!(!message.contains('\u{7}')); + assert_eq!( + scheduled.provisioned().unwrap().next_attempt_at_ms, + Some(4_000) + ); + + let opened = open_provisioned(&store, "environment-2").await; + assert_eq!(opened.state, EnvironmentState::Open); + assert_eq!(opened.state_message, None); + assert_eq!(opened.state_changed_at_ms, 2_200); + assert_eq!(opened.daemon_build, Some(build("0.5.0"))); + let source = opened.provisioned().unwrap(); + assert_eq!(source.next_attempt_at_ms, None); + assert_eq!( + source.provider_target_id, + Some(target("target-environment-2")) + ); + + // Opening twice, or scheduling an open row, is skipped. + let again = store + .open_environment(OpenEnvironment { + environment_id: opened.environment_id.clone(), + daemon_build: build("9.9.9"), + at_ms: 3_000, + }) .await - .expect("second intent"); + .expect("skipped"); + assert_eq!(again, opened); + let unscheduled = store + .schedule_environment_attempt(ScheduleEnvironmentAttempt { + environment_id: opened.environment_id.clone(), + expected_state: EnvironmentState::Provisioning, + next_attempt_at_ms: 5_000, + attempts: 2, + state_message: None, + at_ms: 3_000, + }) + .await + .expect("skipped"); + assert_eq!(unscheduled, opened); } #[tokio::test(flavor = "current_thread")] -async fn provider_observation_populates_only_the_current_incarnation() { +async fn work_lists_due_rows_earliest_first() { let (_, store) = store().await; - let environment = store - .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) - .await - .expect("create"); - let target_id = ProviderTargetId::new("target-1"); - let observed = store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: target_id.clone(), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 3_000, + for (request, environment, at) in [("r-a", "env-a", 3_000), ("r-b", "env-b", 1_500)] { + store + .create_environment(create(request, environment, at)) + .await + .expect("create"); + } + store + .schedule_environment_attempt(ScheduleEnvironmentAttempt { + environment_id: EnvironmentId::new("env-a"), + expected_state: EnvironmentState::Provisioning, + next_attempt_at_ms: 2_500, + attempts: 1, + state_message: None, + at_ms: 3_000, }) .await - .expect("observe"); - assert_eq!(observed.status, EnvironmentStatus::Ready); - assert_eq!(observed.incarnation.provider_target_id, Some(target_id)); + .expect("schedule"); + let open = open_provisioned(&store, "env-open").await; + + let due = store.list_environment_work(2_600, 10).await.expect("work"); + let ids: Vec<&str> = due + .iter() + .map(|record| record.environment_id.as_str()) + .collect(); + assert_eq!(ids, ["env-b", "env-a"]); assert_eq!( - observed.incarnation.power_states, - vec![PowerState::Running, PowerState::Paused] + store.list_environment_work(2_600, 1).await.unwrap().len(), + 1 ); - assert_eq!( - observed.incarnation.template_id, - Some(EnvironmentTemplateId::new("rust-v1")) + assert!( + store + .list_environment_work(1_000, 10) + .await + .unwrap() + .is_empty() ); + + let open_rows = store.list_open_provisioned_environments().await.unwrap(); + assert_eq!(open_rows.len(), 1); + assert_eq!(open_rows[0].environment_id, open.environment_id); } #[tokio::test(flavor = "current_thread")] -async fn public_ingress_is_a_realized_environment_facet() { +async fn failing_applies_to_provisioning_and_open_rows_only() { let (_, store) = store().await; - let environment = store - .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) + let created = store + .create_environment(create("request-1", "environment-1", 2_000)) .await .expect("create"); - let enabled = store - .set_environment_ingress(SetEnvironmentIngress { - environment_id: environment.environment_id.clone(), - enabled: true, - public_endpoint: Some("https://opaque.env.example".to_owned()), - updated_at_ms: 3_000, + let failed = store + .fail_environment(FailEnvironment { + environment_id: created.environment_id.clone(), + message: "protocol mismatch: local 4, remote 3".to_owned(), + daemon_build: Some(DaemonBuild { + version: None, + git_sha: None, + protocol_version: 3, + }), + at_ms: 3_000, }) .await - .expect("enable ingress"); - assert!(enabled.public_ingress_enabled); + .expect("fail"); + assert_eq!(failed.state, EnvironmentState::Failed); + assert_eq!(failed.provisioned().unwrap().next_attempt_at_ms, None); + assert_eq!(failed.daemon_build.as_ref().unwrap().protocol_version, 3); assert_eq!( - enabled.public_endpoint.as_deref(), - Some("https://opaque.env.example") + failed.use_gate(3_000), + EnvironmentUse::Gone { + reason: EnvironmentGoneReason::Failed, + message: Some("protocol mismatch: local 4, remote 3".to_owned()), + } ); - let disabled = store - .set_environment_ingress(SetEnvironmentIngress { - environment_id: environment.environment_id, - enabled: false, - public_endpoint: None, - updated_at_ms: 4_000, + // A failed row cannot fail again; it can close. + let again = store + .fail_environment(FailEnvironment { + environment_id: created.environment_id.clone(), + message: "other".to_owned(), + daemon_build: None, + at_ms: 3_500, }) .await - .expect("disable ingress"); - assert!(!disabled.public_ingress_enabled); - assert_eq!(disabled.public_endpoint, None); -} - -#[tokio::test(flavor = "current_thread")] -async fn external_environment_cannot_enable_provider_managed_ingress() { - let store = InMemoryEnvironmentRegistryStore::new(); - let environment = store - .create_external_environment(CreateExternalEnvironment { - request_id: EnvironmentProvisionRequestId::new("external-request"), - environment_id: EnvironmentId::new("external-environment"), - incarnation_id: EnvironmentIncarnationId::new("external-incarnation"), - connection: EnvironmentConnectionSpec::new( - "ws://envd.example", - EnvironmentTransport::WebSocket, - ), - display_name: None, - metadata: BTreeMap::new(), - created_at_ms: 1_000, + .expect("skipped"); + assert_eq!(again, failed); + let closing = store + .begin_close_environment(BeginCloseEnvironment { + environment_id: created.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 4_000, }) .await + .expect("close failed row"); + assert_eq!(closing.state, EnvironmentState::Closing); + assert_eq!(closing.state_message, None); + + let external_row = store + .create_external_environment(external("ext", "ext-env", 2_000)) + .await .expect("external"); assert!(matches!( store - .set_environment_ingress(SetEnvironmentIngress { - environment_id: environment.environment_id, - enabled: true, - public_endpoint: Some("https://invalid.example".to_owned()), - updated_at_ms: 2_000, + .fail_environment(FailEnvironment { + environment_id: external_row.environment_id, + message: "x".to_owned(), + daemon_build: None, + at_ms: 3_000, }) .await, Err(EnvironmentRegistryError::InvalidInput { .. }) @@ -306,300 +592,632 @@ async fn external_environment_cannot_enable_provider_managed_ingress() { } #[tokio::test(flavor = "current_thread")] -async fn external_environment_persists_a_typed_connection() { - let store = InMemoryEnvironmentRegistryStore::new(); - let environment = store - .create_external_environment(CreateExternalEnvironment { - request_id: EnvironmentProvisionRequestId::new("external-request"), - environment_id: EnvironmentId::new("external-environment"), - incarnation_id: EnvironmentIncarnationId::new("external-incarnation"), - connection: EnvironmentConnectionSpec::new( - "ws://envd.example:19091", - EnvironmentTransport::WebSocket, - ), - display_name: None, - metadata: BTreeMap::new(), - created_at_ms: 1_000, - }) - .await - .expect("create external environment"); - let EnvironmentSource::External { connection } = environment.source else { - panic!("external source") - }; - assert_eq!(connection.endpoint, "ws://envd.example:19091"); -} - -#[tokio::test(flavor = "current_thread")] -async fn disabled_binding_blocks_create_and_live_references_block_delete() { - let (universe_id, store) = store().await; - let environment = store - .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) +async fn a_close_wins_over_work_that_was_in_flight() { + let (_, store) = store().await; + let created = store + .create_environment(create("request-1", "environment-1", 2_000)) .await .expect("create"); - let mut disabled = binding(universe_id, Some(1)); - disabled.status = EnvironmentProviderBindingStatus::Disabled; - store.put_provider_binding(disabled).await.expect("disable"); - assert!(matches!( + let closing = store + .begin_close_environment(BeginCloseEnvironment { + environment_id: created.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 2_500, + }) + .await + .expect("begin close"); + assert_eq!(closing.state, EnvironmentState::Closing); + assert_eq!( + closing.close_reason, + Some(EnvironmentCloseReason::Requested) + ); + assert_eq!( + closing.provisioned().unwrap().next_attempt_at_ms, + Some(2_500) + ); + + // The create that was in flight answers: the target id is recorded so + // the close can address it, but the row stays closing. + let observed = store + .record_target_observation(observation( + &created.environment_id, + Some("ls-3f2a"), + observed(TargetPhase::Active, Some(PowerState::Running), 3_000), + )) + .await + .expect("observe"); + assert_eq!(observed.state, EnvironmentState::Closing); + assert_eq!( + observed.provisioned().unwrap().provider_target_id, + Some(target("ls-3f2a")) + ); + for skipped in [ store - .create_environment(create("request-2", "environment-2", "incarnation-2", 3_000)) - .await, - Err(EnvironmentRegistryError::InvalidInput { .. }) - )); - assert!(matches!( + .open_environment(OpenEnvironment { + environment_id: created.environment_id.clone(), + daemon_build: build("0.5.0"), + at_ms: 3_100, + }) + .await + .expect("open skipped"), store - .delete_provider_binding(universe_id, &EnvironmentProviderBindingId::new("primary")) - .await, - Err(EnvironmentRegistryError::InvalidInput { .. }) - )); - store - .finish_close_environment(FinishCloseEnvironment { - environment_id: environment.environment_id, - observed_at_ms: 4_000, + .fail_environment(FailEnvironment { + environment_id: created.environment_id.clone(), + message: "late".to_owned(), + daemon_build: None, + at_ms: 3_200, + }) + .await + .expect("fail skipped"), + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: created.environment_id.clone(), + reason: EnvironmentCloseReason::KeyRevoked, + at_ms: 3_300, + }) + .await + .expect("close skipped"), + ] { + assert_eq!(skipped.state, EnvironmentState::Closing); + assert_eq!( + skipped.close_reason, + Some(EnvironmentCloseReason::Requested) + ); + } + + // The provision step that read the row while provisioning reschedules + // it: skipped, so the close keeps its own schedule and message. + let rescheduled = store + .schedule_environment_attempt(ScheduleEnvironmentAttempt { + environment_id: created.environment_id.clone(), + expected_state: EnvironmentState::Provisioning, + next_attempt_at_ms: 300_000, + attempts: 3, + state_message: Some("waiting for the provider".to_owned()), + at_ms: 3_400, }) .await - .expect("close"); - store - .delete_provider_binding(universe_id, &EnvironmentProviderBindingId::new("primary")) + .expect("schedule skipped"); + assert_eq!(rescheduled.state, EnvironmentState::Closing); + assert_eq!(rescheduled.state_message, None); + let source = rescheduled.provisioned().unwrap(); + assert_eq!(source.next_attempt_at_ms, Some(2_500)); + assert_eq!(source.attempts, 0); + // The close step's own reschedule still applies. + let close_retry = store + .schedule_environment_attempt(ScheduleEnvironmentAttempt { + environment_id: created.environment_id.clone(), + expected_state: EnvironmentState::Closing, + next_attempt_at_ms: 3_600, + attempts: 1, + state_message: None, + at_ms: 3_500, + }) .await - .expect("delete"); -} + .expect("schedule close retry"); + assert_eq!( + close_retry.provisioned().unwrap().next_attempt_at_ms, + Some(3_600) + ); -#[tokio::test(flavor = "current_thread")] -async fn environment_metadata_filters_require_every_pair() { - let (_universe, store) = store().await; - // Metadata filters by containment: every listed pair must match. - let mut tagged = create("tagged", "env-tagged", "inc-tagged", 3); - tagged.metadata = BTreeMap::from([ - ("source".to_owned(), "harbor".to_owned()), - ("job".to_owned(), "nightly".to_owned()), - ]); - store - .create_environment(tagged) - .await - .expect("create tagged"); - let by_metadata = store - .list_environments(ListEnvironments { - metadata: BTreeMap::from([("job".to_owned(), "nightly".to_owned())]), - ..ListEnvironments::default() + let closed = store + .finish_close_environment(FinishCloseEnvironment { + environment_id: created.environment_id.clone(), + at_ms: 4_000, }) .await - .expect("list by metadata"); - assert_eq!(by_metadata.len(), 1); - assert_eq!(by_metadata[0].environment_id.as_str(), "env-tagged"); - let mismatched = store - .list_environments(ListEnvironments { - metadata: BTreeMap::from([ - ("job".to_owned(), "nightly".to_owned()), - ("trial".to_owned(), "1".to_owned()), - ]), - ..ListEnvironments::default() + .expect("finish"); + assert_eq!(closed.state, EnvironmentState::Closed); + assert_eq!(closed.closed_at_ms, Some(4_000)); + assert_eq!(closed.close_reason, Some(EnvironmentCloseReason::Requested)); + assert_eq!(closed.provisioned().unwrap().next_attempt_at_ms, None); + let finished_again = store + .finish_close_environment(FinishCloseEnvironment { + environment_id: created.environment_id, + at_ms: 5_000, }) .await - .expect("list by mismatched metadata"); - assert!(mismatched.is_empty()); -} - -#[test] -fn idle_policy_validates_monotone_stages_and_picks_the_supported_due_action() { - let policy = EnvironmentIdlePolicy { - pause_after_ms: Some(10), - suspend_after_ms: Some(20), - stop_after_ms: Some(30), - close_after_ms: Some(40), - }; - policy.validate().expect("valid"); - assert!(EnvironmentIdlePolicy::default().validate().is_err()); - assert!( - EnvironmentIdlePolicy { - pause_after_ms: Some(0), - ..EnvironmentIdlePolicy::default() - } - .validate() - .is_err() - ); - assert!( - EnvironmentIdlePolicy { - pause_after_ms: Some(30), - stop_after_ms: Some(10), - ..EnvironmentIdlePolicy::default() - } - .validate() - .is_err() - ); - let incus = [PowerState::Running, PowerState::Paused, PowerState::Stopped]; - assert_eq!(policy.due_action(5, &incus), None); - assert_eq!(policy.due_action(10, &incus), Some(IdleAction::Pause)); - // Suspend is due but unsupported by this provider: pause stays the - // most escalated applicable stage. - assert_eq!(policy.due_action(25, &incus), Some(IdleAction::Pause)); - assert_eq!(policy.due_action(30, &incus), Some(IdleAction::Stop)); - assert_eq!(policy.due_action(40, &incus), Some(IdleAction::Close)); - // Close never needs provider support. - assert_eq!(policy.due_action(40, &[]), Some(IdleAction::Close)); - assert_eq!(policy.due_action(30, &[]), None); + .expect("skipped"); + assert_eq!(finished_again, closed); } #[tokio::test(flavor = "current_thread")] -async fn power_intent_and_idle_policy_are_provisioned_only_and_drive_reconcile_lists() { +async fn observations_keep_the_target_id_and_never_go_back_in_time() { let (_, store) = store().await; - let environment = store - .create_environment(create("request-1", "environment-1", "incarnation-1", 2_000)) + let created = store + .create_environment(create("request-1", "environment-1", 2_000)) .await .expect("create"); - assert_eq!(environment.desired_power, PowerState::Running); - assert!(environment.idle_policy.is_none()); + let pending = store + .record_target_observation(observation( + &created.environment_id, + None, + observed(TargetPhase::Provisioning, None, 2_500), + )) + .await + .expect("no id yet"); + assert_eq!(pending.provisioned().unwrap().provider_target_id, None); + // Natural provider ids are opaque: slashes and colons are fine. + let natural = "arn:aws:ec2:eu-west-1:123456789012:instance/i-0abc"; store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 3_000, - }) + .record_target_observation(observation( + &created.environment_id, + Some(natural), + observed(TargetPhase::Active, Some(PowerState::Running), 3_000), + )) .await - .expect("observe"); - // Ready and converged: nothing to reconcile. - assert!( - store - .list_environments_needing_reconcile() - .await - .unwrap() - .is_empty() - ); - let paused = store - .set_environment_power(SetEnvironmentPower { - environment_id: environment.environment_id.clone(), - desired_power: PowerState::Paused, - updated_at_ms: 4_000, + .expect("id assigned"); + let stale = store + .record_target_observation(RecordTargetObservation { + public_endpoint: Some("https://stale.example".to_owned()), + ..observation( + &created.environment_id, + None, + observed(TargetPhase::Provisioning, None, 2_800), + ) }) .await - .expect("set power"); - assert_eq!(paused.desired_power, PowerState::Paused); - assert!(paused.power_diverges()); - let pending = store.list_environments_needing_reconcile().await.unwrap(); - assert_eq!(pending.len(), 1); + .expect("stale"); + let source = stale.provisioned().unwrap(); + assert_eq!(source.provider_target_id, Some(target(natural))); + assert_eq!(source.observed.as_ref().unwrap().phase, TargetPhase::Active); + assert_eq!(source.public_endpoint, None); + assert_eq!(stale.state, EnvironmentState::Provisioning); - // Observed paused: converged again. - store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Paused, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 5_000, - }) + assert!(matches!( + store + .record_target_observation(observation( + &created.environment_id, + Some("another-target"), + observed(TargetPhase::Active, None, 4_000), + )) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + // Power without the active phase is not a valid observation. + assert!(matches!( + store + .record_target_observation(observation( + &created.environment_id, + None, + observed(TargetPhase::Unknown, Some(PowerState::Paused), 4_000), + )) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + // Another live row of the provider cannot claim the same target. + let other = store + .create_environment(create("request-2", "environment-2", 2_000)) .await - .expect("observe paused"); - assert!( + .expect("other"); + assert!(matches!( store - .list_environments_needing_reconcile() - .await - .unwrap() - .is_empty() - ); + .record_target_observation(observation( + &other.environment_id, + Some(natural), + observed(TargetPhase::Active, None, 4_000), + )) + .await, + Err(EnvironmentRegistryError::AlreadyExists { .. }) + )); +} - // Idle policy only lists ready environments. - let policy = EnvironmentIdlePolicy { - pause_after_ms: Some(60_000), - ..EnvironmentIdlePolicy::default() - }; - store - .set_environment_idle_policy(SetEnvironmentIdlePolicy { - environment_id: environment.environment_id.clone(), - idle_policy: Some(policy.clone()), - updated_at_ms: 6_000, +#[tokio::test(flavor = "current_thread")] +async fn the_provider_ending_a_target_closes_open_and_failed_rows() { + let (_, store) = store().await; + let open = open_provisioned(&store, "env-open").await; + let ended = store + .end_provisioned_environment(EndProvisionedEnvironment { + environment_id: open.environment_id.clone(), + at_ms: 5_000, }) .await - .expect("policy"); - assert!( - store - .list_environments_with_idle_policy() - .await - .unwrap() - .is_empty() + .expect("end"); + assert_eq!(ended.state, EnvironmentState::Closed); + assert_eq!( + ended.close_reason, + Some(EnvironmentCloseReason::ProviderEnded) ); - store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 7_000, + assert_eq!(ended.closed_at_ms, Some(5_000)); + + let provisioning = store + .create_environment(create("r-p", "env-p", 2_000)) + .await + .expect("create"); + let skipped = store + .end_provisioned_environment(EndProvisionedEnvironment { + environment_id: provisioning.environment_id.clone(), + at_ms: 5_000, }) .await - .expect("observe ready"); - let candidates = store.list_environments_with_idle_policy().await.unwrap(); - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].idle_policy, Some(policy)); + .expect("skipped"); + assert_eq!(skipped, provisioning); +} - // External environments have no power control. - let external = store - .create_external_environment(CreateExternalEnvironment { - request_id: EnvironmentProvisionRequestId::new("external-1"), - environment_id: EnvironmentId::new("external-env"), - incarnation_id: EnvironmentIncarnationId::new("external-inc"), - connection: EnvironmentConnectionSpec::new( - "ws://envd.test:9000", - environment_protocol::shared::EnvironmentTransport::WebSocket, - ), - display_name: None, - metadata: BTreeMap::new(), - created_at_ms: 8_000, +#[tokio::test(flavor = "current_thread")] +async fn a_closed_environment_keeps_no_public_endpoint() { + let (_, store) = store().await; + let with_endpoint = + |environment_id: &EnvironmentId, target: ObservedTarget| RecordTargetObservation { + public_endpoint: Some("https://env.example.test".to_owned()), + ..observation(environment_id, None, target) + }; + + // The provider ending the target clears the endpoint. + let open = open_provisioned(&store, "env-ended").await; + let exposed = store + .record_target_observation(with_endpoint( + &open.environment_id, + observed(TargetPhase::Active, Some(PowerState::Running), 4_000), + )) + .await + .expect("observe"); + assert_eq!( + exposed.provisioned().unwrap().public_endpoint.as_deref(), + Some("https://env.example.test") + ); + let ended = store + .end_provisioned_environment(EndProvisionedEnvironment { + environment_id: open.environment_id.clone(), + at_ms: 5_000, }) .await - .expect("external"); - assert!(matches!( - store - .set_environment_power(SetEnvironmentPower { - environment_id: external.environment_id.clone(), - desired_power: PowerState::Paused, - updated_at_ms: 9_000, - }) - .await, - Err(EnvironmentRegistryError::InvalidInput { .. }) - )); + .expect("end"); + assert_eq!(ended.provisioned().unwrap().public_endpoint, None); + // A late observation does not bring it back. + let late = store + .record_target_observation(with_endpoint( + &open.environment_id, + observed(TargetPhase::Closed, None, 6_000), + )) + .await + .expect("late observation"); + assert_eq!(late.state, EnvironmentState::Closed); + assert_eq!(late.provisioned().unwrap().public_endpoint, None); - // Closing environments cannot change power. + // So does finishing a requested close; closing keeps it until then. + let created = store + .create_environment(create("r-close", "env-close", 2_000)) + .await + .expect("create"); store + .record_target_observation(RecordTargetObservation { + provider_target_id: Some(target("ls-close")), + ..with_endpoint( + &created.environment_id, + observed(TargetPhase::Active, Some(PowerState::Running), 3_000), + ) + }) + .await + .expect("observe"); + let closing = store .begin_close_environment(BeginCloseEnvironment { - environment_id: environment.environment_id.clone(), - updated_at_ms: 10_000, + environment_id: created.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 3_500, }) .await - .expect("close"); + .expect("begin close"); + assert!(closing.provisioned().unwrap().public_endpoint.is_some()); + let closed = store + .finish_close_environment(FinishCloseEnvironment { + environment_id: created.environment_id, + at_ms: 4_000, + }) + .await + .expect("finish"); + assert_eq!(closed.provisioned().unwrap().public_endpoint, None); +} + +// --- display status and the use gate ---------------------------------------- + +#[test] +fn display_status_refines_open_rows_by_observation() { + let mut record = create("r", "e", 1_000).record().expect("record"); + assert_eq!( + record.display_status(), + EnvironmentDisplayStatus::Provisioning + ); + record.state = EnvironmentState::Open; + if let EnvironmentSource::Provisioned(source) = &mut record.source { + source.next_attempt_at_ms = None; + } + assert_eq!(record.display_status(), EnvironmentDisplayStatus::Ready); + for (phase, power, status) in [ + (TargetPhase::Active, None, EnvironmentDisplayStatus::Ready), + ( + TargetPhase::Active, + Some(PowerState::Running), + EnvironmentDisplayStatus::Ready, + ), + ( + TargetPhase::Active, + Some(PowerState::Paused), + EnvironmentDisplayStatus::Paused, + ), + ( + TargetPhase::Active, + Some(PowerState::Suspended), + EnvironmentDisplayStatus::Suspended, + ), + ( + TargetPhase::Active, + Some(PowerState::Stopped), + EnvironmentDisplayStatus::Stopped, + ), + ( + TargetPhase::Provisioning, + None, + EnvironmentDisplayStatus::Provisioning, + ), + ( + TargetPhase::Closing, + None, + EnvironmentDisplayStatus::Closing, + ), + ( + TargetPhase::Unknown, + None, + EnvironmentDisplayStatus::Unknown, + ), + (TargetPhase::Failed, None, EnvironmentDisplayStatus::Unknown), + (TargetPhase::Closed, None, EnvironmentDisplayStatus::Unknown), + ] { + if let EnvironmentSource::Provisioned(source) = &mut record.source { + source.observed = Some(observed(phase, power, 1_500)); + } + record.validate().expect("valid"); + assert_eq!(record.display_status(), status, "{phase:?} {power:?}"); + // An observation never blocks use: the dial decides. + assert_eq!(record.use_gate(2_000), EnvironmentUse::Use); + } + for (state, status) in [ + (EnvironmentState::Closing, EnvironmentDisplayStatus::Closing), + (EnvironmentState::Closed, EnvironmentDisplayStatus::Closed), + (EnvironmentState::Failed, EnvironmentDisplayStatus::Failed), + ] { + record.state = state; + assert_eq!(record.display_status(), status); + } +} + +#[test] +fn the_use_gate_covers_every_state() { + let provisioning = create("r", "e", 1_000).record().expect("record"); + assert_eq!( + provisioning.use_gate(1_000), + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Provisioning, + retry_after_ms: WAIT_RETRY_AFTER_MS, + until_ms: None, + } + ); + for (state, reason, message) in [ + ( + EnvironmentState::Closing, + EnvironmentGoneReason::Closing, + None, + ), + ( + EnvironmentState::Closed, + EnvironmentGoneReason::Closed, + None, + ), + ( + EnvironmentState::Failed, + EnvironmentGoneReason::Failed, + Some("broken".to_owned()), + ), + ] { + let record = EnvironmentRecord { + state, + state_message: message.clone(), + ..provisioning.clone() + }; + assert_eq!( + record.use_gate(1_000), + EnvironmentUse::Gone { reason, message } + ); + } +} + +fn registered_record(mode: RegisteredIdentityMode, connected: bool) -> EnvironmentRecord { + let key = mint_registration_key( + EnvironmentRegistrationKeyId::new("rk"), + registration_policy(mode), + 1_000, + ) + .expect("mint") + .record; + let mut record = register("rk", "env", &daemon_key(0x51), 1_000) + .record(&key) + .expect("record"); + if let EnvironmentSource::Registered(source) = &mut record.source { + source.connected = connected; + source.connection_changed_at_ms = 10_000; + } + record +} + +#[test] +fn a_disconnected_registered_daemon_is_waited_for_by_identity_mode() { + let connected = registered_record(RegisteredIdentityMode::Persistent, true); + assert_eq!(connected.use_gate(10_000), EnvironmentUse::Use); + assert_eq!(connected.display_status(), EnvironmentDisplayStatus::Ready); + + let persistent = registered_record(RegisteredIdentityMode::Persistent, false); + assert_eq!(persistent.display_status(), EnvironmentDisplayStatus::Away); + let until = 10_000 + REGISTERED_RECONNECT_WAIT_MS; + assert_eq!( + persistent.use_gate(until - 1), + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Disconnected, + retry_after_ms: WAIT_RETRY_AFTER_MS, + until_ms: Some(until), + } + ); + assert_eq!( + persistent.use_gate(until + 12_500), + EnvironmentUse::Gone { + reason: EnvironmentGoneReason::Away, + message: Some("daemon away for 2m12s".to_owned()), + } + ); + + let ephemeral = registered_record(RegisteredIdentityMode::Ephemeral, false); assert!(matches!( - store - .set_environment_power(SetEnvironmentPower { - environment_id: environment.environment_id.clone(), - desired_power: PowerState::Running, - updated_at_ms: 11_000, - }) - .await, - Err(EnvironmentRegistryError::InvalidInput { .. }) + ephemeral.use_gate(10_001), + EnvironmentUse::Gone { + reason: EnvironmentGoneReason::Away, + .. + } )); } +#[test] +fn away_spans_read_as_compact_durations() { + for (ms, expected) in [ + (-5, "0s"), + (999, "0s"), + (45_000, "45s"), + (120_000, "2m"), + (192_000, "3m12s"), + (3_600_000, "1h"), + (3_900_000, "1h5m"), + ] { + assert_eq!(compact_duration(ms), expected); + } +} + +#[test] +fn route_refusals_map_to_wait_or_gone() { + let failed = DataRouteRefusal::Failed { + message: DescriptiveText::new("disk full").unwrap(), + }; + for (refusal, expected) in [ + ( + DataRouteRefusal::Waking {}, + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Waking, + retry_after_ms: 5_000, + until_ms: None, + }, + ), + ( + DataRouteRefusal::Booting {}, + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Booting, + retry_after_ms: 5_000, + until_ms: None, + }, + ), + ( + DataRouteRefusal::Provisioning {}, + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Provisioning, + retry_after_ms: 5_000, + until_ms: None, + }, + ), + ( + DataRouteRefusal::Disconnected {}, + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Disconnected, + retry_after_ms: 5_000, + until_ms: None, + }, + ), + ( + DataRouteRefusal::Away {}, + EnvironmentUse::Gone { + reason: EnvironmentGoneReason::Away, + message: None, + }, + ), + ( + DataRouteRefusal::Gone {}, + EnvironmentUse::Gone { + reason: EnvironmentGoneReason::Closed, + message: None, + }, + ), + ( + failed, + EnvironmentUse::Gone { + reason: EnvironmentGoneReason::Failed, + message: Some("disk full".to_owned()), + }, + ), + ] { + assert_eq!( + EnvironmentUse::from_route_refusal(&refusal, Some(5_000)), + expected + ); + } + assert_eq!( + EnvironmentUse::from_route_refusal(&DataRouteRefusal::Waking {}, None), + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Waking, + retry_after_ms: WAIT_RETRY_AFTER_MS, + until_ms: None, + } + ); + + // A registered row that reads connected but has no control handle. + let ephemeral = registered_record(RegisteredIdentityMode::Ephemeral, true); + assert_eq!( + ephemeral.bound_route_refusal(&DataRouteRefusal::Disconnected {}, None), + EnvironmentUse::Gone { + reason: EnvironmentGoneReason::Away, + message: Some("daemon is not connected".to_owned()), + } + ); + let persistent = registered_record(RegisteredIdentityMode::Persistent, true); + assert_eq!( + persistent.bound_route_refusal(&DataRouteRefusal::Disconnected {}, Some(1_000)), + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Disconnected, + retry_after_ms: 1_000, + until_ms: None, + } + ); + assert_eq!( + ephemeral.bound_route_refusal(&DataRouteRefusal::Booting {}, None), + EnvironmentUse::from_route_refusal(&DataRouteRefusal::Booting {}, None) + ); +} + +// --- registration keys ------------------------------------------------------- + fn registration_policy(mode: RegisteredIdentityMode) -> RegistrationKeyPolicy { RegistrationKeyPolicy { - display_name: "harbor campaign".to_owned(), + display_name: "fleet".to_owned(), identity_mode: mode, max_active_environments: None, ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, expires_at_ms: None, } } +/// The secret hash [`minted_key`] stores for key `id`, so [`register`] can +/// present it. +fn key_secret_hash(id: &str) -> String { + registration_key_hash(&format!("{REGISTRATION_KEY_SECRET_PREFIX}test-{id}")) +} + async fn minted_key( store: &InMemoryEnvironmentRegistryStore, id: &str, policy: RegistrationKeyPolicy, ) -> MintedRegistrationKey { - let minted = + let mut minted = mint_registration_key(EnvironmentRegistrationKeyId::new(id), policy, 1_000).expect("mint"); + minted.secret.secret_hash = key_secret_hash(id); store .create_registration_key(CreateEnvironmentRegistrationKey { - secret_hash: minted.secret_hash.clone(), + secret_hash: minted.secret.secret_hash.clone(), record: minted.record.clone(), }) .await @@ -619,30 +1237,32 @@ fn register( ) -> CreateRegisteredEnvironment { CreateRegisteredEnvironment { registration_key_id: EnvironmentRegistrationKeyId::new(key), + secret_hash: key_secret_hash(key), environment_id: EnvironmentId::new(environment), - incarnation_id: EnvironmentIncarnationId::new(format!("{environment}-inc")), daemon_public_key: public_key.to_owned(), display_name: Some("worker".to_owned()), metadata: BTreeMap::new(), + daemon_build: build("0.1.0"), created_at_ms: at, } } #[test] -fn daemon_id_and_request_id_derive_from_the_public_key() { +fn daemon_ids_derive_from_the_public_key() { let public_key = [7u8; 32]; let daemon_id = EnvironmentDaemonId::from_public_key(&public_key); assert!(daemon_id.as_str().starts_with(DAEMON_ID_PREFIX)); assert_eq!(daemon_id.as_str().len(), DAEMON_ID_PREFIX.len() + 64); assert_eq!(daemon_id, EnvironmentDaemonId::from_public_key(&public_key)); assert_ne!(daemon_id, EnvironmentDaemonId::from_public_key(&[8u8; 32])); - let request_id = EnvironmentProvisionRequestId::for_daemon(&daemon_id); - assert_eq!( - request_id.as_str(), - format!("{DAEMON_PROVISION_REQUEST_PREFIX}{}", daemon_id.as_str()) - ); assert!(validate_daemon_public_key(&daemon_key(0xab)).is_ok()); assert!(validate_daemon_public_key("ABCD").is_err()); + + let record = registered_record(RegisteredIdentityMode::Persistent, true); + assert_eq!( + record.registered().unwrap().daemon_id(), + EnvironmentDaemonId::from_public_key(&[0x51u8; 32]) + ); } #[test] @@ -653,14 +1273,16 @@ fn minted_registration_keys_return_the_secret_once_and_redact_debug() { 5, ) .expect("mint"); - let secret = minted.secret.expose().to_owned(); + let secret = minted.secret.secret.expose().to_owned(); assert!(secret.starts_with(REGISTRATION_KEY_SECRET_PREFIX)); - assert_eq!(minted.secret_hash, registration_key_hash(&secret)); + assert_eq!(minted.secret.secret_hash, registration_key_hash(&secret)); + assert_eq!(minted.record.key_prefix, minted.secret.key_prefix); assert_eq!( minted.record.key_prefix.len(), REGISTRATION_KEY_DISPLAY_PREFIX_LEN ); assert!(secret.starts_with(&minted.record.key_prefix)); + assert_eq!(minted.record.updated_at_ms, 5); let debug = format!("{minted:?}"); assert!(!debug.contains(&secret[REGISTRATION_KEY_DISPLAY_PREFIX_LEN..])); assert!( @@ -676,24 +1298,220 @@ fn minted_registration_keys_return_the_secret_once_and_redact_debug() { ); } +#[test] +fn offline_close_applies_only_to_persistent_keys() { + let policy = |mode, offline| RegistrationKeyPolicy { + offline_close_after_ms: offline, + ..registration_policy(mode) + }; + let id = || EnvironmentRegistrationKeyId::new("rk"); + mint_registration_key( + id(), + policy(RegisteredIdentityMode::Persistent, Some(60_000)), + 1, + ) + .expect("persistent"); + for refused in [ + policy(RegisteredIdentityMode::Ephemeral, Some(60_000)), + policy(RegisteredIdentityMode::Persistent, Some(0)), + ] { + assert!(matches!( + mint_registration_key(id(), refused, 1), + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn key_settings_replace_the_editable_policy() { + let (_, store) = store().await; + minted_key( + &store, + "rk", + RegistrationKeyPolicy { + max_active_environments: Some(3), + ..registration_policy(RegisteredIdentityMode::Persistent) + }, + ) + .await; + let settings = RegistrationKeySettings { + display_name: "renamed fleet".to_owned(), + max_active_environments: None, + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: Some(3_600_000), + expires_at_ms: Some(99_000), + }; + let updated = store + .update_registration_key(UpdateEnvironmentRegistrationKey { + registration_key_id: EnvironmentRegistrationKeyId::new("rk"), + settings: settings.clone(), + updated_at_ms: 2_000, + }) + .await + .expect("update"); + assert_eq!(updated.display_name, "renamed fleet"); + assert_eq!(updated.max_active_environments, None); + assert_eq!(updated.offline_close_after_ms, Some(3_600_000)); + assert_eq!(updated.identity_mode, RegisteredIdentityMode::Persistent); + assert_eq!(updated.updated_at_ms, 2_000); + + let refused = store + .update_registration_key(UpdateEnvironmentRegistrationKey { + registration_key_id: EnvironmentRegistrationKeyId::new("rk"), + settings: RegistrationKeySettings { + display_name: String::new(), + ..settings + }, + updated_at_ms: 3_000, + }) + .await; + assert!(matches!( + refused, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + assert_eq!( + store + .read_registration_key(&EnvironmentRegistrationKeyId::new("rk")) + .await + .unwrap(), + updated + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn rotation_retires_the_live_secret_and_keeps_the_group() { + let (_, store) = store().await; + let minted = minted_key( + &store, + "rk", + registration_policy(RegisteredIdentityMode::Persistent), + ) + .await; + let admitted = store + .create_registered_environment(register("rk", "env-1", &daemon_key(0x61), 1_500)) + .await + .expect("admit"); + + let fresh = mint_registration_secret(); + let rotated = store + .rotate_registration_key(RotateEnvironmentRegistrationKey { + registration_key_id: EnvironmentRegistrationKeyId::new("rk"), + secret_hash: fresh.secret_hash.clone(), + key_prefix: fresh.key_prefix.clone(), + rotated_at_ms: 2_000, + }) + .await + .expect("rotate"); + assert_eq!(rotated.key_prefix, fresh.key_prefix); + assert_eq!(rotated.updated_at_ms, 2_000); + assert!( + store + .resolve_registration_key(&minted.secret.secret_hash) + .await + .unwrap() + .is_none() + ); + assert_eq!( + store + .resolve_registration_key(&fresh.secret_hash) + .await + .unwrap() + .map(|key| key.registration_key_id), + Some(EnvironmentRegistrationKeyId::new("rk")) + ); + // The admitted daemon still belongs to the key's group. + let usage = store + .registration_key_usage(&EnvironmentRegistrationKeyId::new("rk")) + .await + .unwrap(); + assert_eq!(usage.active, 1); + assert_eq!( + store + .read_environment(&admitted.environment_id) + .await + .unwrap() + .registration_key_id(), + Some(&EnvironmentRegistrationKeyId::new("rk")) + ); + // An admission that resolved the key before the rotation committed + // presents the retired secret and is refused; the live one admits. + assert!(matches!( + store + .create_registered_environment(register("rk", "env-2", &daemon_key(0x62), 2_100)) + .await, + Err(EnvironmentRegistryError::RegistrationKeyUnavailable { + reason: "rotated", + .. + }) + )); + let admitted_fresh = store + .create_registered_environment(CreateRegisteredEnvironment { + secret_hash: fresh.secret_hash.clone(), + ..register("rk", "env-2", &daemon_key(0x62), 2_100) + }) + .await + .expect("admit with the live secret"); + assert_eq!(admitted_fresh.state, EnvironmentState::Open); + + // A retired secret cannot come back. + assert!(matches!( + store + .rotate_registration_key(RotateEnvironmentRegistrationKey { + registration_key_id: EnvironmentRegistrationKeyId::new("rk"), + secret_hash: minted.secret.secret_hash.clone(), + key_prefix: minted.secret.key_prefix.clone(), + rotated_at_ms: 3_000, + }) + .await, + Err(EnvironmentRegistryError::AlreadyExists { .. }) + )); + + store + .revoke_registration_key(RevokeEnvironmentRegistrationKey { + registration_key_id: EnvironmentRegistrationKeyId::new("rk"), + revoked_at_ms: 4_000, + }) + .await + .expect("revoke"); + let another = mint_registration_secret(); + assert!(matches!( + store + .rotate_registration_key(RotateEnvironmentRegistrationKey { + registration_key_id: EnvironmentRegistrationKeyId::new("rk"), + secret_hash: another.secret_hash, + key_prefix: another.key_prefix, + rotated_at_ms: 5_000, + }) + .await, + Err(EnvironmentRegistryError::RegistrationKeyUnavailable { + reason: "revoked", + .. + }) + )); +} + +// --- registered environments ------------------------------------------------- + #[tokio::test(flavor = "current_thread")] async fn registered_admission_is_keyed_by_daemon_identity() { - let (_universe_id, store) = store().await; + let (_, store) = store().await; let minted = minted_key( &store, "rk-eph", registration_policy(RegisteredIdentityMode::Ephemeral), ) .await; - let resolved = store - .resolve_registration_key(&minted.secret_hash) - .await - .expect("resolve") - .expect("known"); - assert_eq!(resolved.registration_key_id.as_str(), "rk-eph"); + assert_eq!( + store + .resolve_registration_key(&minted.secret.secret_hash) + .await + .expect("resolve") + .map(|key| key.registration_key_id), + Some(EnvironmentRegistrationKeyId::new("rk-eph")) + ); assert!( store - .resolve_registration_key("0000") + .resolve_registration_key(&"0".repeat(64)) .await .expect("resolve") .is_none() @@ -701,136 +1519,263 @@ async fn registered_admission_is_keyed_by_daemon_identity() { let public_key = daemon_key(0x11); let created = store - .create_registered_environment(register("rk-eph", "env-a", &public_key, 2_000)) + .create_registered_environment(register("rk-eph", "env-a", &public_key, 2_000)) + .await + .expect("register"); + assert_eq!(created.state, EnvironmentState::Open); + assert_eq!(created.request_id, None); + assert_eq!(created.daemon_build, Some(build("0.1.0"))); + let source = created.registered().expect("registered"); + assert!(source.connected); + assert_eq!(source.identity_mode, RegisteredIdentityMode::Ephemeral); + assert_eq!(source.connection_changed_at_ms, 2_000); + assert_eq!(source.last_seen_at_ms, 2_000); + + // A retried first registration converges on the same environment. + let retried = store + .create_registered_environment(register("rk-eph", "env-b", &public_key, 2_500)) + .await + .expect("retry"); + assert_eq!(retried, created); + assert_eq!( + store + .read_environment_by_daemon_public_key(&public_key) + .await + .expect("read") + .map(|record| record.environment_id), + Some(created.environment_id.clone()) + ); + + let closed = store + .begin_close_environment(BeginCloseEnvironment { + environment_id: created.environment_id.clone(), + reason: EnvironmentCloseReason::Departed, + at_ms: 3_000, + }) + .await + .expect("close"); + assert_eq!(closed.state, EnvironmentState::Closed); + assert_eq!(closed.close_reason, Some(EnvironmentCloseReason::Departed)); + assert_eq!(closed.closed_at_ms, Some(3_000)); + + // The identity is spent: a reconnect is refused, a fresh admission + // returns the closed row. + assert!(matches!( + store + .mark_registered_connected(MarkRegisteredConnected { + environment_id: created.environment_id.clone(), + daemon_build: build("0.2.0"), + at_ms: 4_000, + }) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + let spent = store + .create_registered_environment(register("rk-eph", "env-c", &public_key, 5_000)) + .await + .expect("spent"); + assert_eq!(spent.environment_id, created.environment_id); + assert_eq!(spent.state, EnvironmentState::Closed); +} + +#[tokio::test(flavor = "current_thread")] +async fn connection_writes_happen_on_transitions_and_coarse_stamps() { + let (_, store) = store().await; + minted_key( + &store, + "rk", + registration_policy(RegisteredIdentityMode::Persistent), + ) + .await; + let created = store + .create_registered_environment(register("rk", "env", &daemon_key(0x71), 10_000)) .await .expect("register"); - assert_eq!(created.status, EnvironmentStatus::Ready); - assert_eq!(created.last_seen_at_ms, Some(2_000)); - assert_eq!( - created.source.identity_mode(), - Some(RegisteredIdentityMode::Ephemeral) - ); - assert_eq!( - created.request_id, - EnvironmentProvisionRequestId::for_daemon(&EnvironmentDaemonId::from_public_key( - &[0x11u8; 32] - )) + let id = created.environment_id.clone(); + let stamp = |at_ms| StampRegisteredSeen { + environment_id: id.clone(), + at_ms, + }; + assert!( + !store + .stamp_registered_seen(stamp(10_000 + REGISTERED_LIVENESS_STAMP_MS - 1)) + .await + .unwrap() ); - - // A retried first registration converges on the same environment. - let retried = store - .create_registered_environment(register("rk-eph", "env-b", &public_key, 2_500)) - .await - .expect("retry"); - assert_eq!(retried.environment_id, created.environment_id); - assert_eq!( + assert!( store - .read_environment_by_daemon_public_key(&public_key) + .stamp_registered_seen(stamp(10_000 + REGISTERED_LIVENESS_STAMP_MS)) .await - .expect("read") - .map(|record| record.environment_id), - Some(created.environment_id.clone()) + .unwrap() ); - let offline = store - .observe_registered_environment(ObserveRegisteredEnvironment { - environment_id: created.environment_id.clone(), - observation: RegisteredConnectionObservation::Disconnected, - observed_at_ms: 3_000, - metadata: BTreeMap::new(), + let away = store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: id.clone(), + at_ms: 400_000, }) .await .expect("disconnect"); - assert_eq!(offline.status, EnvironmentStatus::Offline); - assert_eq!(offline.last_seen_at_ms, Some(2_000)); - assert!(offline.registered_daemon_absent(3_000, 60_000)); + let source = away.registered().unwrap(); + assert!(!source.connected); + assert_eq!(source.connection_changed_at_ms, 400_000); + assert_eq!( + source.last_seen_at_ms, + 10_000 + REGISTERED_LIVENESS_STAMP_MS + ); + // A superseded connection's late disconnect writes nothing, and a + // disconnected row takes no stamps. + let again = store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: id.clone(), + at_ms: 500_000, + }) + .await + .expect("skipped"); + assert_eq!(again, away); + assert!( + !store + .stamp_registered_seen(stamp(10_000_000)) + .await + .unwrap() + ); - // A reconnecting daemon may be a newer build: the admission re-records - // its build facts on the row. let back = store - .observe_registered_environment(ObserveRegisteredEnvironment { - environment_id: created.environment_id.clone(), - observation: RegisteredConnectionObservation::Connected, - observed_at_ms: 4_000, - metadata: RegisteredDaemonBuild { - version: Some("0.2.0".to_owned()), - git_sha: Some("bbbb".to_owned()), - protocol_version: 3, - } - .metadata(), + .mark_registered_connected(MarkRegisteredConnected { + environment_id: id.clone(), + daemon_build: build("0.2.0"), + at_ms: 600_000, }) .await .expect("reconnect"); - assert_eq!(back.status, EnvironmentStatus::Ready); - assert_eq!(back.last_seen_at_ms, Some(4_000)); - assert_eq!( - back.metadata - .get(ENVD_VERSION_METADATA_KEY) - .map(String::as_str), - Some("0.2.0") - ); + let source = back.registered().unwrap(); + assert!(source.connected); + assert_eq!(source.connection_changed_at_ms, 600_000); + assert_eq!(source.last_seen_at_ms, 600_000); + assert_eq!(back.daemon_build, Some(build("0.2.0"))); + assert_eq!( - back.metadata - .get(ENVD_GIT_SHA_METADATA_KEY) - .map(String::as_str), - Some("bbbb") + store.repair_registered_connections(700_000).await.unwrap(), + 1 ); + let repaired = store.read_environment(&id).await.unwrap(); + let source = repaired.registered().unwrap(); + assert!(!source.connected); + assert_eq!(source.connection_changed_at_ms, 700_000); + assert_eq!(source.last_seen_at_ms, 600_000); assert_eq!( - back.metadata - .get(ENVD_PROTOCOL_VERSION_METADATA_KEY) - .map(String::as_str), - Some("3") + store.repair_registered_connections(800_000).await.unwrap(), + 0 ); - assert!(!back.registered_daemon_absent(4_500, 60_000)); - assert!(back.registered_daemon_absent(100_000, 60_000)); +} +#[tokio::test(flavor = "current_thread")] +async fn the_registered_sweep_disconnects_stale_rows_and_closes_by_key_policy() { + let (_, store) = store().await; + minted_key( + &store, + "rk-eph", + RegistrationKeyPolicy { + ephemeral_disconnect_grace_ms: Some(60_000), + ..registration_policy(RegisteredIdentityMode::Ephemeral) + }, + ) + .await; + minted_key( + &store, + "rk-off", + RegistrationKeyPolicy { + offline_close_after_ms: Some(600_000), + ..registration_policy(RegisteredIdentityMode::Persistent) + }, + ) + .await; + minted_key( + &store, + "rk-keep", + registration_policy(RegisteredIdentityMode::Persistent), + ) + .await; + let start = 1_000_000; + for (key, environment, seed) in [ + ("rk-eph", "eph-stale", 0x81), + ("rk-eph", "eph-away", 0x82), + ("rk-off", "off-away", 0x83), + ("rk-keep", "keep-away", 0x84), + ("rk-keep", "keep-live", 0x85), + ] { + store + .create_registered_environment(register(key, environment, &daemon_key(seed), start)) + .await + .expect("register"); + } + for environment in ["eph-away", "off-away", "keep-away"] { + store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: EnvironmentId::new(environment), + at_ms: start, + }) + .await + .expect("disconnect"); + } + // keep-live keeps stamping; eph-stale's gateway died silently. + let now = start + REGISTERED_STALE_AFTER_MS + 1; store - .begin_close_environment(BeginCloseEnvironment { - environment_id: created.environment_id.clone(), - updated_at_ms: 5_000, - }) - .await - .expect("begin close"); - let closed = store - .finish_close_environment(FinishCloseEnvironment { - environment_id: created.environment_id.clone(), - observed_at_ms: 5_100, + .stamp_registered_seen(StampRegisteredSeen { + environment_id: EnvironmentId::new("keep-live"), + at_ms: now - 1_000, }) .await - .expect("close"); - assert_eq!(closed.status, EnvironmentStatus::Closed); - // A late heartbeat cannot resurrect a closed environment. - let still_closed = store - .observe_registered_environment(ObserveRegisteredEnvironment { - environment_id: created.environment_id.clone(), - observation: RegisteredConnectionObservation::Connected, - observed_at_ms: 5_200, - metadata: BTreeMap::from([(ENVD_GIT_SHA_METADATA_KEY.to_owned(), "cccc".to_owned())]), - }) + .expect("stamp"); + + let sweep = store + .sweep_registered_environments(now) .await - .expect("late heartbeat"); - assert_eq!(still_closed.status, EnvironmentStatus::Closed); + .expect("sweep"); + assert_eq!(sweep.disconnected, [EnvironmentId::new("eph-stale")]); assert_eq!( - still_closed - .metadata - .get(ENVD_GIT_SHA_METADATA_KEY) - .map(String::as_str), - Some("bbbb") + sweep.closed, + [ + ( + EnvironmentId::new("eph-away"), + EnvironmentCloseReason::EphemeralGrace + ), + ( + EnvironmentId::new("eph-stale"), + EnvironmentCloseReason::EphemeralGrace + ), + ( + EnvironmentId::new("off-away"), + EnvironmentCloseReason::OfflineTimeout + ), + ] ); - - // The identity is spent: the same key with a fresh environment id is - // refused, and a different registration key cannot move it either. - let spent = store - .create_registered_environment(register("rk-eph", "env-c", &public_key, 6_000)) + let stale = store + .read_environment(&EnvironmentId::new("eph-stale")) .await - .expect("dedup"); - assert_eq!(spent.environment_id, created.environment_id); - assert_eq!(spent.status, EnvironmentStatus::Closed); + .unwrap(); + assert_eq!(stale.state, EnvironmentState::Closed); + assert_eq!(stale.registered().unwrap().connection_changed_at_ms, start); + for open in ["keep-away", "keep-live"] { + assert_eq!( + store + .read_environment(&EnvironmentId::new(open)) + .await + .unwrap() + .state, + EnvironmentState::Open + ); + } + let quiet = store + .sweep_registered_environments(now) + .await + .expect("sweep"); + assert_eq!(quiet, RegisteredSweep::default()); } #[tokio::test(flavor = "current_thread")] async fn registration_key_policy_gates_admission_without_touching_reconnects() { - let (_universe_id, store) = store().await; + let (_, store) = store().await; minted_key( &store, "rk-cap", @@ -862,14 +1807,8 @@ async fn registration_key_policy_gates_admission_without_touching_reconnects() { store .begin_close_environment(BeginCloseEnvironment { environment_id: first.environment_id.clone(), - updated_at_ms: 3_000, - }) - .await - .expect("begin close"); - store - .finish_close_environment(FinishCloseEnvironment { - environment_id: first.environment_id.clone(), - observed_at_ms: 3_100, + reason: EnvironmentCloseReason::Requested, + at_ms: 3_000, }) .await .expect("close"); @@ -906,28 +1845,32 @@ async fn registration_key_policy_gates_admission_without_touching_reconnects() { .await .expect("revoke again"); assert_eq!(again.revoked_at_ms, Some(4_000)); - let refused = store - .create_registered_environment(register("rk-cap", "env-3", &daemon_key(0x23), 5_000)) - .await - .expect_err("revoked"); assert!(matches!( - refused, - EnvironmentRegistryError::RegistrationKeyUnavailable { + store + .create_registered_environment(register("rk-cap", "env-3", &daemon_key(0x23), 5_000)) + .await, + Err(EnvironmentRegistryError::RegistrationKeyUnavailable { reason: "revoked", .. - } + }) )); // Known daemons never consult the key. + store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: EnvironmentId::new("env-2"), + at_ms: 5_400, + }) + .await + .expect("disconnect"); let reconnect = store - .observe_registered_environment(ObserveRegisteredEnvironment { + .mark_registered_connected(MarkRegisteredConnected { environment_id: EnvironmentId::new("env-2"), - observation: RegisteredConnectionObservation::Connected, - observed_at_ms: 5_500, - metadata: BTreeMap::new(), + daemon_build: build("0.1.1"), + at_ms: 5_500, }) .await .expect("reconnect after revoke"); - assert_eq!(reconnect.status, EnvironmentStatus::Ready); + assert!(reconnect.registered().unwrap().connected); minted_key( &store, @@ -938,36 +1881,222 @@ async fn registration_key_policy_gates_admission_without_touching_reconnects() { }, ) .await; - let expired = store - .create_registered_environment(register("rk-exp", "env-4", &daemon_key(0x24), 2_000)) - .await - .expect_err("expired"); assert!(matches!( - expired, - EnvironmentRegistryError::RegistrationKeyUnavailable { + store + .create_registered_environment(register("rk-exp", "env-4", &daemon_key(0x24), 2_000)) + .await, + Err(EnvironmentRegistryError::RegistrationKeyUnavailable { reason: "expired", .. - } + }) )); - let keys = store.list_registration_keys().await.expect("list"); - assert_eq!(keys.len(), 2); + assert_eq!(store.list_registration_keys().await.unwrap().len(), 2); } #[tokio::test(flavor = "current_thread")] -async fn registered_environments_list_by_key() { - let (_universe_id, store) = store().await; - minted_key( - &store, - "rk-a", - registration_policy(RegisteredIdentityMode::Persistent), - ) - .await; +async fn metadata_is_caller_only_and_bounded() { + let (_, store) = store().await; minted_key( &store, - "rk-b", + "rk", registration_policy(RegisteredIdentityMode::Persistent), ) .await; + let mut full = register("rk", "env-full", &daemon_key(0x31), 1_000); + full.metadata = (0..MAX_METADATA_ENTRIES as usize) + .map(|index| (format!("caller.key{index}"), "v".to_owned())) + .collect(); + store + .create_registered_environment(full) + .await + .expect("full caller allowance"); + + let mut over = register("rk", "env-over", &daemon_key(0x32), 1_000); + over.metadata = (0..=MAX_METADATA_ENTRIES as usize) + .map(|index| (format!("caller.key{index}"), "v".to_owned())) + .collect(); + assert!(matches!( + store.create_registered_environment(over).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + let mut reserved = register("rk", "env-reserved", &daemon_key(0x33), 1_000); + reserved.metadata = BTreeMap::from([("lightspeed.envd.version".to_owned(), "1".to_owned())]); + assert!(matches!( + store.create_registered_environment(reserved).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); +} + +// --- updates and listing ----------------------------------------------------- + +#[tokio::test(flavor = "current_thread")] +async fn updates_replace_name_and_metadata_until_closed() { + let store = InMemoryEnvironmentRegistryStore::new(); + let created = store + .create_external_environment(external("ext", "ext-env", 1_000)) + .await + .expect("external"); + let update = |name: Option<&str>, at| UpdateEnvironment { + environment_id: created.environment_id.clone(), + display_name: name.map(str::to_owned), + metadata: BTreeMap::from([("team".to_owned(), "infra".to_owned())]), + updated_at_ms: at, + }; + let updated = store + .update_environment(update(Some("builder"), 2_000)) + .await + .expect("update"); + assert_eq!(updated.display_name.as_deref(), Some("builder")); + assert_eq!( + updated.metadata.get("team").map(String::as_str), + Some("infra") + ); + assert_eq!(updated.updated_at_ms, 2_000); + + // A refused change leaves the stored record untouched. + assert!(matches!( + store.update_environment(update(Some(""), 2_500)).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + assert_eq!( + store + .read_environment(&created.environment_id) + .await + .unwrap(), + updated + ); + + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: created.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 3_000, + }) + .await + .expect("close"); + assert!(matches!( + store.update_environment(update(None, 4_000)).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); +} + +#[tokio::test(flavor = "current_thread")] +async fn listing_pages_by_id_and_hides_closed_rows_by_default() { + let (_, store) = store().await; + for index in 0..5 { + let mut request = create(&format!("r-{index}"), &format!("env-{index}"), 2_000); + if index % 2 == 0 { + request.metadata = BTreeMap::from([("job".to_owned(), "nightly".to_owned())]); + } + store.create_environment(request).await.expect("create"); + } + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: EnvironmentId::new("env-1"), + reason: EnvironmentCloseReason::Requested, + at_ms: 3_000, + }) + .await + .expect("close"); + store + .finish_close_environment(FinishCloseEnvironment { + environment_id: EnvironmentId::new("env-1"), + at_ms: 3_100, + }) + .await + .expect("closed"); + + let ids = |page: &EnvironmentPage| -> Vec { + page.environments + .iter() + .map(|record| record.environment_id.to_string()) + .collect() + }; + let first = store + .list_environments(ListEnvironments { + limit: 2, + ..ListEnvironments::default() + }) + .await + .expect("page 1"); + assert_eq!(ids(&first), ["env-0", "env-2"]); + assert_eq!(first.next_after, Some(EnvironmentId::new("env-2"))); + let second = store + .list_environments(ListEnvironments { + limit: 2, + after: first.next_after.clone(), + ..ListEnvironments::default() + }) + .await + .expect("page 2"); + assert_eq!(ids(&second), ["env-3", "env-4"]); + assert_eq!(second.next_after, None); + + let with_closed = store + .list_environments(ListEnvironments { + include_closed: true, + ..ListEnvironments::default() + }) + .await + .expect("all"); + assert_eq!(with_closed.environments.len(), 5); + let only_closed = store + .list_environments(ListEnvironments { + state: Some(EnvironmentState::Closed), + ..ListEnvironments::default() + }) + .await + .expect("closed"); + assert_eq!(ids(&only_closed), ["env-1"]); + + let tagged = store + .list_environments(ListEnvironments { + metadata: BTreeMap::from([("job".to_owned(), "nightly".to_owned())]), + ..ListEnvironments::default() + }) + .await + .expect("metadata"); + assert_eq!(ids(&tagged), ["env-0", "env-2", "env-4"]); + let mismatched = store + .list_environments(ListEnvironments { + metadata: BTreeMap::from([ + ("job".to_owned(), "nightly".to_owned()), + ("trial".to_owned(), "1".to_owned()), + ]), + ..ListEnvironments::default() + }) + .await + .expect("mismatched"); + assert!(mismatched.environments.is_empty()); + + for limit in [0, LIST_ENVIRONMENTS_MAX_LIMIT + 1] { + assert!(matches!( + store + .list_environments(ListEnvironments { + limit, + ..ListEnvironments::default() + }) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + } + assert_eq!( + ListEnvironments::default().limit, + LIST_ENVIRONMENTS_DEFAULT_LIMIT + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn registered_environments_list_by_key() { + let (_, store) = store().await; + for key in ["rk-a", "rk-b"] { + minted_key( + &store, + key, + registration_policy(RegisteredIdentityMode::Persistent), + ) + .await; + } let a = store .create_registered_environment(register("rk-a", "env-a", &daemon_key(0x31), 2_000)) .await @@ -976,60 +2105,45 @@ async fn registered_environments_list_by_key() { .create_registered_environment(register("rk-b", "env-b", &daemon_key(0x32), 2_000)) .await .expect("b"); - let by_key = store .list_environments(ListEnvironments { - metadata: Default::default(), registration_key_id: Some(EnvironmentRegistrationKeyId::new("rk-a")), ..ListEnvironments::default() }) .await .expect("list"); - assert_eq!(by_key.len(), 1); - assert_eq!(by_key[0].environment_id, a.environment_id); + assert_eq!(by_key.environments.len(), 1); + assert_eq!(by_key.environments[0].environment_id, a.environment_id); } -/// The gateway stamps reserved entries on top of whatever a daemon sent, so -/// a daemon using its whole allowance must still be admitted; the allowance -/// itself stays enforced on the caller's keys. -#[tokio::test(flavor = "current_thread")] -async fn reserved_metadata_sits_on_top_of_the_caller_allowance() { - use environment_protocol::registration::MAX_METADATA_ENTRIES; - - let (_universe_id, store) = store().await; - minted_key( - &store, - "rk-meta", - registration_policy(RegisteredIdentityMode::Persistent), - ) - .await; - let reserved = RegisteredDaemonBuild { - version: Some("0.1.0".to_owned()), - git_sha: Some("a".repeat(40)), - protocol_version: 2, +#[test] +fn states_and_close_reasons_round_trip_their_stored_names() { + for state in [ + EnvironmentState::Provisioning, + EnvironmentState::Open, + EnvironmentState::Closing, + EnvironmentState::Closed, + EnvironmentState::Failed, + ] { + assert_eq!(EnvironmentState::parse(state.as_str()), Some(state)); + assert_eq!( + serde_json::to_value(state).unwrap(), + serde_json::json!(state.as_str()) + ); } - .metadata(); - - let mut full = register("rk-meta", "env-meta", &daemon_key(0x31), 1_000); - full.metadata = (0..MAX_METADATA_ENTRIES) - .map(|index| (format!("caller.key{index}"), "v".to_owned())) - .collect(); - full.metadata.extend(reserved.clone()); - let created = store - .create_registered_environment(full) - .await - .expect("full caller allowance plus reserved entries"); - assert_eq!( - created.metadata.len(), - MAX_METADATA_ENTRIES + reserved.len() - ); - - let mut over = register("rk-meta", "env-over", &daemon_key(0x32), 1_000); - over.metadata = (0..=MAX_METADATA_ENTRIES) - .map(|index| (format!("caller.key{index}"), "v".to_owned())) - .collect(); - assert!(matches!( - store.create_registered_environment(over).await, - Err(EnvironmentRegistryError::InvalidInput { .. }) - )); + for reason in [ + EnvironmentCloseReason::Requested, + EnvironmentCloseReason::KeyRevoked, + EnvironmentCloseReason::EphemeralGrace, + EnvironmentCloseReason::Departed, + EnvironmentCloseReason::OfflineTimeout, + EnvironmentCloseReason::ProviderEnded, + ] { + assert_eq!(EnvironmentCloseReason::parse(reason.as_str()), Some(reason)); + assert_eq!( + serde_json::to_value(reason).unwrap(), + serde_json::json!(reason.as_str()) + ); + } + assert_eq!(EnvironmentState::parse("ready"), None); } diff --git a/crates/harness/src/storage/blobs.rs b/crates/harness/src/storage/blobs.rs index 25f9fdd2f..580488ed3 100644 --- a/crates/harness/src/storage/blobs.rs +++ b/crates/harness/src/storage/blobs.rs @@ -126,19 +126,26 @@ pub trait BlobSource: Send { async fn read_chunk(&mut self, max_bytes: usize) -> Result, BlobStoreError>; } +/// Largest range one [`BlobStore::read_blob_range`] call may request; stores +/// refuse larger ranges, so callers moving bigger chunks read several ranges. +pub const MAX_BLOB_RANGE_BYTES: usize = 1024 * 1024; + #[async_trait] pub trait BlobStore: Send + Sync { async fn put_bytes(&self, bytes: Vec) -> Result; - /// Bounded range access for transfer. Ranges are verified by the receiving whole-file - /// hasher. The fallback deliberately rejects large blobs instead of buffering them. + /// Bounded range access for transfer, at most [`MAX_BLOB_RANGE_BYTES`] per call. Ranges + /// are verified by the receiving whole-file hasher. The fallback deliberately rejects + /// large blobs instead of buffering them. async fn read_blob_range( &self, blob_ref: &BlobRef, offset: u64, max_bytes: usize, ) -> Result, BlobStoreError> { - if max_bytes > 1024 * 1024 || self.stat_blob(blob_ref).await?.byte_len > 8 * 1024 * 1024 { + if max_bytes > MAX_BLOB_RANGE_BYTES + || self.stat_blob(blob_ref).await?.byte_len > 8 * 1024 * 1024 + { return Err(BlobStoreError::Store { message: "streaming blob reads unsupported by this store".into(), }); diff --git a/crates/harness/src/storage/mod.rs b/crates/harness/src/storage/mod.rs index 983605a77..486021833 100644 --- a/crates/harness/src/storage/mod.rs +++ b/crates/harness/src/storage/mod.rs @@ -14,7 +14,8 @@ pub use crate::session::{ pub use blobs::{ BlobCacheLimits, BlobCacheStats, BlobEdge, BlobGraphStore, BlobInfo, BlobSource, BlobStore, BlobStoreError, CachedBlobStore, ENGINE_BLOB_CONTENTS, InMemoryBlobCache, InMemoryBlobStore, - collect_blob_refs, ensure_harness_blobs, harness_blob_refs, record_contains_edges, + MAX_BLOB_RANGE_BYTES, collect_blob_refs, ensure_harness_blobs, harness_blob_refs, + record_contains_edges, }; pub use session::{ AdvanceSessionCheckpoint, AppendSessionEvents, AppendSessionEventsResult, CreateClonedSession, diff --git a/crates/llm-runtime/tests/fixtures/builtin_catalogs.json b/crates/llm-runtime/tests/fixtures/builtin_catalogs.json index fdb51e733..65234b66b 100644 --- a/crates/llm-runtime/tests/fixtures/builtin_catalogs.json +++ b/crates/llm-runtime/tests/fixtures/builtin_catalogs.json @@ -3304,7 +3304,7 @@ "type": "object" }, "job_id": { - "description": "Your name for the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read.", + "description": "Your name for this run of the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read. An id names one run: to run a command again, choose a new id.", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", "type": "string" }, @@ -6672,7 +6672,7 @@ "type": "object" }, "job_id": { - "description": "Your name for the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read.", + "description": "Your name for this run of the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read. An id names one run: to run a command again, choose a new id.", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", "type": "string" }, @@ -10112,7 +10112,7 @@ "type": "object" }, "job_id": { - "description": "Your name for the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read.", + "description": "Your name for this run of the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read. An id names one run: to run a command again, choose a new id.", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", "type": "string" }, diff --git a/crates/profiles/src/lib.rs b/crates/profiles/src/lib.rs index 0b88113b7..9e9c9f8ea 100644 --- a/crates/profiles/src/lib.rs +++ b/crates/profiles/src/lib.rs @@ -161,10 +161,11 @@ impl ProfileSourceExt for ProfileSource { } pub fn validate_profile_document(document: &ProfileDocument) -> Result<(), ProfileError> { - environment_protocol::registration::validate_registration_metadata(None, &document.metadata) - .map_err(|message| ProfileError::InvalidInput { - message: format!("invalid metadata: {message}"), - })?; + environment_protocol::metadata::validate_caller_metadata(&document.metadata).map_err( + |error| ProfileError::InvalidInput { + message: format!("invalid metadata: {error}"), + }, + )?; if let Some(value) = document .retention .as_ref() diff --git a/crates/store-fs/src/blob.rs b/crates/store-fs/src/blob.rs index 0023d17f7..a1c91a93f 100644 --- a/crates/store-fs/src/blob.rs +++ b/crates/store-fs/src/blob.rs @@ -93,7 +93,7 @@ impl BlobStore for FsBlobStore { max_bytes: usize, ) -> Result, BlobStoreError> { use tokio::io::{AsyncReadExt, AsyncSeekExt}; - if max_bytes > 1024 * 1024 { + if max_bytes > harness::storage::MAX_BLOB_RANGE_BYTES { return Err(BlobStoreError::Store { message: "blob range limit exceeded".into(), }); diff --git a/crates/store-pg/migrations/005_environments.sql b/crates/store-pg/migrations/005_environments.sql index fec0c6683..4d1aa3e1c 100644 --- a/crates/store-pg/migrations/005_environments.sql +++ b/crates/store-pg/migrations/005_environments.sql @@ -1,18 +1,59 @@ --- Environment compute: deployment providers, universe bindings, and --- durable logical environments with incarnation-scoped physical facts. +-- Environment compute: deployment providers, universe bindings, registration +-- keys and their secrets, and universe environments of three kinds. +-- +-- Design notes: +-- - A provider is reached at one validated controller endpoint URL (ws://, +-- wss:// or unix://). Its bearer credential is optional; when present it is +-- sealed with AES-256-GCM by the store adapter, exactly like auth_secrets. +-- data_route_base is optional operator configuration; without it the data +-- routes derive from the controller endpoint. +-- - A binding is a universe's use of one provider, keyed by the provider. +-- - A registration key is an admission policy and the group of the +-- environments it admitted. It stores no counters: registration and active +-- counts derive from environment rows carrying registration_key_id. Only +-- SHA-256 hashes of its server-generated secrets are stored, in a child +-- table, so a rotation keeps the group. +-- - One environments row covers three kinds. Lightspeed owns a five-state +-- lifecycle; the observed_* columns and public_endpoint are what the +-- provider last reported, for display only. A registered row's identity +-- mode is read from its key. +-- - The daemon public key is the identity. One public key maps to at most +-- one environment in the whole deployment, ever: the unique index covers +-- closed rows too, so a closed environment's daemon cannot register again +-- without a fresh local identity. CREATE TABLE IF NOT EXISTS environment_providers ( provider_id text PRIMARY KEY, display_name text, - controller_connection_json jsonb NOT NULL, - metadata_json jsonb NOT NULL DEFAULT '{}', + controller_endpoint text NOT NULL, + data_route_base text, + credential_key_id text, + credential_nonce bytea, + credential_ciphertext bytea, created_at_ms bigint NOT NULL, updated_at_ms bigint NOT NULL, CONSTRAINT environment_providers_provider_id_format CHECK (provider_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$'), - CONSTRAINT environment_providers_json_objects CHECK ( - jsonb_typeof(controller_connection_json) = 'object' - AND jsonb_typeof(metadata_json) = 'object' + CONSTRAINT environment_providers_display_name_not_empty + CHECK (display_name IS NULL OR display_name <> ''), + CONSTRAINT environment_providers_controller_endpoint_format CHECK ( + controller_endpoint ~ '^(wss?|unix)://[^[:space:][:cntrl:]]+$' + AND octet_length(controller_endpoint) <= 2048 + ), + CONSTRAINT environment_providers_data_route_base_format CHECK ( + data_route_base IS NULL + OR ( + data_route_base ~ '^wss?://[^[:space:][:cntrl:]]+$' + AND octet_length(data_route_base) <= 2048 + ) + ), + CONSTRAINT environment_providers_credential_all_or_none CHECK ( + (credential_key_id IS NULL AND credential_nonce IS NULL AND credential_ciphertext IS NULL) + OR ( + credential_key_id IS NOT NULL AND credential_key_id <> '' + AND credential_nonce IS NOT NULL AND octet_length(credential_nonce) = 12 + AND credential_ciphertext IS NOT NULL + ) ), CONSTRAINT environment_providers_times_valid CHECK ( created_at_ms >= 0 AND updated_at_ms >= created_at_ms @@ -21,252 +62,267 @@ CREATE TABLE IF NOT EXISTS environment_providers ( CREATE TABLE IF NOT EXISTS environment_provider_bindings ( universe_id uuid NOT NULL REFERENCES universes (universe_id) ON DELETE CASCADE, - binding_id text NOT NULL, provider_id text NOT NULL REFERENCES environment_providers (provider_id) ON DELETE RESTRICT, - status text NOT NULL, - revision bigint NOT NULL, - metadata_json jsonb NOT NULL DEFAULT '{}', + enabled boolean NOT NULL, created_at_ms bigint NOT NULL, updated_at_ms bigint NOT NULL, - PRIMARY KEY (universe_id, binding_id), - UNIQUE (universe_id, provider_id), - CONSTRAINT environment_provider_bindings_binding_id_format - CHECK (binding_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$'), - CONSTRAINT environment_provider_bindings_status_known - CHECK (status IN ('enabled', 'disabled')), - CONSTRAINT environment_provider_bindings_revision_positive CHECK (revision > 0), - CONSTRAINT environment_provider_bindings_metadata_object - CHECK (jsonb_typeof(metadata_json) = 'object'), + PRIMARY KEY (universe_id, provider_id), CONSTRAINT environment_provider_bindings_times_valid CHECK ( created_at_ms >= 0 AND updated_at_ms >= created_at_ms ) ); --- Key-based outbound environment registration: reusable, universe-scoped --- registration keys and the registered environment source they admit. --- --- Design notes: --- - A registration key is an admission policy and the group of the --- environments it admitted. It stores no counters: registration and active --- counts derive from environment rows carrying registration_key_id. --- - Only the SHA-256 hash of the server-generated secret is stored, exactly --- like api_keys. key_prefix is the display handle. --- - identity_mode is key policy copied onto every environment the key --- admits; the daemon neither knows nor chooses it. --- - The daemon public key is the identity. One public key maps to at most --- one environment in the whole deployment, ever: the unique index covers --- closed rows too, so a closed environment's daemon cannot register again --- without a fresh local identity. --- - last_seen_at_ms is the gateway's heartbeat stamp on the control --- connection; the lifecycle reconciler derives ephemeral cleanup and stale --- Ready repair from it. - CREATE TABLE IF NOT EXISTS environment_registration_keys ( universe_id uuid NOT NULL REFERENCES universes (universe_id) ON DELETE CASCADE, registration_key_id text NOT NULL, display_name text NOT NULL, - key_prefix text NOT NULL, - secret_hash text NOT NULL, identity_mode text NOT NULL, max_active_environments integer, ephemeral_disconnect_grace_ms bigint, + offline_close_after_ms bigint, expires_at_ms bigint, - created_at_ms bigint NOT NULL, revoked_at_ms bigint, + created_at_ms bigint NOT NULL, + updated_at_ms bigint NOT NULL, PRIMARY KEY (universe_id, registration_key_id), CONSTRAINT environment_registration_keys_id_format CHECK (registration_key_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$'), CONSTRAINT environment_registration_keys_display_name_bounded CHECK (display_name <> '' AND length(display_name) <= 128), - CONSTRAINT environment_registration_keys_key_prefix_not_empty - CHECK (key_prefix <> ''), - CONSTRAINT environment_registration_keys_secret_hash_format - CHECK (secret_hash ~ '^[0-9a-f]{64}$'), CONSTRAINT environment_registration_keys_identity_mode_known CHECK (identity_mode IN ('persistent', 'ephemeral')), CONSTRAINT environment_registration_keys_limits_positive CHECK ( (max_active_environments IS NULL OR max_active_environments > 0) AND (ephemeral_disconnect_grace_ms IS NULL OR ephemeral_disconnect_grace_ms > 0) + AND (offline_close_after_ms IS NULL OR offline_close_after_ms > 0) ), + CONSTRAINT environment_registration_keys_offline_close_persistent + CHECK (offline_close_after_ms IS NULL OR identity_mode = 'persistent'), CONSTRAINT environment_registration_keys_times_valid CHECK ( - created_at_ms >= 0 + created_at_ms >= 0 AND updated_at_ms >= created_at_ms AND (expires_at_ms IS NULL OR expires_at_ms >= 0) AND (revoked_at_ms IS NULL OR revoked_at_ms >= created_at_ms) ) ); -CREATE UNIQUE INDEX IF NOT EXISTS environment_registration_keys_secret_hash_idx - ON environment_registration_keys (secret_hash); +-- Secrets of a key: rotation retires one and adds another, so the key's +-- group, allowlists and identity mode survive. secret_hash is SHA-256 of the +-- server-generated secret; it is unique deployment-wide, retired secrets +-- included. +CREATE TABLE IF NOT EXISTS environment_registration_key_secrets ( + secret_hash text PRIMARY KEY, + universe_id uuid NOT NULL, + registration_key_id text NOT NULL, + key_prefix text NOT NULL, + created_at_ms bigint NOT NULL, + retired_at_ms bigint, + FOREIGN KEY (universe_id, registration_key_id) + REFERENCES environment_registration_keys (universe_id, registration_key_id) + ON DELETE CASCADE, + CONSTRAINT environment_registration_key_secrets_hash_format + CHECK (secret_hash ~ '^[0-9a-f]{64}$'), + CONSTRAINT environment_registration_key_secrets_prefix_not_empty + CHECK (key_prefix <> ''), + CONSTRAINT environment_registration_key_secrets_times_valid CHECK ( + created_at_ms >= 0 AND (retired_at_ms IS NULL OR retired_at_ms >= created_at_ms) + ) +); -CREATE UNIQUE INDEX IF NOT EXISTS environment_registration_keys_key_prefix_idx - ON environment_registration_keys (key_prefix); +CREATE UNIQUE INDEX IF NOT EXISTS environment_registration_key_secrets_live_idx + ON environment_registration_key_secrets (universe_id, registration_key_id) + WHERE retired_at_ms IS NULL; CREATE TABLE IF NOT EXISTS environments ( universe_id uuid NOT NULL REFERENCES universes (universe_id) ON DELETE CASCADE, environment_id text NOT NULL, - request_id text NOT NULL, source_kind text NOT NULL, + state text NOT NULL, + state_message text, + close_reason text, + state_changed_at_ms bigint NOT NULL, + request_id text, + display_name text, + metadata_json jsonb NOT NULL DEFAULT '{}', + daemon_version text, + daemon_git_sha text, + daemon_protocol_version integer, + -- provisioned (no FK on provider_id: closed rows outlive providers) provider_id text, - binding_id text, - daemon_connection_json jsonb, + template_id text, + create_idle_json jsonb, + provider_target_id text, + next_attempt_at_ms bigint, + attempts integer NOT NULL DEFAULT 0, + observed_phase text, + observed_power text, + observed_message text, + observed_at_ms bigint, + public_endpoint text, + -- external + daemon_endpoint text, + -- registered registration_key_id text, - daemon_id text, daemon_public_key text, - identity_mode text, + connected boolean, + connection_changed_at_ms bigint, last_seen_at_ms bigint, - display_name text, - status text NOT NULL, - -- Lightspeed-owned power intent and optional staged idle policy. Observed - -- power remains in status; activity is read from the daemon on demand. - desired_power text NOT NULL DEFAULT 'running', - idle_policy_json jsonb, - current_incarnation_id text NOT NULL, - public_ingress_enabled boolean NOT NULL DEFAULT false, - public_endpoint text, - metadata_json jsonb NOT NULL DEFAULT '{}', created_at_ms bigint NOT NULL, updated_at_ms bigint NOT NULL, + closed_at_ms bigint, PRIMARY KEY (universe_id, environment_id), - UNIQUE (universe_id, request_id), + -- Deferrable so a universe delete can check at commit, after its + -- cascade has removed the environments too. CONSTRAINT environments_registration_key_fk FOREIGN KEY (universe_id, registration_key_id) REFERENCES environment_registration_keys (universe_id, registration_key_id) - ON DELETE RESTRICT, + ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE, CONSTRAINT environments_ids_format CHECK ( environment_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$' - AND request_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$' - AND current_incarnation_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$' + AND (request_id IS NULL OR request_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') + AND (template_id IS NULL OR template_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') + AND ( + provider_target_id IS NULL + OR ( + octet_length(provider_target_id) BETWEEN 1 AND 255 + AND provider_target_id !~ '[[:cntrl:]]' + ) + ) ), CONSTRAINT environments_source_known CHECK (source_kind IN ('provisioned', 'external', 'registered')), + CONSTRAINT environments_state_known + CHECK (state IN ('provisioning', 'open', 'closing', 'closed', 'failed')), + CONSTRAINT environments_state_per_kind + CHECK (source_kind = 'provisioned' OR state IN ('open', 'closed')), + CONSTRAINT environments_close_reason CHECK ( + (close_reason IS NULL) = (state NOT IN ('closing', 'closed')) + AND ( + close_reason IS NULL + OR close_reason IN ( + 'requested', 'key_revoked', 'ephemeral_grace', + 'departed', 'offline_timeout', 'provider_ended' + ) + ) + ), + CONSTRAINT environments_closed_at + CHECK ((closed_at_ms IS NULL) = (state <> 'closed')), + CONSTRAINT environments_state_message_bounded CHECK ( + state_message IS NULL OR octet_length(state_message) BETWEEN 1 AND 1024 + ), + CONSTRAINT environments_work_schedule CHECK ( + (next_attempt_at_ms IS NOT NULL) + = (source_kind = 'provisioned' AND state IN ('provisioning', 'closing')) + AND attempts >= 0 + ), + CONSTRAINT environments_daemon_build CHECK ( + (daemon_protocol_version IS NOT NULL OR (daemon_version IS NULL AND daemon_git_sha IS NULL)) + AND (daemon_version IS NULL OR daemon_version <> '') + AND (daemon_git_sha IS NULL OR daemon_git_sha <> '') + ), CONSTRAINT environments_source_fields CHECK ( ( source_kind = 'provisioned' - AND provider_id IS NOT NULL AND binding_id IS NOT NULL - AND daemon_connection_json IS NULL - AND registration_key_id IS NULL AND daemon_id IS NULL - AND daemon_public_key IS NULL AND identity_mode IS NULL - AND last_seen_at_ms IS NULL + AND request_id IS NOT NULL AND provider_id IS NOT NULL AND template_id IS NOT NULL + AND daemon_endpoint IS NULL + AND registration_key_id IS NULL AND daemon_public_key IS NULL + AND connected IS NULL AND connection_changed_at_ms IS NULL AND last_seen_at_ms IS NULL ) OR ( source_kind = 'external' - AND provider_id IS NULL AND binding_id IS NULL - AND jsonb_typeof(daemon_connection_json) = 'object' - AND registration_key_id IS NULL AND daemon_id IS NULL - AND daemon_public_key IS NULL AND identity_mode IS NULL - AND last_seen_at_ms IS NULL + AND request_id IS NOT NULL + AND provider_id IS NULL AND template_id IS NULL AND create_idle_json IS NULL + AND provider_target_id IS NULL + AND observed_phase IS NULL AND public_endpoint IS NULL + AND daemon_endpoint IS NOT NULL + AND daemon_endpoint ~ '^(wss?|unix)://[^[:space:][:cntrl:]]+$' + AND octet_length(daemon_endpoint) <= 2048 + AND registration_key_id IS NULL AND daemon_public_key IS NULL + AND connected IS NULL AND connection_changed_at_ms IS NULL AND last_seen_at_ms IS NULL ) OR ( source_kind = 'registered' - AND provider_id IS NULL AND binding_id IS NULL - AND daemon_connection_json IS NULL + AND request_id IS NULL + AND provider_id IS NULL AND template_id IS NULL AND create_idle_json IS NULL + AND provider_target_id IS NULL + AND observed_phase IS NULL AND public_endpoint IS NULL + AND daemon_endpoint IS NULL AND registration_key_id IS NOT NULL - AND daemon_id ~ '^daemon_[0-9a-f]{64}$' - AND daemon_public_key ~ '^[0-9a-f]{64}$' - AND identity_mode IN ('persistent', 'ephemeral') - AND (last_seen_at_ms IS NULL OR last_seen_at_ms >= 0) + AND daemon_public_key IS NOT NULL AND daemon_public_key ~ '^[0-9a-f]{64}$' + AND connected IS NOT NULL + AND connection_changed_at_ms IS NOT NULL AND connection_changed_at_ms >= 0 + AND last_seen_at_ms IS NOT NULL AND last_seen_at_ms >= 0 ) ), - CONSTRAINT environments_status_known CHECK (status IN ( - 'provisioning', 'booting', 'ready', 'paused', 'suspended', 'offline', - 'closing', 'closed', 'failed', 'unknown' - )), - CONSTRAINT environments_desired_power_known - CHECK (desired_power IN ('running', 'paused', 'suspended', 'stopped')), - CONSTRAINT environments_idle_policy_object - CHECK (idle_policy_json IS NULL OR jsonb_typeof(idle_policy_json) = 'object'), - CONSTRAINT environments_power_provisioned_only CHECK ( - source_kind = 'provisioned' - OR (desired_power = 'running' AND idle_policy_json IS NULL) + CONSTRAINT environments_observation CHECK ( + (observed_phase IS NULL) = (observed_at_ms IS NULL) + AND ( + observed_phase IS NULL + OR observed_phase IN ('provisioning', 'active', 'closing', 'closed', 'failed', 'unknown') + ) + AND ( + observed_power IS NULL + OR ( + observed_phase IS NOT NULL AND observed_phase = 'active' + AND observed_power IN ('running', 'paused', 'suspended', 'stopped') + ) + ) + AND ( + observed_message IS NULL + OR (observed_phase IS NOT NULL AND octet_length(observed_message) BETWEEN 1 AND 1024) + ) + AND ( + public_endpoint IS NULL + OR (public_endpoint <> '' AND octet_length(public_endpoint) <= 2048 AND state <> 'closed') + ) ), - CONSTRAINT environments_metadata_object - CHECK (jsonb_typeof(metadata_json) = 'object'), - CONSTRAINT environments_public_ingress_fields CHECK ( - (public_ingress_enabled AND source_kind = 'provisioned' AND public_endpoint IS NOT NULL AND public_endpoint <> '') - OR (NOT public_ingress_enabled AND public_endpoint IS NULL) + CONSTRAINT environments_json_objects CHECK ( + jsonb_typeof(metadata_json) = 'object' + AND (create_idle_json IS NULL OR jsonb_typeof(create_idle_json) = 'object') ), CONSTRAINT environments_times_valid CHECK ( created_at_ms >= 0 AND updated_at_ms >= created_at_ms + AND state_changed_at_ms >= created_at_ms + AND (closed_at_ms IS NULL OR closed_at_ms >= created_at_ms) ) ); -CREATE INDEX IF NOT EXISTS environments_binding_status_idx - ON environments (universe_id, binding_id, status, environment_id); +CREATE UNIQUE INDEX IF NOT EXISTS environments_request_id_idx + ON environments (universe_id, request_id) + WHERE request_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS environments_idle_policy_idx - ON environments (universe_id) - WHERE idle_policy_json IS NOT NULL AND status = 'ready'; +CREATE INDEX IF NOT EXISTS environments_provider_state_idx + ON environments (universe_id, provider_id, state) + WHERE provider_id IS NOT NULL; --- Deployment-wide: one daemon identity, one environment, ever. +CREATE INDEX IF NOT EXISTS environments_registration_key_state_idx + ON environments (universe_id, registration_key_id, state) + WHERE registration_key_id IS NOT NULL; + +-- The reconciler's work queue (provisioning and closing provisioned rows). +CREATE INDEX IF NOT EXISTS environments_work_idx + ON environments (next_attempt_at_ms) + WHERE next_attempt_at_ms IS NOT NULL; + +-- Deployment-wide: one daemon identity, one environment, ever (closed rows included). CREATE UNIQUE INDEX IF NOT EXISTS environments_daemon_public_key_idx ON environments (daemon_public_key) WHERE daemon_public_key IS NOT NULL; -CREATE INDEX IF NOT EXISTS environments_registration_key_idx - ON environments (universe_id, registration_key_id, status) - WHERE registration_key_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS environments_provider_target_idx + ON environments (provider_id, provider_target_id) + WHERE provider_target_id IS NOT NULL AND state <> 'closed'; -CREATE INDEX IF NOT EXISTS environments_metadata_idx - ON environments USING gin (metadata_json jsonb_path_ops); +CREATE INDEX IF NOT EXISTS environments_registered_open_idx + ON environments (universe_id, connected, connection_changed_at_ms) + WHERE source_kind = 'registered' AND state = 'open'; -CREATE TABLE IF NOT EXISTS environment_incarnations ( - universe_id uuid NOT NULL, - environment_id text NOT NULL, - incarnation_id text NOT NULL, - provision_request_id text, - provider_target_id text, - template_id text, - adoption_source_target text, - power_states_json jsonb NOT NULL DEFAULT '[]', - created_at_ms bigint NOT NULL, - updated_at_ms bigint NOT NULL, - PRIMARY KEY (universe_id, environment_id, incarnation_id), - FOREIGN KEY (universe_id, environment_id) - REFERENCES environments (universe_id, environment_id) ON DELETE CASCADE, - UNIQUE (universe_id, provision_request_id), - CONSTRAINT environment_incarnations_ids_format CHECK ( - incarnation_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$' - AND ( - provision_request_id IS NULL - OR provision_request_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$' - ) - AND ( - template_id IS NULL - OR template_id ~ '^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$' - ) - AND ( - adoption_source_target IS NULL - OR ( - length(adoption_source_target) BETWEEN 1 AND 255 - AND adoption_source_target !~ '[[:cntrl:]]' - ) - ) - ), - CONSTRAINT environment_incarnations_source_fields CHECK ( - ( - provision_request_id IS NOT NULL - AND ((template_id IS NOT NULL) <> (adoption_source_target IS NOT NULL)) - ) - OR ( - provision_request_id IS NULL - AND provider_target_id IS NULL - AND template_id IS NULL - AND adoption_source_target IS NULL - ) - ), - CONSTRAINT environment_incarnations_power_states_array - CHECK (jsonb_typeof(power_states_json) = 'array'), - CONSTRAINT environment_incarnations_times_valid CHECK ( - created_at_ms >= 0 AND updated_at_ms >= created_at_ms - ) -); +-- The display sweep's scan set. +CREATE INDEX IF NOT EXISTS environments_provisioned_open_idx + ON environments (universe_id) + WHERE source_kind = 'provisioned' AND state = 'open'; --- Incarnations point back to environments; add the other side after both exist. -ALTER TABLE environments - ADD CONSTRAINT environments_current_incarnation_fk - FOREIGN KEY (universe_id, environment_id, current_incarnation_id) - REFERENCES environment_incarnations (universe_id, environment_id, incarnation_id) - DEFERRABLE INITIALLY DEFERRED; +CREATE INDEX IF NOT EXISTS environments_metadata_idx + ON environments USING gin (metadata_json jsonb_path_ops); CREATE TABLE IF NOT EXISTS environment_credentials ( universe_id uuid NOT NULL, @@ -281,12 +337,17 @@ CREATE TABLE IF NOT EXISTS environment_credentials ( PRIMARY KEY (universe_id, environment_id, env_name), FOREIGN KEY (universe_id, environment_id) REFERENCES environments (universe_id, environment_id) ON DELETE CASCADE, + -- Deferrable so a universe delete can check at commit, after its + -- cascade has removed the bindings too. FOREIGN KEY (universe_id, grant_id) - REFERENCES auth_grants (universe_id, grant_id) ON DELETE RESTRICT, + REFERENCES auth_grants (universe_id, grant_id) + ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE, FOREIGN KEY (universe_id, auth_provider_id) - REFERENCES auth_providers (universe_id, provider_id) ON DELETE RESTRICT, + REFERENCES auth_providers (universe_id, provider_id) + ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE, FOREIGN KEY (universe_id, secret_id) - REFERENCES auth_secrets (universe_id, secret_id) ON DELETE RESTRICT, + REFERENCES auth_secrets (universe_id, secret_id) + ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE, CONSTRAINT environment_credentials_env_name_format CHECK (env_name ~ '^[A-Za-z_][A-Za-z0-9_]{0,127}$'), CONSTRAINT environment_credentials_source_kind_known @@ -302,35 +363,24 @@ CREATE TABLE IF NOT EXISTS environment_credentials ( ); COMMENT ON TABLE environment_providers IS - 'Operator-registered provider identity and controller connection; protocol and presence are observed transiently.'; -COMMENT ON COLUMN environment_providers.metadata_json IS - 'Non-authoritative operator metadata; never provider capability, health, or allocation policy.'; + 'Operator-registered provider identity, controller endpoint, optional data route base and optional sealed credential.'; +COMMENT ON COLUMN environment_providers.data_route_base IS + 'Operator-configured ws:// or wss:// base of the provider data routes; null derives it from the controller endpoint.'; +COMMENT ON COLUMN environment_providers.credential_ciphertext IS + 'AES-256-GCM sealed bearer credential for the controller and data routes, AAD = provider id; all three credential columns are null without a credential.'; COMMENT ON TABLE environment_provider_bindings IS - 'Revisioned universe routing and admission binding to one provider; it is not a provider policy document.'; -COMMENT ON COLUMN environment_provider_bindings.metadata_json IS - 'Non-authoritative binding labels; never provider template, quota, capacity, or ingress policy.'; -COMMENT ON TABLE environments IS - 'Universe-owned logical environments. External environments store a Lightspeed-reachable envd endpoint; provisioned environments retain provider routing linkage.'; -COMMENT ON TABLE environment_incarnations IS - 'Lightspeed-authorized environment generations with stable provider retry and target linkage; not provider inventory or live gateway state.'; -COMMENT ON COLUMN environment_incarnations.provider_target_id IS - 'Opaque provider-scoped target handle returned by createTarget; interpreted with the environment provider identity.'; -COMMENT ON COLUMN environment_incarnations.adoption_source_target IS - 'Provider-native source reference for an explicit operator-managed adoption; retained for idempotent lifecycle retries.'; -COMMENT ON COLUMN environments.public_endpoint IS - 'Provider-realized public HTTPS endpoint; port, private target, proxy configuration, TLS, health, and policy remain provider-owned.'; -COMMENT ON COLUMN environments.desired_power IS - 'Lightspeed-owned power intent (running|paused|suspended|stopped); converged by the lifecycle reconciler, observed state is status.'; -COMMENT ON COLUMN environments.idle_policy_json IS - 'Optional staged idle policy {pauseAfterMs,suspendAfterMs,stopAfterMs,closeAfterMs}; applied by the power reaper from the daemon idle report.'; -COMMENT ON COLUMN environment_incarnations.power_states_json IS - 'Provider-reported power states this target supports, observed with the target id.'; -COMMENT ON TABLE environment_credentials IS - 'Universe-owned credential bindings for an environment.'; - + 'A universe''s use of one provider; enabled admits creates, template listing and data routes.'; COMMENT ON TABLE environment_registration_keys IS 'Reusable universe-scoped admission policies for outbound envd registration; each key is also the group of the environments it admitted.'; +COMMENT ON TABLE environment_registration_key_secrets IS + 'SHA-256 hashes of a registration key''s secrets; exactly one is live, rotation retires it and adds another.'; +COMMENT ON TABLE environments IS + 'Universe-owned environments of three kinds (provisioned, external, registered) with the five-state lifecycle Lightspeed owns.'; +COMMENT ON COLUMN environments.create_idle_json IS + 'Create-time idle policy passed to target/create on every retry; never interpreted by Lightspeed.'; COMMENT ON COLUMN environments.daemon_public_key IS 'Lowercase hex Ed25519 public key of the registered daemon; the identity, unique across the deployment including closed rows.'; COMMENT ON COLUMN environments.last_seen_at_ms IS - 'Gateway heartbeat stamp on the registered control connection; stale under ready means the gateway stopped without recording the disconnect.'; + 'Coarse registered liveness: written at connect, at disconnect, and at most every 5 minutes while connected.'; +COMMENT ON TABLE environment_credentials IS + 'Universe-owned credential bindings for an environment.'; diff --git a/crates/store-pg/src/auth.rs b/crates/store-pg/src/auth.rs index f97f23b03..7462f4205 100644 --- a/crates/store-pg/src/auth.rs +++ b/crates/store-pg/src/auth.rs @@ -21,11 +21,11 @@ use sqlx::Row; use crate::PgStore; -const LOCAL_KEY_ID: &str = "local-v1"; +pub(crate) const LOCAL_KEY_ID: &str = "local-v1"; const NONCE_LEN: usize = 12; impl PgStore { - fn secrets_cipher(&self) -> Result<(Aes256Gcm, &'static str), AuthRegistryError> { + pub(crate) fn secrets_cipher(&self) -> Result<(Aes256Gcm, &'static str), AuthRegistryError> { let Some(key) = &self.config.secrets_master_key else { return Err(AuthRegistryError::Store { message: "secrets master key is not configured; set LIGHTSPEED_SECRETS_MASTER_KEY" @@ -49,7 +49,7 @@ impl PgStore { } } -fn seal_secret( +pub(crate) fn seal_secret( cipher: &Aes256Gcm, aad: &[u8], value: &SecretValue, @@ -72,7 +72,7 @@ fn seal_secret( Ok((nonce.to_vec(), ciphertext)) } -fn open_secret( +pub(crate) fn open_secret( cipher: &Aes256Gcm, aad: &[u8], nonce: &[u8], diff --git a/crates/store-pg/src/blob.rs b/crates/store-pg/src/blob.rs index 6b2863b53..1d62008c0 100644 --- a/crates/store-pg/src/blob.rs +++ b/crates/store-pg/src/blob.rs @@ -214,7 +214,7 @@ impl BlobStore for PgStore { max_bytes: usize, ) -> Result, BlobStoreError> { use object_store::ObjectStoreExt; - if max_bytes > 1024 * 1024 { + if max_bytes > harness::storage::MAX_BLOB_RANGE_BYTES { return Err(BlobStoreError::Store { message: "blob range limit exceeded".into(), }); diff --git a/crates/store-pg/src/deployment.rs b/crates/store-pg/src/deployment.rs index b0f8b1662..a50e7bc35 100644 --- a/crates/store-pg/src/deployment.rs +++ b/crates/store-pg/src/deployment.rs @@ -150,11 +150,20 @@ pub async fn list_universe_object_keys( /// Delete the universe row; every universe-scoped table cascades from it, /// including its keys. `false` when the universe did not exist. +/// +/// Deferrable foreign keys between universe-scoped tables (environments to +/// registration keys, environment credentials to auth objects) are checked +/// at commit, so the cascade order between their tables does not matter. pub async fn delete_universe(pool: &PgPool, universe_id: Uuid) -> Result { + let mut tx = pool.begin().await?; + sqlx::query("SET CONSTRAINTS ALL DEFERRED") + .execute(&mut *tx) + .await?; let result = sqlx::query("DELETE FROM universes WHERE universe_id = $1") .bind(universe_id) - .execute(pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(result.rows_affected() > 0) } diff --git a/crates/store-pg/src/environment.rs b/crates/store-pg/src/environment.rs index 61d20da40..bbe9a21df 100644 --- a/crates/store-pg/src/environment.rs +++ b/crates/store-pg/src/environment.rs @@ -1,66 +1,72 @@ +//! Environment providers, universe bindings and environments. +//! +//! Every environment write locks its row and applies the domain request's +//! pure `apply` in one transaction, so this store and the in-memory one +//! share each transition's skip rule: a write whose expected state no longer +//! holds leaves the row as it is. The writes that run often or in bulk (the +//! liveness stamp, the deployment sweeps) are single +//! conditional statements with the same rules. + // Formatted SQL uses internal schema fragments; request values use bind parameters. +use std::collections::{BTreeMap, HashMap}; + use async_trait::async_trait; -use auth::{AuthGrantId, AuthProviderId, SecretId}; -use environment_protocol::shared::ProviderTargetId; +use auth::{AuthGrantId, AuthProviderId, SecretId, SecretValue}; +use environment_protocol::{ids::ProviderTargetId, routes::DataRouteBase}; use environments::{ - AdoptEnvironment, BeginCloseEnvironment, CreateEnvironment, CreateExternalEnvironment, - CreateRegisteredEnvironment, EnvironmentCredentialRecord, EnvironmentCredentialSource, - EnvironmentCredentialStore, EnvironmentDaemonId, EnvironmentId, EnvironmentIncarnationId, - EnvironmentIncarnationRecord, EnvironmentProviderBindingId, EnvironmentProviderBindingRecord, - EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, - EnvironmentProviderRecord, EnvironmentProviderStore, EnvironmentProvisionRequestId, - EnvironmentRecord, EnvironmentRegistrationKeyId, EnvironmentRegistryError, EnvironmentSource, - EnvironmentStatus, EnvironmentStore, EnvironmentTemplateId, FailEnvironmentLifecycle, - FinishCloseEnvironment, ListEnvironmentCredentials, ListEnvironmentProviders, ListEnvironments, - ObserveProvisionedEnvironment, ObserveRegisteredEnvironment, PowerState, - PutEnvironmentCredential, PutEnvironmentProvider, PutEnvironmentProviderBinding, - RegisteredConnectionObservation, RegisteredIdentityMode, SetEnvironmentIdlePolicy, - SetEnvironmentIngress, SetEnvironmentPower, + BeginCloseEnvironment, CreateEnvironment, CreateExternalEnvironment, + CreateRegisteredEnvironment, DEFAULT_EPHEMERAL_DISCONNECT_GRACE_MS, DaemonBuild, + EndProvisionedEnvironment, EnvironmentCloseReason, EnvironmentCredentialRecord, + EnvironmentCredentialSource, EnvironmentCredentialStore, EnvironmentEndpoint, EnvironmentId, + EnvironmentPage, EnvironmentProviderBindingRecord, EnvironmentProviderBindingStore, + EnvironmentProviderId, EnvironmentProviderRecord, EnvironmentProviderStore, + EnvironmentProvisionRequestId, EnvironmentRecord, EnvironmentRegistrationKeyId, + EnvironmentRegistrationKeyRecord, EnvironmentRegistrationKeyStore, EnvironmentRegistryError, + EnvironmentSource, EnvironmentState, EnvironmentStore, EnvironmentTemplateId, FailEnvironment, + FinishCloseEnvironment, ListEnvironmentCredentials, ListEnvironments, MarkRegisteredConnected, + MarkRegisteredDisconnected, ObservedTarget, OpenEnvironment, PowerState, + ProviderCredentialChange, ProvisionedSource, PutEnvironmentCredential, PutEnvironmentProvider, + PutEnvironmentProviderBinding, REGISTERED_LIVENESS_STAMP_MS, REGISTERED_STALE_AFTER_MS, + RecordTargetObservation, RegisteredIdentityMode, RegisteredSource, RegisteredSweep, + ScheduleEnvironmentAttempt, StampRegisteredSeen, TargetPhase, UpdateEnvironment, + sweep_registered_environment, validate_daemon_public_key, }; -use sqlx::{Postgres, Row, Transaction}; +use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow}; use uuid::Uuid; -use crate::PgStore; +use crate::{ + PgStore, PgStoreError, + auth::{LOCAL_KEY_ID, open_secret, seal_secret}, +}; const PROVIDER_COLUMNS: &str = r#" - provider_id, display_name, controller_connection_json, metadata_json, - created_at_ms, updated_at_ms + provider_id, display_name, controller_endpoint, data_route_base, + credential_key_id IS NOT NULL AS has_credential, created_at_ms, updated_at_ms "#; -const BINDING_COLUMNS: &str = r#" - universe_id, binding_id, provider_id, status, revision, metadata_json, - created_at_ms, updated_at_ms -"#; +const BINDING_COLUMNS: &str = "universe_id, provider_id, enabled, created_at_ms, updated_at_ms"; +/// Columns of `environments e` plus the identity mode of a registered row's +/// key `k` (see [`ENVIRONMENT_FROM`]). const ENVIRONMENT_COLUMNS: &str = r#" - e.environment_id, e.request_id, e.source_kind, e.provider_id, e.binding_id, e.daemon_connection_json, - e.registration_key_id, e.daemon_id, e.daemon_public_key, e.identity_mode, e.last_seen_at_ms, - e.display_name, e.status, e.desired_power, e.idle_policy_json, - e.public_ingress_enabled, e.public_endpoint, e.metadata_json, - e.created_at_ms, e.updated_at_ms, - i.incarnation_id, i.provision_request_id, i.provider_target_id, - i.template_id, i.adoption_source_target, i.power_states_json, - i.created_at_ms AS incarnation_created_at_ms, - i.updated_at_ms AS incarnation_updated_at_ms + e.environment_id, e.source_kind, e.state, e.state_message, e.close_reason, + e.state_changed_at_ms, e.request_id, e.display_name, e.metadata_json, + e.daemon_version, e.daemon_git_sha, e.daemon_protocol_version, + e.provider_id, e.template_id, e.create_idle_json, e.provider_target_id, + e.next_attempt_at_ms, e.attempts, + e.observed_phase, e.observed_power, e.observed_message, e.observed_at_ms, + e.public_endpoint, e.daemon_endpoint, + e.registration_key_id, e.daemon_public_key, e.connected, + e.connection_changed_at_ms, e.last_seen_at_ms, + e.created_at_ms, e.updated_at_ms, e.closed_at_ms, + k.identity_mode "#; -/// SQL predicate: the observed steady power state differs from the desired -/// one, so the lifecycle reconciler has power work to do. -const POWER_DIVERGES_SQL: &str = r#"( - e.source_kind = 'provisioned' AND ( - (e.status = 'ready' AND e.desired_power <> 'running') - OR (e.status = 'paused' AND e.desired_power <> 'paused') - OR (e.status = 'suspended' AND e.desired_power <> 'suspended') - OR (e.status = 'offline' AND e.desired_power <> 'stopped') - ) -)"#; - -const ENVIRONMENT_JOIN: &str = r#" +/// Registered rows read their identity mode from the key that admitted them. +const ENVIRONMENT_FROM: &str = r#" FROM environments e - JOIN environment_incarnations i - ON i.universe_id = e.universe_id - AND i.environment_id = e.environment_id - AND i.incarnation_id = e.current_incarnation_id + LEFT JOIN environment_registration_keys k + ON k.universe_id = e.universe_id AND k.registration_key_id = e.registration_key_id "#; const CREDENTIAL_COLUMNS: &str = r#" @@ -74,34 +80,72 @@ impl EnvironmentProviderStore for PgStore { &self, request: PutEnvironmentProvider, ) -> Result { - let record = request.into_record()?; + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin environment provider put", error))?; + let query = format!( + "SELECT {PROVIDER_COLUMNS} FROM environment_providers WHERE provider_id = $1 FOR UPDATE" + ); + // A row that no longer decodes is replaced as if it were absent. + let existing = sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(request.provider_id.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|error| sql_error("lock environment provider", error))? + .and_then(|row| decoded_or_logged(provider_from_row(&row), "environment provider")); + let credential = request.credential.clone(); + let record = request.into_record(existing.as_ref())?; + let (keep, key_id, nonce, ciphertext) = match credential { + ProviderCredentialChange::Keep => (true, None, None, None), + ProviderCredentialChange::Set(credential) => { + let (key_id, nonce, ciphertext) = + self.seal_provider_credential(&record.provider_id, &credential)?; + (false, Some(key_id), Some(nonce), Some(ciphertext)) + } + ProviderCredentialChange::Clear => (false, None, None, None), + }; let query = format!( r#" INSERT INTO environment_providers ( - provider_id, display_name, controller_connection_json, - metadata_json, created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,$4,$5,$6) + provider_id, display_name, controller_endpoint, data_route_base, + credential_key_id, credential_nonce, credential_ciphertext, + created_at_ms, updated_at_ms + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (provider_id) DO UPDATE SET display_name = EXCLUDED.display_name, - controller_connection_json = EXCLUDED.controller_connection_json, - metadata_json = EXCLUDED.metadata_json, + controller_endpoint = EXCLUDED.controller_endpoint, + data_route_base = EXCLUDED.data_route_base, + credential_key_id = CASE WHEN $10::boolean + THEN environment_providers.credential_key_id ELSE EXCLUDED.credential_key_id END, + credential_nonce = CASE WHEN $10::boolean + THEN environment_providers.credential_nonce ELSE EXCLUDED.credential_nonce END, + credential_ciphertext = CASE WHEN $10::boolean + THEN environment_providers.credential_ciphertext + ELSE EXCLUDED.credential_ciphertext END, + created_at_ms = EXCLUDED.created_at_ms, updated_at_ms = EXCLUDED.updated_at_ms RETURNING {PROVIDER_COLUMNS} - "# + "# ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(record.provider_id.as_str()) .bind(record.display_name.as_deref()) - .bind(json_value( - "encode provider controller connection", - &record.controller_connection, - )?) - .bind(json_value("encode provider metadata", &record.metadata)?) + .bind(record.controller_endpoint.as_str()) + .bind(record.data_route_base.as_ref().map(DataRouteBase::as_str)) + .bind(key_id) + .bind(nonce) + .bind(ciphertext) .bind(record.created_at_ms) .bind(record.updated_at_ms) - .fetch_one(&self.pool) + .bind(keep) + .fetch_one(&mut *tx) .await .map_err(|error| sql_error("put environment provider", error))?; + tx.commit() + .await + .map_err(|error| sql_error("commit environment provider put", error))?; provider_from_row(&row) } @@ -120,9 +164,52 @@ impl EnvironmentProviderStore for PgStore { provider_from_row(&row) } + async fn read_provider_credential( + &self, + provider_id: &EnvironmentProviderId, + ) -> Result, EnvironmentRegistryError> { + let row = sqlx::query( + "SELECT credential_key_id, credential_nonce, credential_ciphertext FROM environment_providers WHERE provider_id = $1", + ) + .bind(provider_id.as_str()) + .fetch_optional(&self.pool) + .await + .map_err(|error| sql_error("read environment provider credential", error))? + .ok_or_else(|| not_found("environment_provider", provider_id))?; + let key_id: Option = row + .try_get("credential_key_id") + .map_err(|error| sql_error("decode provider credential key id", error))?; + let nonce: Option> = row + .try_get("credential_nonce") + .map_err(|error| sql_error("decode provider credential nonce", error))?; + let ciphertext: Option> = row + .try_get("credential_ciphertext") + .map_err(|error| sql_error("decode provider credential", error))?; + let (key_id, nonce, ciphertext) = match (key_id, nonce, ciphertext) { + (None, None, None) => return Ok(None), + (Some(key_id), Some(nonce), Some(ciphertext)) => (key_id, nonce, ciphertext), + _ => return Err(store_message("provider credential columns are incomplete")), + }; + if key_id != LOCAL_KEY_ID { + return Err(store_message(format!( + "unsupported provider credential key id '{key_id}'" + ))); + } + let (cipher, _) = self + .secrets_cipher() + .map_err(|error| store_message(error.to_string()))?; + open_secret( + &cipher, + &provider_credential_aad(provider_id), + &nonce, + &ciphertext, + ) + .map(Some) + .map_err(|error| store_message(format!("open provider credential: {error}"))) + } + async fn list_providers( &self, - _request: ListEnvironmentProviders, ) -> Result, EnvironmentRegistryError> { let query = format!("SELECT {PROVIDER_COLUMNS} FROM environment_providers ORDER BY provider_id"); @@ -130,104 +217,141 @@ impl EnvironmentProviderStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list environment providers", error))?; - rows.iter().map(provider_from_row).collect() + Ok(decode_rows( + &rows, + provider_from_row, + "environment provider", + )) } async fn delete_provider( &self, provider_id: &EnvironmentProviderId, ) -> Result { + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin environment provider delete", error))?; + let query = format!( + "SELECT {PROVIDER_COLUMNS} FROM environment_providers WHERE provider_id = $1 FOR UPDATE" + ); + sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(provider_id.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|error| sql_error("lock environment provider", error))? + .ok_or_else(|| not_found("environment_provider", provider_id))?; let bindings: i64 = sqlx::query_scalar( "SELECT count(*) FROM environment_provider_bindings WHERE provider_id = $1", ) .bind(provider_id.as_str()) - .fetch_one(&self.pool) + .fetch_one(&mut *tx) .await .map_err(|error| sql_error("count environment provider bindings", error))?; if bindings != 0 { return invalid("environment provider is referenced by a universe binding"); } + let active: i64 = sqlx::query_scalar( + "SELECT count(*) FROM environments WHERE provider_id = $1 AND state <> 'closed'", + ) + .bind(provider_id.as_str()) + .fetch_one(&mut *tx) + .await + .map_err(|error| sql_error("count environment provider environments", error))?; + if active != 0 { + return invalid("environment provider is referenced by a non-closed environment"); + } let query = format!( "DELETE FROM environment_providers WHERE provider_id = $1 RETURNING {PROVIDER_COLUMNS}" ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(provider_id.as_str()) - .fetch_optional(&self.pool) + .fetch_one(&mut *tx) .await - .map_err(map_provider_delete_error)? - .ok_or_else(|| not_found("environment_provider", provider_id))?; + .map_err(map_provider_delete_error)?; + tx.commit() + .await + .map_err(|error| sql_error("commit environment provider delete", error))?; provider_from_row(&row) } } +impl PgStore { + fn seal_provider_credential( + &self, + provider_id: &EnvironmentProviderId, + credential: &SecretValue, + ) -> Result<(&'static str, Vec, Vec), EnvironmentRegistryError> { + let (cipher, key_id) = self + .secrets_cipher() + .map_err(|error| store_message(error.to_string()))?; + let (nonce, ciphertext) = + seal_secret(&cipher, &provider_credential_aad(provider_id), credential) + .map_err(|error| store_message(format!("seal provider credential: {error}")))?; + Ok((key_id, nonce, ciphertext)) + } +} + +/// Binds a sealed provider credential to its provider, so ciphertexts +/// cannot be moved between provider rows. +fn provider_credential_aad(provider_id: &EnvironmentProviderId) -> Vec { + format!("environment-provider/{}/credential", provider_id.as_str()).into_bytes() +} + #[async_trait] impl EnvironmentProviderBindingStore for PgStore { async fn put_provider_binding( &self, request: PutEnvironmentProviderBinding, ) -> Result { - if request.updated_at_ms < 0 { - return invalid("updated_at_ms must be nonnegative"); - } let mut tx = self .pool .begin() .await - .map_err(|error| sql_error("begin binding put", error))?; - let existing = sqlx::query( - "SELECT revision, provider_id FROM environment_provider_bindings WHERE universe_id = $1 AND binding_id = $2 FOR UPDATE" - ).bind(request.universe_id).bind(request.binding_id.as_str()).fetch_optional(&mut *tx).await - .map_err(|error| sql_error("read provider binding revision", error))?; - let actual: Option = existing - .as_ref() - .map(|row| row.try_get("revision")) - .transpose() - .map_err(|error| sql_error("decode provider binding revision", error))?; - if let Some(row) = &existing { - let provider_id: String = row - .try_get("provider_id") - .map_err(|error| sql_error("decode provider binding provider", error))?; - if provider_id != request.provider_id.as_str() { - return invalid("provider_id is immutable for an existing binding"); - } - } - let actual_u64 = actual.map(|value| value as u64); - if actual_u64 != request.expected_revision { - return Err(EnvironmentRegistryError::RevisionConflict { - kind: "environment_provider_binding", - id: request.binding_id.to_string(), - expected: request.expected_revision, - actual: actual_u64, - }); + .map_err(|error| sql_error("begin provider binding put", error))?; + let provider: Option = sqlx::query_scalar( + "SELECT provider_id FROM environment_providers WHERE provider_id = $1 FOR SHARE", + ) + .bind(request.provider_id.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|error| sql_error("lock binding provider", error))?; + if provider.is_none() { + return Err(not_found("environment_provider", &request.provider_id)); } - let revision = actual.unwrap_or(0) + 1; + let query = format!( + "SELECT {BINDING_COLUMNS} FROM environment_provider_bindings WHERE universe_id = $1 AND provider_id = $2 FOR UPDATE" + ); + let existing = sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(request.universe_id) + .bind(request.provider_id.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|error| sql_error("lock provider binding", error))? + .and_then(|row| decoded_or_logged(binding_from_row(&row), "provider binding")); + let record = request.into_record(existing.as_ref())?; let query = format!( r#" INSERT INTO environment_provider_bindings ( - universe_id, binding_id, provider_id, status, revision, - metadata_json, created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$7) - ON CONFLICT (universe_id, binding_id) DO UPDATE SET - status = EXCLUDED.status, - metadata_json = EXCLUDED.metadata_json, - revision = EXCLUDED.revision, updated_at_ms = EXCLUDED.updated_at_ms + universe_id, provider_id, enabled, created_at_ms, updated_at_ms + ) VALUES ($1,$2,$3,$4,$5) + ON CONFLICT (universe_id, provider_id) DO UPDATE SET + enabled = EXCLUDED.enabled, + created_at_ms = EXCLUDED.created_at_ms, + updated_at_ms = EXCLUDED.updated_at_ms RETURNING {BINDING_COLUMNS} - "# + "# ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) - .bind(request.universe_id) - .bind(request.binding_id.as_str()) - .bind(request.provider_id.as_str()) - .bind(binding_status_to_str(request.status)) - .bind(revision) - .bind(json_value( - "encode provider binding metadata", - &request.metadata, - )?) - .bind(request.updated_at_ms) + .bind(record.universe_id) + .bind(record.provider_id.as_str()) + .bind(record.enabled) + .bind(record.created_at_ms) + .bind(record.updated_at_ms) .fetch_one(&mut *tx) .await - .map_err(|error| map_binding_write_error("put provider binding", error))?; + .map_err(|error| map_binding_write_error(error, record.universe_id))?; tx.commit() .await .map_err(|error| sql_error("commit provider binding", error))?; @@ -237,18 +361,18 @@ impl EnvironmentProviderBindingStore for PgStore { async fn read_provider_binding( &self, universe_id: Uuid, - binding_id: &EnvironmentProviderBindingId, + provider_id: &EnvironmentProviderId, ) -> Result { let query = format!( - "SELECT {BINDING_COLUMNS} FROM environment_provider_bindings WHERE universe_id = $1 AND binding_id = $2" + "SELECT {BINDING_COLUMNS} FROM environment_provider_bindings WHERE universe_id = $1 AND provider_id = $2" ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(universe_id) - .bind(binding_id.as_str()) + .bind(provider_id.as_str()) .fetch_optional(&self.pool) .await .map_err(|error| sql_error("read provider binding", error))? - .ok_or_else(|| not_found("environment_provider_binding", binding_id))?; + .ok_or_else(|| not_found("environment_provider_binding", provider_id))?; binding_from_row(&row) } @@ -257,380 +381,259 @@ impl EnvironmentProviderBindingStore for PgStore { universe_id: Uuid, ) -> Result, EnvironmentRegistryError> { let query = format!( - "SELECT {BINDING_COLUMNS} FROM environment_provider_bindings WHERE universe_id = $1 ORDER BY binding_id" + "SELECT {BINDING_COLUMNS} FROM environment_provider_bindings WHERE universe_id = $1 ORDER BY provider_id" ); let rows = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(universe_id) .fetch_all(&self.pool) .await .map_err(|error| sql_error("list provider bindings", error))?; - rows.iter().map(binding_from_row).collect() + Ok(decode_rows( + &rows, + binding_from_row, + "environment provider binding", + )) } async fn delete_provider_binding( &self, universe_id: Uuid, - binding_id: &EnvironmentProviderBindingId, + provider_id: &EnvironmentProviderId, ) -> Result { + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin provider binding delete", error))?; + let query = format!( + "SELECT {BINDING_COLUMNS} FROM environment_provider_bindings WHERE universe_id = $1 AND provider_id = $2 FOR UPDATE" + ); + sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(universe_id) + .bind(provider_id.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|error| sql_error("lock provider binding", error))? + .ok_or_else(|| not_found("environment_provider_binding", provider_id))?; let active: i64 = sqlx::query_scalar( - "SELECT count(*) FROM environments WHERE universe_id = $1 AND binding_id = $2 AND status <> 'closed'" - ).bind(universe_id).bind(binding_id.as_str()).fetch_one(&self.pool).await - .map_err(|error| sql_error("count binding environments", error))?; + "SELECT count(*) FROM environments WHERE universe_id = $1 AND provider_id = $2 AND state <> 'closed'", + ) + .bind(universe_id) + .bind(provider_id.as_str()) + .fetch_one(&mut *tx) + .await + .map_err(|error| sql_error("count binding environments", error))?; if active != 0 { return invalid("provider binding is referenced by a non-closed environment"); } let query = format!( - "DELETE FROM environment_provider_bindings WHERE universe_id = $1 AND binding_id = $2 RETURNING {BINDING_COLUMNS}" + "DELETE FROM environment_provider_bindings WHERE universe_id = $1 AND provider_id = $2 RETURNING {BINDING_COLUMNS}" ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(universe_id) - .bind(binding_id.as_str()) - .fetch_optional(&self.pool) + .bind(provider_id.as_str()) + .fetch_one(&mut *tx) + .await + .map_err(|error| sql_error("delete provider binding", error))?; + tx.commit() .await - .map_err(|error| sql_error("delete provider binding", error))? - .ok_or_else(|| not_found("environment_provider_binding", binding_id))?; + .map_err(|error| sql_error("commit provider binding delete", error))?; binding_from_row(&row) } } -#[async_trait] -impl EnvironmentStore for PgStore { - async fn create_environment( +impl PgStore { + /// Apply one write to one environment: the row is locked, `change` runs + /// on a copy, and the copy is validated and written only when `change` + /// reports a change. Returns the current record and whether it changed. + async fn change_environment( &self, - request: CreateEnvironment, - ) -> Result { - self.ensure_universe() - .await - .map_err(|error| store_error("ensure universe", error))?; - if request.created_at_ms < 0 { - return invalid("environment timestamp must be nonnegative"); - } + environment_id: &EnvironmentId, + action: &'static str, + change: F, + ) -> Result<(EnvironmentRecord, bool), EnvironmentRegistryError> + where + F: FnOnce(&mut EnvironmentRecord) -> Result + Send, + { let mut tx = self .pool .begin() .await - .map_err(|error| sql_error("begin environment create", error))?; - if let Some(existing_id) = sqlx::query_scalar::<_, String>( - "SELECT environment_id FROM environments WHERE universe_id = $1 AND request_id = $2", - ) - .bind(self.config.universe_id) - .bind(request.request_id.as_str()) - .fetch_optional(&mut *tx) - .await - .map_err(|error| sql_error("deduplicate environment create", error))? - { - tx.commit() - .await - .map_err(|error| sql_error("commit environment dedup", error))?; - let environment_id = EnvironmentId::try_new(existing_id) - .map_err(|error| store_message(format!("decode environment id: {error}")))?; - return self.read_environment(&environment_id).await; + .map_err(|error| sql_error(action, error))?; + let current = lock_environment(&mut tx, self.config.universe_id, environment_id).await?; + let mut candidate = current.clone(); + if !change(&mut candidate)? { + return Ok((current, false)); } + candidate.validate()?; + let columns = EnvironmentColumns::of(&candidate)?; + update_environment_row(&mut tx, self.config.universe_id, &candidate, columns) + .await + .map_err(|error| map_environment_write_error(action, error))?; + tx.commit() + .await + .map_err(|error| sql_error(action, error))?; + Ok((candidate, true)) + } + + async fn find_environment_by_request_id( + &self, + request_id: &EnvironmentProvisionRequestId, + ) -> Result, EnvironmentRegistryError> { let query = format!( - "SELECT {BINDING_COLUMNS} FROM environment_provider_bindings WHERE universe_id = $1 AND binding_id = $2 FOR UPDATE" + "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_FROM} WHERE e.universe_id = $1 AND e.request_id = $2" ); - let binding_row = sqlx::query(sqlx::AssertSqlSafe(query)) + let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(self.config.universe_id) - .bind(request.binding_id.as_str()) - .fetch_optional(&mut *tx) + .bind(request_id.as_str()) + .fetch_optional(&self.pool) .await - .map_err(|error| sql_error("lock provider binding", error))? - .ok_or_else(|| not_found("environment_provider_binding", &request.binding_id))?; - let binding = binding_from_row(&binding_row)?; - if binding.status != EnvironmentProviderBindingStatus::Enabled { - return invalid("environment provider binding is disabled"); + .map_err(|error| sql_error("read environment by request id", error))?; + row.as_ref().map(environment_from_row).transpose() + } + + /// The stored row a retried create with `request_id` converges on: + /// itself when `matches` holds, otherwise `AlreadyExists`. + async fn converge_create( + &self, + request_id: &EnvironmentProvisionRequestId, + matches: impl FnOnce(&EnvironmentRecord) -> bool, + ) -> Result, EnvironmentRegistryError> { + match self.find_environment_by_request_id(request_id).await? { + None => Ok(None), + Some(existing) if matches(&existing) => Ok(Some(existing)), + Some(_) => Err(EnvironmentRegistryError::AlreadyExists { + kind: "environment_request", + id: request_id.to_string(), + }), } - sqlx::query( - r#" - INSERT INTO environments ( - universe_id, environment_id, request_id, source_kind, provider_id, binding_id, - display_name, status, current_incarnation_id, metadata_json, - idle_policy_json, created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,'provisioned',$4,$5,$6,'provisioning',$7,$8,$9,$10,$10) - "#, - ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.request_id.as_str()) - .bind(binding.provider_id.as_str()) - .bind(request.binding_id.as_str()) - .bind(request.display_name.as_deref()) - .bind(request.incarnation_id.as_str()) - .bind(json_value( - "encode environment metadata", - &request.metadata, - )?) - .bind( - request - .idle_policy - .as_ref() - .map(|policy| json_value("encode idle policy", policy)) - .transpose()?, - ) - .bind(request.created_at_ms) - .execute(&mut *tx) - .await - .map_err(map_environment_insert_error)?; - sqlx::query( - r#" - INSERT INTO environment_incarnations ( - universe_id, environment_id, incarnation_id, provision_request_id, - template_id, created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,$4,$5,$6,$6) - "#, - ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.incarnation_id.as_str()) - .bind(request.request_id.as_str()) - .bind(request.template_id.as_str()) - .bind(request.created_at_ms) - .execute(&mut *tx) - .await - .map_err(|error| sql_error("insert environment incarnation", error))?; - tx.commit() + } + + /// Insert a provisioned or external row unless its request id is + /// already taken; a concurrent create with the same request id converges + /// like a retry. + async fn insert_requested_environment( + &self, + mut tx: Transaction<'_, Postgres>, + record: EnvironmentRecord, + matches: impl FnOnce(&EnvironmentRecord) -> bool, + ) -> Result { + let columns = EnvironmentColumns::of(&record)?; + let inserted = insert_environment_row(&mut tx, self.config.universe_id, &record, columns) .await - .map_err(|error| sql_error("commit environment create", error))?; - self.read_environment(&request.environment_id).await + .map_err(|error| map_environment_write_error("insert environment", error))?; + if inserted { + tx.commit() + .await + .map_err(|error| sql_error("commit environment create", error))?; + return Ok(record); + } + drop(tx); + let request_id = record + .request_id + .as_ref() + .ok_or_else(|| store_message("environment insert conflicted without a request id"))?; + self.converge_create(request_id, matches) + .await? + .ok_or_else(|| store_message("conflicting environment request is gone")) } +} - async fn adopt_environment( +#[async_trait] +impl EnvironmentStore for PgStore { + async fn create_environment( &self, - request: AdoptEnvironment, + request: CreateEnvironment, ) -> Result { + let record = request.record()?; self.ensure_universe() .await .map_err(|error| store_error("ensure universe", error))?; - if request.created_at_ms < 0 { - return invalid("environment timestamp must be nonnegative"); - } - if request.source_target.is_empty() - || request.source_target.len() > 255 - || request.source_target.chars().any(char::is_control) + if let Some(existing) = self + .converge_create(&request.request_id, |existing| request.matches(existing)) + .await? { - return invalid( - "adoption source must be non-empty, at most 255 bytes, and contain no control characters", - ); + return Ok(existing); } let mut tx = self .pool .begin() .await - .map_err(|error| sql_error("begin environment adoption", error))?; - if let Some(existing_id) = sqlx::query_scalar::<_, String>( - "SELECT environment_id FROM environments WHERE universe_id = $1 AND request_id = $2", + .map_err(|error| sql_error("begin environment create", error))?; + // Share-lock the binding so it cannot be deleted or disabled while + // the row that references it is inserted. + let enabled: Option = sqlx::query_scalar( + "SELECT enabled FROM environment_provider_bindings WHERE universe_id = $1 AND provider_id = $2 FOR SHARE", ) .bind(self.config.universe_id) - .bind(request.request_id.as_str()) + .bind(request.provider_id.as_str()) .fetch_optional(&mut *tx) .await - .map_err(|error| sql_error("deduplicate environment adoption", error))? - { - tx.commit() - .await - .map_err(|error| sql_error("commit environment adoption dedup", error))?; - let environment_id = EnvironmentId::try_new(existing_id) - .map_err(|error| store_message(format!("decode environment id: {error}")))?; - return self.read_environment(&environment_id).await; - } - let query = format!( - "SELECT {BINDING_COLUMNS} FROM environment_provider_bindings WHERE universe_id = $1 AND binding_id = $2 FOR UPDATE" - ); - let binding_row = sqlx::query(sqlx::AssertSqlSafe(query)) - .bind(self.config.universe_id) - .bind(request.binding_id.as_str()) - .fetch_optional(&mut *tx) - .await - .map_err(|error| sql_error("lock provider binding for adoption", error))? - .ok_or_else(|| not_found("environment_provider_binding", &request.binding_id))?; - let binding = binding_from_row(&binding_row)?; - if binding.status != EnvironmentProviderBindingStatus::Enabled { - return invalid("environment provider binding is disabled"); + .map_err(|error| sql_error("lock provider binding", error))?; + match enabled { + None => { + return Err(not_found( + "environment_provider_binding", + &request.provider_id, + )); + } + Some(false) => return invalid("environment provider binding is disabled"), + Some(true) => {} } - sqlx::query( - r#" - INSERT INTO environments ( - universe_id, environment_id, request_id, source_kind, provider_id, binding_id, - display_name, status, current_incarnation_id, metadata_json, - created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,'provisioned',$4,$5,$6,'provisioning',$7,$8,$9,$9) - "#, - ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.request_id.as_str()) - .bind(binding.provider_id.as_str()) - .bind(request.binding_id.as_str()) - .bind(request.display_name.as_deref()) - .bind(request.incarnation_id.as_str()) - .bind(json_value( - "encode environment metadata", - &request.metadata, - )?) - .bind(request.created_at_ms) - .execute(&mut *tx) - .await - .map_err(map_environment_insert_error)?; - sqlx::query( - r#" - INSERT INTO environment_incarnations ( - universe_id, environment_id, incarnation_id, provision_request_id, - adoption_source_target, created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,$4,$5,$6,$6) - "#, - ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.incarnation_id.as_str()) - .bind(request.request_id.as_str()) - .bind(&request.source_target) - .bind(request.created_at_ms) - .execute(&mut *tx) - .await - .map_err(|error| sql_error("insert adopted environment incarnation", error))?; - tx.commit() + self.insert_requested_environment(tx, record, |existing| request.matches(existing)) .await - .map_err(|error| sql_error("commit environment adoption", error))?; - self.read_environment(&request.environment_id).await } async fn create_external_environment( &self, request: CreateExternalEnvironment, ) -> Result { + let record = request.record()?; self.ensure_universe() .await .map_err(|error| store_error("ensure universe", error))?; - if request.created_at_ms < 0 { - return invalid("environment timestamp must be nonnegative"); + if let Some(existing) = self + .converge_create(&request.request_id, |existing| request.matches(existing)) + .await? + { + return Ok(existing); } - let candidate = EnvironmentRecord { - environment_id: request.environment_id.clone(), - request_id: request.request_id.clone(), - source: EnvironmentSource::External { - connection: request.connection.clone(), - }, - display_name: request.display_name.clone(), - status: EnvironmentStatus::Ready, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: request.incarnation_id.clone(), - provision_request_id: None, - provider_target_id: None, - template_id: None, - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: request.metadata.clone(), - last_seen_at_ms: None, - created_at_ms: request.created_at_ms, - updated_at_ms: request.created_at_ms, - }; - candidate.validate()?; - let mut tx = self + let tx = self .pool .begin() .await .map_err(|error| sql_error("begin external environment create", error))?; - if let Some(existing_id) = sqlx::query_scalar::<_, String>( - "SELECT environment_id FROM environments WHERE universe_id = $1 AND request_id = $2", - ) - .bind(self.config.universe_id) - .bind(request.request_id.as_str()) - .fetch_optional(&mut *tx) - .await - .map_err(|error| sql_error("deduplicate external environment create", error))? - { - tx.commit() - .await - .map_err(|error| sql_error("commit external environment dedup", error))?; - let environment_id = EnvironmentId::try_new(existing_id) - .map_err(|error| store_message(format!("decode environment id: {error}")))?; - return self.read_environment(&environment_id).await; - } - sqlx::query( - r#" - INSERT INTO environments ( - universe_id, environment_id, request_id, source_kind, daemon_connection_json, - display_name, status, current_incarnation_id, metadata_json, - created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,'external',$4,$5,'ready',$6,$7,$8,$8) - "#, - ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.request_id.as_str()) - .bind(json_value("encode daemon connection", &request.connection)?) - .bind(request.display_name.as_deref()) - .bind(request.incarnation_id.as_str()) - .bind(json_value( - "encode external environment metadata", - &request.metadata, - )?) - .bind(request.created_at_ms) - .execute(&mut *tx) - .await - .map_err(map_environment_insert_error)?; - sqlx::query( - r#" - INSERT INTO environment_incarnations ( - universe_id, environment_id, incarnation_id, created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,$4,$4) - "#, - ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.incarnation_id.as_str()) - .bind(request.created_at_ms) - .execute(&mut *tx) - .await - .map_err(|error| sql_error("insert external environment incarnation", error))?; - tx.commit() + self.insert_requested_environment(tx, record, |existing| request.matches(existing)) .await - .map_err(|error| sql_error("commit external environment create", error))?; - self.read_environment(&request.environment_id).await } async fn create_registered_environment( &self, request: CreateRegisteredEnvironment, ) -> Result { + if request.created_at_ms < 0 { + return invalid("created_at_ms must be nonnegative"); + } + validate_daemon_public_key(&request.daemon_public_key)?; self.ensure_universe() .await .map_err(|error| store_error("ensure universe", error))?; - if request.created_at_ms < 0 { - return invalid("environment timestamp must be nonnegative"); + // A known identity converges on its row, closed included: the + // identity is spent, and the caller refuses a closed one. + if let Some(existing) = self + .read_environment_by_daemon_public_key(&request.daemon_public_key) + .await? + { + return Ok(existing); } - let daemon_id = request.daemon_id()?; - let request_id = EnvironmentProvisionRequestId::for_daemon(&daemon_id); let mut tx = self .pool .begin() .await .map_err(|error| sql_error("begin registered environment create", error))?; - if let Some(existing_id) = sqlx::query_scalar::<_, String>( - "SELECT environment_id FROM environments WHERE universe_id = $1 AND request_id = $2", - ) - .bind(self.config.universe_id) - .bind(request_id.as_str()) - .fetch_optional(&mut *tx) - .await - .map_err(|error| sql_error("deduplicate registered environment create", error))? - { - tx.commit() - .await - .map_err(|error| sql_error("commit registered environment dedup", error))?; - let environment_id = EnvironmentId::try_new(existing_id) - .map_err(|error| store_message(format!("decode environment id: {error}")))?; - return self.read_environment(&environment_id).await; - } // Lock the key row: concurrent registrations against one key - // serialize here, so the count below is exact. + // serialize here, so the count below is exact. Rotation takes the + // same lock, so the secret check below sees every committed one. let key = crate::environment_registration::lock_registration_key( &mut tx, self.config.universe_id, @@ -638,9 +641,27 @@ impl EnvironmentStore for PgStore { ) .await?; key.check_admits(request.created_at_ms)?; + let live: bool = sqlx::query_scalar( + r#" + SELECT EXISTS ( + SELECT 1 FROM environment_registration_key_secrets + WHERE universe_id = $1 AND registration_key_id = $2 + AND secret_hash = $3 AND retired_at_ms IS NULL + ) + "#, + ) + .bind(self.config.universe_id) + .bind(key.registration_key_id.as_str()) + .bind(&request.secret_hash) + .fetch_one(&mut *tx) + .await + .map_err(|error| sql_error("check registration key secret", error))?; + if !live { + return Err(request.secret_retired()); + } if let Some(limit) = key.max_active_environments { let active: i64 = sqlx::query_scalar( - "SELECT count(*) FROM environments WHERE universe_id = $1 AND registration_key_id = $2 AND status <> 'closed'", + "SELECT count(*) FROM environments WHERE universe_id = $1 AND registration_key_id = $2 AND state <> 'closed'", ) .bind(self.config.universe_id) .bind(key.registration_key_id.as_str()) @@ -654,51 +675,52 @@ impl EnvironmentStore for PgStore { }); } } - sqlx::query( - r#" - INSERT INTO environments ( - universe_id, environment_id, request_id, source_kind, - registration_key_id, daemon_id, daemon_public_key, identity_mode, last_seen_at_ms, - display_name, status, current_incarnation_id, metadata_json, - created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,'registered',$4,$5,$6,$7,$8,$9,'ready',$10,$11,$8,$8) - "#, - ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request_id.as_str()) - .bind(key.registration_key_id.as_str()) - .bind(daemon_id.as_str()) - .bind(&request.daemon_public_key) - .bind(key.identity_mode.as_str()) - .bind(request.created_at_ms) - .bind(request.display_name.as_deref()) - .bind(request.incarnation_id.as_str()) - .bind(json_value( - "encode registered environment metadata", - &request.metadata, - )?) - .execute(&mut *tx) - .await - .map_err(|error| map_registered_insert_error(error, &daemon_id))?; - sqlx::query( - r#" - INSERT INTO environment_incarnations ( - universe_id, environment_id, incarnation_id, created_at_ms, updated_at_ms - ) VALUES ($1,$2,$3,$4,$4) - "#, - ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.incarnation_id.as_str()) - .bind(request.created_at_ms) - .execute(&mut *tx) - .await - .map_err(|error| sql_error("insert registered environment incarnation", error))?; - tx.commit() + let record = request.record(&key)?; + let columns = EnvironmentColumns::of(&record)?; + match insert_environment_row(&mut tx, self.config.universe_id, &record, columns).await { + Ok(_) => { + tx.commit() + .await + .map_err(|error| sql_error("commit registered environment create", error))?; + Ok(record) + } + Err(error) if constraint_of(&error) == Some("environments_daemon_public_key_idx") => { + drop(tx); + // A concurrent admission of the same identity in this + // universe won; elsewhere the identity belongs to another + // universe. + self.read_environment_by_daemon_public_key(&request.daemon_public_key) + .await? + .ok_or_else(|| EnvironmentRegistryError::AlreadyExists { + kind: "daemon_identity", + id: request + .daemon_id() + .map(|daemon_id| daemon_id.to_string()) + .unwrap_or_else(|_| request.daemon_public_key.clone()), + }) + } + Err(error) => Err(map_environment_write_error( + "insert registered environment", + error, + )), + } + } + + async fn read_environment( + &self, + environment_id: &EnvironmentId, + ) -> Result { + let query = format!( + "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_FROM} WHERE e.universe_id = $1 AND e.environment_id = $2" + ); + let row = sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(self.config.universe_id) + .bind(environment_id.as_str()) + .fetch_optional(&self.pool) .await - .map_err(|error| sql_error("commit registered environment create", error))?; - self.read_environment(&request.environment_id).await + .map_err(|error| sql_error("read environment", error))? + .ok_or_else(|| not_found("environment", environment_id))?; + environment_from_row(&row) } async fn read_environment_by_daemon_public_key( @@ -706,7 +728,7 @@ impl EnvironmentStore for PgStore { daemon_public_key: &str, ) -> Result, EnvironmentRegistryError> { let query = format!( - "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1 AND e.daemon_public_key = $2" + "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_FROM} WHERE e.universe_id = $1 AND e.daemon_public_key = $2" ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(self.config.universe_id) @@ -717,341 +739,487 @@ impl EnvironmentStore for PgStore { row.as_ref().map(environment_from_row).transpose() } - async fn observe_registered_environment( + async fn list_environments( &self, - request: ObserveRegisteredEnvironment, - ) -> Result { - if request.observed_at_ms < 0 { - return invalid("observed_at_ms must be nonnegative"); + request: ListEnvironments, + ) -> Result { + request.validate()?; + let mut query = + format!("SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_FROM} WHERE e.universe_id = $1"); + let mut next = 2; + let mut parameter = |query: &mut String, predicate: &str| { + query.push_str(&format!(" AND {predicate}${next}")); + next += 1; + }; + if request.provider_id.is_some() { + parameter(&mut query, "e.provider_id = "); + } + if request.registration_key_id.is_some() { + parameter(&mut query, "e.registration_key_id = "); } - let current = self.read_environment(&request.environment_id).await?; - if !current.is_registered() { - return invalid("connection observations apply only to registered environments"); + if request.state.is_some() { + parameter(&mut query, "e.state = "); } - // One conditional statement rather than read-modify-write: a close - // that lands between a read and a write must never be overwritten - // by a late connect or disconnect observation. - let observation = match request.observation { - RegisteredConnectionObservation::Connected => "connected", - RegisteredConnectionObservation::Heartbeat => "heartbeat", - RegisteredConnectionObservation::Disconnected => "disconnected", + // Appended only when present so containment can use the GIN index. + if !request.metadata.is_empty() { + parameter(&mut query, "e.metadata_json @> "); + } + if request.after.is_some() { + parameter(&mut query, "e.environment_id > "); + } + if !request.include_closed && request.state != Some(EnvironmentState::Closed) { + query.push_str(" AND e.state <> 'closed'"); + } + query.push_str(&format!(" ORDER BY e.environment_id LIMIT ${next}")); + let mut sql = sqlx::query(sqlx::AssertSqlSafe(query)).bind(self.config.universe_id); + if let Some(id) = &request.provider_id { + sql = sql.bind(id.as_str().to_owned()); + } + if let Some(id) = &request.registration_key_id { + sql = sql.bind(id.as_str().to_owned()); + } + if let Some(state) = request.state { + sql = sql.bind(state.as_str()); + } + if !request.metadata.is_empty() { + sql = sql.bind(metadata_filter_json(&request.metadata)); + } + if let Some(after) = &request.after { + sql = sql.bind(after.as_str().to_owned()); + } + let limit = request.limit as usize; + // One row past the page tells whether another page follows. + let mut rows = sql + .bind(i64::from(request.limit) + 1) + .fetch_all(&self.pool) + .await + .map_err(|error| sql_error("list environments", error))?; + let next_after = if rows.len() > limit { + rows.truncate(limit); + rows.last() + .map(|row| parse_id(row, "environment_id", EnvironmentId::try_new)) + .transpose()? + } else { + None }; - sqlx::query( - r#" - UPDATE environments SET - status = CASE - WHEN status IN ('closing', 'closed') THEN status - WHEN $3 = 'connected' THEN 'ready' - WHEN $3 = 'disconnected' THEN 'offline' - ELSE status END, - last_seen_at_ms = CASE - WHEN status IN ('closing', 'closed') THEN last_seen_at_ms - WHEN $3 IN ('connected', 'heartbeat') THEN $4 - ELSE last_seen_at_ms END, - metadata_json = CASE - WHEN status IN ('closing', 'closed') THEN metadata_json - ELSE metadata_json || $5::jsonb END, - updated_at_ms = CASE - WHEN status IN ('closing', 'closed') THEN updated_at_ms - ELSE GREATEST(updated_at_ms, $4) END - WHERE universe_id = $1 AND environment_id = $2 AND source_kind = 'registered' - "#, + Ok(EnvironmentPage { + environments: decode_rows(&rows, environment_from_row, "environment"), + next_after, + }) + } + + async fn update_environment( + &self, + request: UpdateEnvironment, + ) -> Result { + self.change_environment(&request.environment_id, "update environment", |record| { + request.apply(record) + }) + .await + .map(|(record, _)| record) + } + + async fn begin_close_environment( + &self, + request: BeginCloseEnvironment, + ) -> Result { + self.change_environment( + &request.environment_id, + "begin environment close", + |record| request.apply(record), ) - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(observation) - .bind(request.observed_at_ms) - .bind(json_value( - "encode daemon build metadata", - &request.metadata, - )?) - .execute(&self.pool) .await - .map_err(|error| sql_error("observe registered environment", error))?; - self.read_environment(&request.environment_id).await + .map(|(record, _)| record) } - async fn list_open_registered_environments( + async fn finish_close_environment( &self, - ) -> Result, EnvironmentRegistryError> { - let query = format!( - "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1 AND e.source_kind = 'registered' AND e.status NOT IN ('closing','closed') ORDER BY e.environment_id" - ); - let rows = sqlx::query(sqlx::AssertSqlSafe(query)) - .bind(self.config.universe_id) - .fetch_all(&self.pool) - .await - .map_err(|error| sql_error("list open registered environments", error))?; - rows.iter().map(environment_from_row).collect() + request: FinishCloseEnvironment, + ) -> Result { + self.change_environment( + &request.environment_id, + "finish environment close", + |record| request.apply(record), + ) + .await + .map(|(record, _)| record) } - async fn read_environment( + async fn mark_registered_connected( &self, - environment_id: &EnvironmentId, + request: MarkRegisteredConnected, ) -> Result { - read_environment_on( - &self.pool, - self.config.universe_id, - "e.environment_id = $2", - environment_id.as_str(), - "read environment", + self.change_environment( + &request.environment_id, + "mark registered environment connected", + |record| request.apply(record), ) .await + .map(|(record, _)| record) } - async fn read_environment_by_request_id( + async fn mark_registered_disconnected( &self, - request_id: &EnvironmentProvisionRequestId, + request: MarkRegisteredDisconnected, ) -> Result { - read_environment_on( - &self.pool, - self.config.universe_id, - "e.request_id = $2", - request_id.as_str(), - "read environment request", + self.change_environment( + &request.environment_id, + "mark registered environment disconnected", + |record| request.apply(record), ) .await + .map(|(record, _)| record) } - async fn list_environments( + async fn stamp_registered_seen( &self, - request: ListEnvironments, - ) -> Result, EnvironmentRegistryError> { - let mut query = - format!("SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1"); - let mut next = 2; - if request.provider_id.is_some() { - query.push_str(&format!(" AND e.provider_id = ${next}")); - next += 1; - } - if request.binding_id.is_some() { - query.push_str(&format!(" AND e.binding_id = ${next}")); - next += 1; - } - if request.status.is_some() { - query.push_str(&format!(" AND e.status = ${next}")); - next += 1; - } - if request.registration_key_id.is_some() { - query.push_str(&format!(" AND e.registration_key_id = ${next}")); - next += 1; - } - // Appended only when present so containment can use the GIN index. - if !request.metadata.is_empty() { - query.push_str(&format!(" AND e.metadata_json @> ${next}")); - } - query.push_str(" ORDER BY e.environment_id"); - let mut sql = sqlx::query(sqlx::AssertSqlSafe(query)).bind(self.config.universe_id); - if let Some(id) = request.provider_id { - sql = sql.bind(id.to_string()); - } - if let Some(id) = request.binding_id { - sql = sql.bind(id.to_string()); + request: StampRegisteredSeen, + ) -> Result { + if request.at_ms < 0 { + return invalid("at_ms must be nonnegative"); } - if let Some(status) = request.status { - sql = sql.bind(environment_status_to_str(status)); + // One round trip: the stamp is conditional, and the row's kind + // tells a skipped stamp from a missing or non-registered row. + let row = sqlx::query( + r#" + WITH target AS ( + SELECT source_kind FROM environments + WHERE universe_id = $1 AND environment_id = $2 + ), stamped AS ( + UPDATE environments SET + last_seen_at_ms = $3, + updated_at_ms = GREATEST(updated_at_ms, $3) + WHERE universe_id = $1 AND environment_id = $2 + AND source_kind = 'registered' AND state = 'open' AND connected + AND last_seen_at_ms <= $4 + RETURNING 1 + ) + SELECT (SELECT source_kind FROM target) AS source_kind, + EXISTS (SELECT 1 FROM stamped) AS stamped + "#, + ) + .bind(self.config.universe_id) + .bind(request.environment_id.as_str()) + .bind(request.at_ms) + .bind(request.at_ms.saturating_sub(REGISTERED_LIVENESS_STAMP_MS)) + .fetch_one(&self.pool) + .await + .map_err(|error| sql_error("stamp registered environment", error))?; + let source_kind: Option = row + .try_get("source_kind") + .map_err(|error| sql_error("decode stamped source kind", error))?; + match source_kind.as_deref() { + None => Err(not_found("environment", &request.environment_id)), + Some("registered") => row + .try_get("stamped") + .map_err(|error| sql_error("decode stamp result", error)), + Some(_) => invalid("connection writes apply only to registered environments"), } - if let Some(id) = request.registration_key_id { - sql = sql.bind(id.to_string()); + } + + async fn sweep_registered_environments( + &self, + now_ms: i64, + ) -> Result { + // Only rows one of the three sweep rules can move; the rules + // themselves run in the domain, under the row lock. A connected row + // can close only after rule 1 marks it disconnected, so rule 1 + // covers it. + let candidates: Vec<(String, String)> = sqlx::query_as( + r#" + SELECT e.environment_id, e.registration_key_id + FROM environments e + JOIN environment_registration_keys k + ON k.universe_id = e.universe_id AND k.registration_key_id = e.registration_key_id + WHERE e.universe_id = $1 AND e.source_kind = 'registered' AND e.state = 'open' + AND ( + (e.connected AND e.last_seen_at_ms < $2 - $3) + OR ( + NOT e.connected AND k.identity_mode = 'ephemeral' + AND e.connection_changed_at_ms < $2 - COALESCE(k.ephemeral_disconnect_grace_ms, $4) + ) + OR ( + NOT e.connected AND k.identity_mode = 'persistent' + AND k.offline_close_after_ms IS NOT NULL + AND e.connection_changed_at_ms < $2 - k.offline_close_after_ms + ) + ) + ORDER BY e.environment_id + "#, + ) + .bind(self.config.universe_id) + .bind(now_ms) + .bind(REGISTERED_STALE_AFTER_MS) + .bind(i64::try_from(DEFAULT_EPHEMERAL_DISCONNECT_GRACE_MS).unwrap_or(i64::MAX)) + .fetch_all(&self.pool) + .await + .map_err(|error| sql_error("list registered sweep candidates", error))?; + let mut keys: HashMap = HashMap::new(); + let mut sweep = RegisteredSweep::default(); + for (environment_id, registration_key_id) in candidates { + let key = match keys.get(®istration_key_id) { + Some(key) => key.clone(), + None => { + let id = EnvironmentRegistrationKeyId::try_new(registration_key_id.clone()) + .map_err(|error| { + store_message(format!("decode registration key id: {error}")) + })?; + let key = self.read_registration_key(&id).await?; + keys.insert(registration_key_id, key.clone()); + key + } + }; + let environment_id = EnvironmentId::try_new(environment_id) + .map_err(|error| store_message(format!("decode environment id: {error}")))?; + let mut outcome = (false, None); + let changed = self + .change_environment(&environment_id, "sweep registered environment", |record| { + outcome = sweep_registered_environment(record, &key, now_ms); + Ok(outcome.0 || outcome.1.is_some()) + }) + .await; + if let Err(error) = changed { + tracing::error!(%error, environment_id = %environment_id, "registered sweep skipped a row"); + continue; + } + if outcome.0 { + sweep.disconnected.push(environment_id.clone()); + } + if let Some(reason) = outcome.1 { + sweep.closed.push((environment_id, reason)); + } } - if !request.metadata.is_empty() { - sql = sql.bind(metadata_filter_json(&request.metadata)); + Ok(sweep) + } + + async fn repair_registered_connections( + &self, + now_ms: i64, + ) -> Result { + if now_ms < 0 { + return invalid("now_ms must be nonnegative"); } - let rows = sql - .fetch_all(&self.pool) + repair_connections(&self.pool, Some(self.config.universe_id), now_ms) .await - .map_err(|error| sql_error("list environments", error))?; - rows.iter().map(environment_from_row).collect() + .map_err(|error| sql_error("repair registered connections", error)) } - async fn list_environments_needing_reconcile( + async fn list_environment_work( &self, + now_ms: i64, + limit: u32, ) -> Result, EnvironmentRegistryError> { let query = format!( - "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1 AND (e.status IN ('provisioning','booting','closing','unknown') OR {POWER_DIVERGES_SQL}) ORDER BY e.updated_at_ms, e.environment_id" + "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_FROM} WHERE e.universe_id = $1 AND e.next_attempt_at_ms <= $2 ORDER BY e.next_attempt_at_ms, e.environment_id LIMIT $3" ); let rows = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(self.config.universe_id) + .bind(now_ms) + .bind(i64::from(limit)) .fetch_all(&self.pool) .await - .map_err(|error| sql_error("list environments needing reconcile", error))?; - rows.iter().map(environment_from_row).collect() + .map_err(|error| sql_error("list environment work", error))?; + Ok(decode_rows(&rows, environment_from_row, "environment")) } - async fn list_environments_with_idle_policy( + async fn list_open_provisioned_environments( &self, ) -> Result, EnvironmentRegistryError> { let query = format!( - "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1 AND e.status = 'ready' AND e.idle_policy_json IS NOT NULL ORDER BY e.updated_at_ms, e.environment_id" + "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_FROM} WHERE e.universe_id = $1 AND e.source_kind = 'provisioned' AND e.state = 'open' ORDER BY e.environment_id" ); let rows = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(self.config.universe_id) .fetch_all(&self.pool) .await - .map_err(|error| sql_error("list environments with idle policy", error))?; - rows.iter().map(environment_from_row).collect() + .map_err(|error| sql_error("list open provisioned environments", error))?; + Ok(decode_rows(&rows, environment_from_row, "environment")) } - async fn observe_provisioned_environment( + async fn record_target_observation( &self, - request: ObserveProvisionedEnvironment, + request: RecordTargetObservation, ) -> Result { - let current = self.read_environment(&request.environment_id).await?; - if request.observed_at_ms < current.incarnation.updated_at_ms { - return Ok(current); - } - if current - .incarnation - .provider_target_id - .as_ref() - .is_some_and(|id| id != &request.provider_target_id) - { - return invalid("provider target conflicts with current incarnation"); - } - sqlx::query("UPDATE environment_incarnations SET provider_target_id=$4, power_states_json=$5, updated_at_ms=$6 WHERE universe_id=$1 AND environment_id=$2 AND incarnation_id=$3") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(current.incarnation.incarnation_id.as_str()) - .bind(request.provider_target_id.as_str()) - .bind(json_value("encode power states", &request.power_states)?) - .bind(request.observed_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("observe environment incarnation", error))?; - sqlx::query("UPDATE environments SET status=$3, updated_at_ms=$4 WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(environment_status_to_str(request.status)).bind(request.observed_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("observe environment", error))?; - self.read_environment(&request.environment_id).await + self.change_environment( + &request.environment_id, + "record target observation", + |record| request.apply(record), + ) + .await + .map(|(record, _)| record) } - async fn fail_environment_lifecycle( + async fn schedule_environment_attempt( &self, - request: FailEnvironmentLifecycle, + request: ScheduleEnvironmentAttempt, ) -> Result { - let mut metadata = self - .read_environment(&request.environment_id) - .await? - .metadata; - metadata.insert("lifecycleError".to_owned(), request.message); - sqlx::query("UPDATE environments SET status='failed', metadata_json=$3, updated_at_ms=$4 WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(json_value("encode environment metadata", &metadata)?).bind(request.observed_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("fail environment", error))?; - self.read_environment(&request.environment_id).await + self.change_environment( + &request.environment_id, + "schedule environment attempt", + |record| request.apply(record), + ) + .await + .map(|(record, _)| record) } - async fn begin_close_environment( + async fn open_environment( &self, - request: BeginCloseEnvironment, + request: OpenEnvironment, ) -> Result { - sqlx::query("UPDATE environments SET status=CASE WHEN status='closed' THEN status ELSE 'closing' END, updated_at_ms=GREATEST(updated_at_ms,$3) WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(request.updated_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("begin environment close", error))?; - self.read_environment(&request.environment_id).await + self.change_environment(&request.environment_id, "open environment", |record| { + request.apply(record) + }) + .await + .map(|(record, _)| record) } - async fn finish_close_environment( + async fn fail_environment( &self, - request: FinishCloseEnvironment, + request: FailEnvironment, ) -> Result { - let current = self.read_environment(&request.environment_id).await?; - let mut tx = self - .pool - .begin() - .await - .map_err(|error| sql_error("begin finish close", error))?; - sqlx::query("UPDATE environment_incarnations SET updated_at_ms=$4 WHERE universe_id=$1 AND environment_id=$2 AND incarnation_id=$3") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(current.incarnation.incarnation_id.as_str()).bind(request.observed_at_ms) - .execute(&mut *tx).await.map_err(|error| sql_error("close environment incarnation", error))?; - sqlx::query("UPDATE environments SET status='closed', public_ingress_enabled=false, public_endpoint=NULL, updated_at_ms=$3 WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id).bind(request.environment_id.as_str()).bind(request.observed_at_ms) - .execute(&mut *tx).await.map_err(|error| sql_error("finish environment close", error))?; - tx.commit() - .await - .map_err(|error| sql_error("commit environment close", error))?; - self.read_environment(&request.environment_id).await + self.change_environment(&request.environment_id, "fail environment", |record| { + request.apply(record) + }) + .await + .map(|(record, _)| record) } - async fn set_environment_ingress( + async fn end_provisioned_environment( &self, - request: SetEnvironmentIngress, + request: EndProvisionedEnvironment, ) -> Result { - validate_ingress_request_time(request.updated_at_ms)?; - let environment = self.read_environment(&request.environment_id).await?; - if !matches!(environment.source, EnvironmentSource::Provisioned { .. }) { - return invalid_store("provider-managed ingress requires a provisioned environment"); - } - if request.enabled - && matches!( - environment.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) - { - return invalid_store("cannot enable ingress for a closing environment"); - } - if request.enabled != request.public_endpoint.is_some() { - return invalid_store( - "enabled ingress requires a public endpoint and disabled ingress forbids one", - ); - } - sqlx::query("UPDATE environments SET public_ingress_enabled=$3, public_endpoint=$4, updated_at_ms=GREATEST(updated_at_ms,$5) WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.enabled) - .bind(request.public_endpoint) - .bind(request.updated_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("set environment ingress", error))?; - self.read_environment(&request.environment_id).await + self.change_environment( + &request.environment_id, + "end provisioned environment", + |record| request.apply(record), + ) + .await + .map(|(record, _)| record) } +} - async fn set_environment_power( - &self, - request: SetEnvironmentPower, - ) -> Result { - validate_ingress_request_time(request.updated_at_ms)?; - let environment = self.read_environment(&request.environment_id).await?; - check_power_mutable(&environment)?; - sqlx::query("UPDATE environments SET desired_power=$3, updated_at_ms=GREATEST(updated_at_ms,$4) WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind(request.desired_power.as_str()) - .bind(request.updated_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("set environment power", error))?; - self.read_environment(&request.environment_id).await - } +/// Connected registered rows become disconnected as of `now_ms`, in one +/// universe or in all of them. +async fn repair_connections( + pool: &PgPool, + universe_id: Option, + now_ms: i64, +) -> Result { + let result = sqlx::query( + r#" + UPDATE environments SET + connected = false, + connection_changed_at_ms = $2, + updated_at_ms = GREATEST(updated_at_ms, $2) + WHERE ($1::uuid IS NULL OR universe_id = $1) + AND source_kind = 'registered' AND connected + "#, + ) + .bind(universe_id) + .bind(now_ms) + .execute(pool) + .await?; + Ok(result.rows_affected()) +} - async fn set_environment_idle_policy( - &self, - request: SetEnvironmentIdlePolicy, - ) -> Result { - validate_ingress_request_time(request.updated_at_ms)?; - if let Some(policy) = &request.idle_policy { - policy.validate()?; - } - let environment = self.read_environment(&request.environment_id).await?; - check_power_mutable(&environment)?; - sqlx::query("UPDATE environments SET idle_policy_json=$3, updated_at_ms=GREATEST(updated_at_ms,$4) WHERE universe_id=$1 AND environment_id=$2") - .bind(self.config.universe_id) - .bind(request.environment_id.as_str()) - .bind( - request - .idle_policy - .as_ref() - .map(|policy| json_value("encode idle policy", policy)) - .transpose()?, - ) - .bind(request.updated_at_ms) - .execute(&self.pool).await.map_err(|error| sql_error("set environment idle policy", error))?; - self.read_environment(&request.environment_id).await - } +// --------------------------------------------------------------------------- +// Deployment-level functions: the environment-gateway role's loops run +// these across universes before they pick a universe-scoped store. +// --------------------------------------------------------------------------- + +/// Universes holding provisioned rows whose next attempt is due at `now_ms`. +pub async fn list_universes_with_environment_work( + pool: &PgPool, + now_ms: i64, +) -> Result, PgStoreError> { + list_universes( + pool, + "SELECT DISTINCT universe_id FROM environments WHERE next_attempt_at_ms <= $1 ORDER BY universe_id", + Some(now_ms), + ) + .await +} + +/// Universes holding open provisioned rows: the display sweep's scan set. +pub async fn list_universes_with_open_provisioned_environments( + pool: &PgPool, +) -> Result, PgStoreError> { + list_universes( + pool, + "SELECT DISTINCT universe_id FROM environments WHERE source_kind = 'provisioned' AND state = 'open' ORDER BY universe_id", + None, + ) + .await } -fn check_power_mutable(record: &EnvironmentRecord) -> Result<(), EnvironmentRegistryError> { - if !matches!(record.source, EnvironmentSource::Provisioned { .. }) { - return invalid_store("only provisioned environments have power control"); +/// Universes holding open registered rows (a registered row is only ever +/// `open` or `closed`): the registered sweep's scan set. +pub async fn list_universes_with_registered_environments( + pool: &PgPool, +) -> Result, PgStoreError> { + list_universes( + pool, + "SELECT DISTINCT universe_id FROM environments WHERE source_kind = 'registered' AND state = 'open' ORDER BY universe_id", + None, + ) + .await +} + +async fn list_universes( + pool: &PgPool, + query: &'static str, + now_ms: Option, +) -> Result, PgStoreError> { + let mut sql = sqlx::query_as::<_, (Uuid,)>(query); + if let Some(now_ms) = now_ms { + sql = sql.bind(now_ms); } - if matches!( - record.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed | EnvironmentStatus::Failed - ) { - return invalid_store("cannot change power of a closing, closed, or failed environment"); + let rows = sql.fetch_all(pool).await?; + Ok(rows.into_iter().map(|(universe_id,)| universe_id).collect()) +} + +/// At gateway startup, in every universe: connected registered rows become +/// disconnected with `connection_changed_at_ms = now_ms`, since the +/// connections the previous gateway held ended now, and persistent rows get +/// their full reconnect wait while their daemons reconnect. +/// `last_seen_at_ms` is kept. Assumes a single gateway replica. +pub async fn repair_registered_connections( + pool: &PgPool, + now_ms: i64, +) -> Result { + Ok(repair_connections(pool, None, now_ms).await?) +} + +/// Of `keys` (universe, environment id), those whose live routes must end: +/// the row is gone or not `open`, or it is provisioned and its universe's +/// binding of the provider is missing or disabled. +pub async fn list_revoked_environment_routes( + pool: &PgPool, + keys: &[(Uuid, String)], +) -> Result, PgStoreError> { + if keys.is_empty() { + return Ok(Vec::new()); } - Ok(()) + let (universes, environments): (Vec, Vec) = keys.iter().cloned().unzip(); + let rows: Vec<(Uuid, String)> = sqlx::query_as( + r#" + SELECT route.universe_id, route.environment_id + FROM unnest($1::uuid[], $2::text[]) AS route (universe_id, environment_id) + LEFT JOIN environments e + ON e.universe_id = route.universe_id AND e.environment_id = route.environment_id + LEFT JOIN environment_provider_bindings b + ON b.universe_id = e.universe_id AND b.provider_id = e.provider_id + WHERE e.environment_id IS NULL + OR e.state <> 'open' + OR (e.source_kind = 'provisioned' AND (b.provider_id IS NULL OR NOT b.enabled)) + ORDER BY route.universe_id, route.environment_id + "#, + ) + .bind(universes) + .bind(environments) + .fetch_all(pool) + .await?; + Ok(rows) } #[async_trait] @@ -1101,7 +1269,11 @@ impl EnvironmentCredentialStore for PgStore { .fetch_all(&self.pool) .await .map_err(|error| sql_error("list environment credentials", error))?; - rows.iter().map(credential_from_row).collect() + Ok(decode_rows( + &rows, + credential_from_row, + "environment credential", + )) } async fn unbind_credential( @@ -1129,39 +1301,272 @@ impl EnvironmentCredentialStore for PgStore { } } -async fn read_environment_on( - pool: &sqlx::PgPool, +// --------------------------------------------------------------------------- +// Rows +// --------------------------------------------------------------------------- + +async fn lock_environment( + tx: &mut Transaction<'_, Postgres>, universe_id: Uuid, - predicate: &'static str, - value: &str, - action: &str, + environment_id: &EnvironmentId, ) -> Result { let query = format!( - "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_JOIN} WHERE e.universe_id = $1 AND {predicate}" + "SELECT {ENVIRONMENT_COLUMNS} {ENVIRONMENT_FROM} WHERE e.universe_id = $1 AND e.environment_id = $2 FOR UPDATE OF e" ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(universe_id) - .bind(value) - .fetch_optional(pool) + .bind(environment_id.as_str()) + .fetch_optional(&mut **tx) .await - .map_err(|error| sql_error(action, error))? - .ok_or_else(|| not_found("environment", &value))?; + .map_err(|error| sql_error("lock environment", error))? + .ok_or_else(|| not_found("environment", environment_id))?; environment_from_row(&row) } -fn provider_from_row( - row: &sqlx::postgres::PgRow, -) -> Result { - let provider_id = parse_id(row, "provider_id", EnvironmentProviderId::try_new)?; - let connection: environments::EnvironmentConnectionSpec = - json_column(row, "controller_connection_json")?; +/// Every stored column of an environment record, as bound to SQL. +struct EnvironmentColumns { + source_kind: &'static str, + state: &'static str, + state_message: Option, + close_reason: Option<&'static str>, + state_changed_at_ms: i64, + request_id: Option, + display_name: Option, + metadata_json: serde_json::Value, + daemon_version: Option, + daemon_git_sha: Option, + daemon_protocol_version: Option, + provider_id: Option, + template_id: Option, + create_idle_json: Option, + provider_target_id: Option, + next_attempt_at_ms: Option, + attempts: i32, + observed_phase: Option<&'static str>, + observed_power: Option<&'static str>, + observed_message: Option, + observed_at_ms: Option, + public_endpoint: Option, + daemon_endpoint: Option, + registration_key_id: Option, + daemon_public_key: Option, + connected: Option, + connection_changed_at_ms: Option, + last_seen_at_ms: Option, + created_at_ms: i64, + updated_at_ms: i64, + closed_at_ms: Option, +} + +impl EnvironmentColumns { + fn of(record: &EnvironmentRecord) -> Result { + let mut columns = Self { + source_kind: record.source.kind_str(), + state: record.state.as_str(), + state_message: record.state_message.clone(), + close_reason: record.close_reason.map(EnvironmentCloseReason::as_str), + state_changed_at_ms: record.state_changed_at_ms, + request_id: record.request_id.as_ref().map(|id| id.as_str().to_owned()), + display_name: record.display_name.clone(), + metadata_json: metadata_filter_json(&record.metadata), + daemon_version: None, + daemon_git_sha: None, + daemon_protocol_version: None, + provider_id: None, + template_id: None, + create_idle_json: None, + provider_target_id: None, + next_attempt_at_ms: None, + attempts: 0, + observed_phase: None, + observed_power: None, + observed_message: None, + observed_at_ms: None, + public_endpoint: None, + daemon_endpoint: None, + registration_key_id: None, + daemon_public_key: None, + connected: None, + connection_changed_at_ms: None, + last_seen_at_ms: None, + created_at_ms: record.created_at_ms, + updated_at_ms: record.updated_at_ms, + closed_at_ms: record.closed_at_ms, + }; + if let Some(build) = &record.daemon_build { + columns.daemon_version = build.version.clone(); + columns.daemon_git_sha = build.git_sha.clone(); + columns.daemon_protocol_version = Some( + i32::try_from(build.protocol_version) + .map_err(|_| invalid_error("daemon protocol version is out of range"))?, + ); + } + match &record.source { + EnvironmentSource::Provisioned(source) => { + columns.provider_id = Some(source.provider_id.as_str().to_owned()); + columns.template_id = Some(source.template_id.as_str().to_owned()); + columns.create_idle_json = source + .idle + .as_ref() + .map(|policy| json_value("encode create idle policy", policy)) + .transpose()?; + columns.provider_target_id = source + .provider_target_id + .as_ref() + .map(|id| id.as_str().to_owned()); + columns.next_attempt_at_ms = source.next_attempt_at_ms; + columns.attempts = i32::try_from(source.attempts).unwrap_or(i32::MAX); + if let Some(observed) = &source.observed { + columns.observed_phase = Some(target_phase_str(observed.phase)); + columns.observed_power = observed.power.map(PowerState::as_str); + columns.observed_message = observed.message.clone(); + columns.observed_at_ms = Some(observed.observed_at_ms); + } + columns.public_endpoint = source.public_endpoint.clone(); + } + EnvironmentSource::External { endpoint } => { + columns.daemon_endpoint = Some(endpoint.as_str().to_owned()); + } + EnvironmentSource::Registered(source) => { + columns.registration_key_id = Some(source.registration_key_id.as_str().to_owned()); + columns.daemon_public_key = Some(source.daemon_public_key.clone()); + columns.connected = Some(source.connected); + columns.connection_changed_at_ms = Some(source.connection_changed_at_ms); + columns.last_seen_at_ms = Some(source.last_seen_at_ms); + } + } + Ok(columns) + } +} + +/// Insert a new row. Returns false, inserting nothing, when the request id +/// is already taken in the universe. +async fn insert_environment_row( + tx: &mut Transaction<'_, Postgres>, + universe_id: Uuid, + record: &EnvironmentRecord, + columns: EnvironmentColumns, +) -> Result { + let result = sqlx::query( + r#" + INSERT INTO environments ( + universe_id, environment_id, source_kind, state, state_message, close_reason, + state_changed_at_ms, request_id, display_name, metadata_json, + daemon_version, daemon_git_sha, daemon_protocol_version, + provider_id, template_id, create_idle_json, provider_target_id, + next_attempt_at_ms, attempts, + observed_phase, observed_power, observed_message, observed_at_ms, + public_endpoint, daemon_endpoint, + registration_key_id, daemon_public_key, connected, + connection_changed_at_ms, last_seen_at_ms, + created_at_ms, updated_at_ms, closed_at_ms + ) VALUES ( + $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17, + $18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33 + ) + ON CONFLICT (universe_id, request_id) WHERE request_id IS NOT NULL DO NOTHING + "#, + ) + .bind(universe_id) + .bind(record.environment_id.as_str()) + .bind(columns.source_kind) + .bind(columns.state) + .bind(columns.state_message) + .bind(columns.close_reason) + .bind(columns.state_changed_at_ms) + .bind(columns.request_id) + .bind(columns.display_name) + .bind(columns.metadata_json) + .bind(columns.daemon_version) + .bind(columns.daemon_git_sha) + .bind(columns.daemon_protocol_version) + .bind(columns.provider_id) + .bind(columns.template_id) + .bind(columns.create_idle_json) + .bind(columns.provider_target_id) + .bind(columns.next_attempt_at_ms) + .bind(columns.attempts) + .bind(columns.observed_phase) + .bind(columns.observed_power) + .bind(columns.observed_message) + .bind(columns.observed_at_ms) + .bind(columns.public_endpoint) + .bind(columns.daemon_endpoint) + .bind(columns.registration_key_id) + .bind(columns.daemon_public_key) + .bind(columns.connected) + .bind(columns.connection_changed_at_ms) + .bind(columns.last_seen_at_ms) + .bind(columns.created_at_ms) + .bind(columns.updated_at_ms) + .bind(columns.closed_at_ms) + .execute(&mut **tx) + .await?; + Ok(result.rows_affected() == 1) +} + +/// Write the mutable columns of a locked row. The kind, its identity +/// columns and its create inputs never change after the insert. +async fn update_environment_row( + tx: &mut Transaction<'_, Postgres>, + universe_id: Uuid, + record: &EnvironmentRecord, + columns: EnvironmentColumns, +) -> Result<(), sqlx::Error> { + sqlx::query( + r#" + UPDATE environments SET + state = $3, state_message = $4, close_reason = $5, state_changed_at_ms = $6, + display_name = $7, metadata_json = $8, + daemon_version = $9, daemon_git_sha = $10, daemon_protocol_version = $11, + provider_target_id = $12, next_attempt_at_ms = $13, attempts = $14, + observed_phase = $15, observed_power = $16, observed_message = $17, + observed_at_ms = $18, public_endpoint = $19, + connected = $20, connection_changed_at_ms = $21, last_seen_at_ms = $22, + updated_at_ms = $23, closed_at_ms = $24 + WHERE universe_id = $1 AND environment_id = $2 + "#, + ) + .bind(universe_id) + .bind(record.environment_id.as_str()) + .bind(columns.state) + .bind(columns.state_message) + .bind(columns.close_reason) + .bind(columns.state_changed_at_ms) + .bind(columns.display_name) + .bind(columns.metadata_json) + .bind(columns.daemon_version) + .bind(columns.daemon_git_sha) + .bind(columns.daemon_protocol_version) + .bind(columns.provider_target_id) + .bind(columns.next_attempt_at_ms) + .bind(columns.attempts) + .bind(columns.observed_phase) + .bind(columns.observed_power) + .bind(columns.observed_message) + .bind(columns.observed_at_ms) + .bind(columns.public_endpoint) + .bind(columns.connected) + .bind(columns.connection_changed_at_ms) + .bind(columns.last_seen_at_ms) + .bind(columns.updated_at_ms) + .bind(columns.closed_at_ms) + .execute(&mut **tx) + .await?; + Ok(()) +} + +fn provider_from_row(row: &PgRow) -> Result { let record = EnvironmentProviderRecord { - provider_id, - display_name: row - .try_get("display_name") - .map_err(|e| sql_error("decode provider display_name", e))?, - controller_connection: connection, - metadata: json_column(row, "metadata_json")?, + provider_id: parse_id(row, "provider_id", EnvironmentProviderId::try_new)?, + display_name: optional_string(row, "display_name")?, + controller_endpoint: parse_id(row, "controller_endpoint", |value| { + EnvironmentEndpoint::parse(&value) + })?, + data_route_base: optional_id(row, "data_route_base", DataRouteBase::new)?, + has_credential: row + .try_get("has_credential") + .map_err(|e| sql_error("decode provider credential presence", e))?, created_at_ms: scalar(row, "created_at_ms")?, updated_at_ms: scalar(row, "updated_at_ms")?, }; @@ -1170,17 +1575,16 @@ fn provider_from_row( } fn binding_from_row( - row: &sqlx::postgres::PgRow, + row: &PgRow, ) -> Result { let record = EnvironmentProviderBindingRecord { universe_id: row .try_get("universe_id") .map_err(|e| sql_error("decode binding universe", e))?, - binding_id: parse_id(row, "binding_id", EnvironmentProviderBindingId::try_new)?, provider_id: parse_id(row, "provider_id", EnvironmentProviderId::try_new)?, - status: binding_status_from_str(&column(row, "status")?)?, - revision: i64_to_u64(scalar(row, "revision")?, "revision")?, - metadata: json_column(row, "metadata_json")?, + enabled: row + .try_get("enabled") + .map_err(|e| sql_error("decode binding enabled", e))?, created_at_ms: scalar(row, "created_at_ms")?, updated_at_ms: scalar(row, "updated_at_ms")?, }; @@ -1188,92 +1592,103 @@ fn binding_from_row( Ok(record) } -fn environment_from_row( - row: &sqlx::postgres::PgRow, -) -> Result { +fn environment_from_row(row: &PgRow) -> Result { let source_kind = column(row, "source_kind")?; - let provider_id: Option = row - .try_get("provider_id") - .map_err(|e| sql_error("decode environment provider", e))?; - let binding_id: Option = row - .try_get("binding_id") - .map_err(|e| sql_error("decode environment binding", e))?; let source = match source_kind.as_str() { - "provisioned" => EnvironmentSource::Provisioned { - provider_id: EnvironmentProviderId::try_new( - provider_id.ok_or_else(|| store_message("missing provider id"))?, - ) - .map_err(|e| store_message(format!("decode provider id: {e}")))?, - binding_id: EnvironmentProviderBindingId::try_new( - binding_id.ok_or_else(|| store_message("missing binding id"))?, + "provisioned" => EnvironmentSource::Provisioned(ProvisionedSource { + provider_id: parse_id(row, "provider_id", EnvironmentProviderId::try_new)?, + template_id: parse_id(row, "template_id", EnvironmentTemplateId::try_new)?, + idle: optional_json_column(row, "create_idle_json")?, + provider_target_id: optional_id(row, "provider_target_id", ProviderTargetId::new)?, + observed: observed_from_row(row)?, + public_endpoint: optional_string(row, "public_endpoint")?, + next_attempt_at_ms: optional_i64(row, "next_attempt_at_ms")?, + attempts: u32::try_from( + row.try_get::("attempts") + .map_err(|e| sql_error("decode attempts", e))?, ) - .map_err(|e| store_message(format!("decode binding id: {e}")))?, - }, + .map_err(|_| store_message("attempts is negative"))?, + }), "external" => EnvironmentSource::External { - connection: json_column(row, "daemon_connection_json")?, + endpoint: parse_id(row, "daemon_endpoint", |value| { + EnvironmentEndpoint::parse(&value) + })?, }, - "registered" => EnvironmentSource::Registered { + "registered" => EnvironmentSource::Registered(RegisteredSource { registration_key_id: parse_id( row, "registration_key_id", EnvironmentRegistrationKeyId::try_new, )?, - daemon_id: parse_id(row, "daemon_id", EnvironmentDaemonId::try_new)?, daemon_public_key: column(row, "daemon_public_key")?, identity_mode: identity_mode_from_str(&column(row, "identity_mode")?)?, - }, + connected: row + .try_get::, _>("connected") + .map_err(|e| sql_error("decode connected", e))? + .ok_or_else(|| store_message("registered environment without connected"))?, + connection_changed_at_ms: required_i64(row, "connection_changed_at_ms")?, + last_seen_at_ms: required_i64(row, "last_seen_at_ms")?, + }), other => { return Err(store_message(format!( "unknown environment source: {other}" ))); } }; - let target: Option = row - .try_get("provider_target_id") - .map_err(|e| sql_error("decode provider target", e))?; + let daemon_build = row + .try_get::, _>("daemon_protocol_version") + .map_err(|e| sql_error("decode daemon protocol version", e))? + .map( + |protocol_version| -> Result { + Ok(DaemonBuild { + version: optional_string(row, "daemon_version")?, + git_sha: optional_string(row, "daemon_git_sha")?, + protocol_version: u32::try_from(protocol_version) + .map_err(|_| store_message("daemon protocol version is negative"))?, + }) + }, + ) + .transpose()?; + let state = column(row, "state")?; let record = EnvironmentRecord { environment_id: parse_id(row, "environment_id", EnvironmentId::try_new)?, - request_id: parse_id(row, "request_id", EnvironmentProvisionRequestId::try_new)?, + request_id: optional_id(row, "request_id", EnvironmentProvisionRequestId::try_new)?, source, - display_name: row - .try_get("display_name") - .map_err(|e| sql_error("decode environment display", e))?, - status: environment_status_from_str(&column(row, "status")?)?, - desired_power: power_state_from_str(&column(row, "desired_power")?)?, - idle_policy: optional_json_column(row, "idle_policy_json")?, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: parse_id(row, "incarnation_id", EnvironmentIncarnationId::try_new)?, - provision_request_id: optional_id( - row, - "provision_request_id", - EnvironmentProvisionRequestId::try_new, - )?, - provider_target_id: target.map(ProviderTargetId::new), - template_id: optional_id(row, "template_id", EnvironmentTemplateId::try_new)?, - adoption_source_target: row - .try_get("adoption_source_target") - .map_err(|e| sql_error("decode adoption source target", e))?, - power_states: json_column(row, "power_states_json")?, - created_at_ms: scalar(row, "incarnation_created_at_ms")?, - updated_at_ms: scalar(row, "incarnation_updated_at_ms")?, - }, - public_ingress_enabled: row - .try_get("public_ingress_enabled") - .map_err(|e| sql_error("decode public ingress enabled", e))?, - public_endpoint: row - .try_get("public_endpoint") - .map_err(|e| sql_error("decode public endpoint", e))?, + display_name: optional_string(row, "display_name")?, metadata: json_column(row, "metadata_json")?, - last_seen_at_ms: row - .try_get("last_seen_at_ms") - .map_err(|e| sql_error("decode last seen", e))?, + state: EnvironmentState::parse(&state) + .ok_or_else(|| store_message(format!("unknown environment state: {state}")))?, + state_message: optional_string(row, "state_message")?, + close_reason: optional_string(row, "close_reason")? + .map(|reason| { + EnvironmentCloseReason::parse(&reason) + .ok_or_else(|| store_message(format!("unknown close reason: {reason}"))) + }) + .transpose()?, + state_changed_at_ms: scalar(row, "state_changed_at_ms")?, + daemon_build, created_at_ms: scalar(row, "created_at_ms")?, updated_at_ms: scalar(row, "updated_at_ms")?, + closed_at_ms: optional_i64(row, "closed_at_ms")?, }; record.validate()?; Ok(record) } +fn observed_from_row(row: &PgRow) -> Result, EnvironmentRegistryError> { + let Some(phase) = optional_string(row, "observed_phase")? else { + return Ok(None); + }; + Ok(Some(ObservedTarget { + phase: target_phase_from_str(&phase)?, + power: optional_string(row, "observed_power")? + .map(|power| power_state_from_str(&power)) + .transpose()?, + message: optional_string(row, "observed_message")?, + observed_at_ms: required_i64(row, "observed_at_ms")?, + })) +} + pub(crate) fn identity_mode_from_str( value: &str, ) -> Result { @@ -1281,19 +1696,45 @@ pub(crate) fn identity_mode_from_str( .ok_or_else(|| store_message(format!("unknown identity mode: {value}"))) } +fn target_phase_str(phase: TargetPhase) -> &'static str { + match phase { + TargetPhase::Provisioning => "provisioning", + TargetPhase::Active => "active", + TargetPhase::Closing => "closing", + TargetPhase::Closed => "closed", + TargetPhase::Failed => "failed", + TargetPhase::Unknown => "unknown", + } +} + +fn target_phase_from_str(value: &str) -> Result { + [ + TargetPhase::Provisioning, + TargetPhase::Active, + TargetPhase::Closing, + TargetPhase::Closed, + TargetPhase::Failed, + TargetPhase::Unknown, + ] + .into_iter() + .find(|phase| target_phase_str(*phase) == value) + .ok_or_else(|| store_message(format!("unknown target phase: {value}"))) +} + +fn power_state_from_str(value: &str) -> Result { + PowerState::ALL + .into_iter() + .find(|state| state.as_str() == value) + .ok_or_else(|| store_message(format!("unknown environment power state: {value}"))) +} + fn credential_from_row( - row: &sqlx::postgres::PgRow, + row: &PgRow, ) -> Result { let kind = column(row, "source_kind")?; - let grant: Option = row - .try_get("grant_id") - .map_err(|e| sql_error("decode grant id", e))?; - let provider: Option = row - .try_get("auth_provider_id") - .map_err(|e| sql_error("decode auth provider id", e))?; - let secret: Option = row - .try_get("secret_id") - .map_err(|e| sql_error("decode secret id", e))?; + let grant = optional_string(row, "grant_id")?; + let provider = optional_string(row, "auth_provider_id")?; + let secret = optional_string(row, "secret_id")?; let source = match kind.as_str() { "auth_grant" => EnvironmentCredentialSource::AuthGrant { grant_id: AuthGrantId::try_new(grant.ok_or_else(|| store_message("missing grant id"))?) @@ -1341,30 +1782,12 @@ fn credential_source_columns( } } -fn binding_status_to_str(value: EnvironmentProviderBindingStatus) -> &'static str { - match value { - EnvironmentProviderBindingStatus::Enabled => "enabled", - EnvironmentProviderBindingStatus::Disabled => "disabled", - } -} -fn binding_status_from_str( - value: &str, -) -> Result { - match value { - "enabled" => Ok(EnvironmentProviderBindingStatus::Enabled), - "disabled" => Ok(EnvironmentProviderBindingStatus::Disabled), - other => Err(store_message(format!("unknown binding status: {other}"))), - } -} -fn power_state_from_str(value: &str) -> Result { - PowerState::ALL - .into_iter() - .find(|state| state.as_str() == value) - .ok_or_else(|| store_message(format!("unknown environment power state: {value}"))) -} +// --------------------------------------------------------------------------- +// Decoding and error helpers +// --------------------------------------------------------------------------- fn optional_json_column( - row: &sqlx::postgres::PgRow, + row: &PgRow, name: &str, ) -> Result, EnvironmentRegistryError> { let value: Option = row @@ -1377,52 +1800,8 @@ fn optional_json_column( .transpose() } -fn environment_status_to_str(value: EnvironmentStatus) -> &'static str { - match value { - EnvironmentStatus::Provisioning => "provisioning", - EnvironmentStatus::Booting => "booting", - EnvironmentStatus::Ready => "ready", - EnvironmentStatus::Paused => "paused", - EnvironmentStatus::Suspended => "suspended", - EnvironmentStatus::Offline => "offline", - EnvironmentStatus::Closing => "closing", - EnvironmentStatus::Closed => "closed", - EnvironmentStatus::Failed => "failed", - EnvironmentStatus::Unknown => "unknown", - } -} -fn environment_status_from_str(value: &str) -> Result { - match value { - "provisioning" => Ok(EnvironmentStatus::Provisioning), - "booting" => Ok(EnvironmentStatus::Booting), - "ready" => Ok(EnvironmentStatus::Ready), - "paused" => Ok(EnvironmentStatus::Paused), - "suspended" => Ok(EnvironmentStatus::Suspended), - "offline" => Ok(EnvironmentStatus::Offline), - "closing" => Ok(EnvironmentStatus::Closing), - "closed" => Ok(EnvironmentStatus::Closed), - "failed" => Ok(EnvironmentStatus::Failed), - "unknown" => Ok(EnvironmentStatus::Unknown), - other => Err(store_message(format!( - "unknown environment status: {other}" - ))), - } -} - -fn validate_ingress_request_time(value: i64) -> Result<(), EnvironmentRegistryError> { - if value < 0 { - invalid("updated_at_ms must be nonnegative") - } else { - Ok(()) - } -} - -fn invalid_store(message: impl Into) -> Result { - invalid(message) -} - fn parse_id( - row: &sqlx::postgres::PgRow, + row: &PgRow, name: &str, parse: impl FnOnce(String) -> Result, ) -> Result @@ -1431,38 +1810,54 @@ where { parse(column(row, name)?).map_err(|e| store_message(format!("decode {name}: {e}"))) } + fn optional_id( - row: &sqlx::postgres::PgRow, + row: &PgRow, name: &str, parse: impl FnOnce(String) -> Result, ) -> Result, EnvironmentRegistryError> where E: std::fmt::Display, { - let value: Option = row - .try_get(name) - .map_err(|e| sql_error("decode optional id", e))?; - value + optional_string(row, name)? .map(parse) .transpose() .map_err(|e| store_message(format!("decode {name}: {e}"))) } -fn column(row: &sqlx::postgres::PgRow, name: &str) -> Result { + +fn optional_string(row: &PgRow, name: &str) -> Result, EnvironmentRegistryError> { + row.try_get(name) + .map_err(|e| sql_error(&format!("decode {name}"), e)) +} + +fn optional_i64(row: &PgRow, name: &str) -> Result, EnvironmentRegistryError> { + row.try_get(name) + .map_err(|e| sql_error(&format!("decode {name}"), e)) +} + +fn required_i64(row: &PgRow, name: &str) -> Result { + optional_i64(row, name)?.ok_or_else(|| store_message(format!("missing {name}"))) +} + +fn column(row: &PgRow, name: &str) -> Result { row.try_get(name) - .map_err(|e| sql_error("decode environment column", e)) + .map_err(|e| sql_error(&format!("decode {name}"), e)) } -fn scalar(row: &sqlx::postgres::PgRow, name: &str) -> Result { + +fn scalar(row: &PgRow, name: &str) -> Result { row.try_get(name) - .map_err(|e| sql_error("decode integer column", e)) + .map_err(|e| sql_error(&format!("decode {name}"), e)) } + fn json_value( action: &str, value: &T, ) -> Result { serde_json::to_value(value).map_err(|e| store_message(format!("{action}: {e}"))) } + fn json_column( - row: &sqlx::postgres::PgRow, + row: &PgRow, name: &str, ) -> Result { let value: serde_json::Value = row @@ -1470,71 +1865,67 @@ fn json_column( .map_err(|e| sql_error("decode json column", e))?; serde_json::from_value(value).map_err(|e| store_message(format!("decode {name}: {e}"))) } -fn i64_to_u64(value: i64, name: &str) -> Result { - value - .try_into() - .map_err(|_| store_message(format!("{name} is negative"))) -} + pub(crate) fn not_found(kind: &'static str, id: &impl ToString) -> EnvironmentRegistryError { EnvironmentRegistryError::NotFound { kind, id: id.to_string(), } } + pub(crate) fn invalid(message: impl Into) -> Result { Err(invalid_error(message)) } + fn invalid_error(message: impl Into) -> EnvironmentRegistryError { EnvironmentRegistryError::InvalidInput { message: message.into(), } } -fn store_error(action: &str, error: crate::PgStoreError) -> EnvironmentRegistryError { + +fn store_error(action: &str, error: PgStoreError) -> EnvironmentRegistryError { store_message(format!("{action}: {error}")) } + pub(crate) fn sql_error(action: &str, error: sqlx::Error) -> EnvironmentRegistryError { store_message(format!("{action}: {error}")) } + pub(crate) fn store_message(message: impl Into) -> EnvironmentRegistryError { EnvironmentRegistryError::Store { message: message.into(), } } -fn map_environment_insert_error(error: sqlx::Error) -> EnvironmentRegistryError { - if let Some(db) = error.as_database_error() - && db.constraint() == Some("environments_pkey") - { - return EnvironmentRegistryError::AlreadyExists { + +pub(crate) fn constraint_of(error: &sqlx::Error) -> Option<&str> { + error.as_database_error().and_then(|db| db.constraint()) +} + +fn map_environment_write_error(action: &str, error: sqlx::Error) -> EnvironmentRegistryError { + match constraint_of(&error) { + Some("environments_pkey") => EnvironmentRegistryError::AlreadyExists { kind: "environment", id: "duplicate environment id".to_owned(), - }; - } - sql_error("insert environment", error) -} -fn map_registered_insert_error( - error: sqlx::Error, - daemon_id: &EnvironmentDaemonId, -) -> EnvironmentRegistryError { - if let Some(db) = error.as_database_error() - && db.constraint() == Some("environments_daemon_public_key_idx") - { - return EnvironmentRegistryError::AlreadyExists { - kind: "daemon_identity", - id: daemon_id.to_string(), - }; + }, + Some("environments_provider_target_idx") => EnvironmentRegistryError::AlreadyExists { + kind: "provider_target", + id: "target recorded for another open environment".to_owned(), + }, + Some("environments_registration_key_fk") => { + invalid_error("environment registration key does not exist") + } + _ => sql_error(action, error), } - map_environment_insert_error(error) } -fn map_binding_write_error(action: &str, error: sqlx::Error) -> EnvironmentRegistryError { - if let Some(db) = error.as_database_error() - && db.constraint() == Some("environment_provider_bindings_universe_id_provider_id_key") + +fn map_binding_write_error(error: sqlx::Error, universe_id: Uuid) -> EnvironmentRegistryError { + if error + .as_database_error() + .is_some_and(|database| database.code().as_deref() == Some("23503")) { - return EnvironmentRegistryError::AlreadyExists { - kind: "environment_provider_binding", - id: "universe/provider".to_owned(), - }; + return not_found("universe", &universe_id); } - sql_error(action, error) + sql_error("put provider binding", error) } fn map_provider_delete_error(error: sqlx::Error) -> EnvironmentRegistryError { @@ -1547,14 +1938,32 @@ fn map_provider_delete_error(error: sqlx::Error) -> EnvironmentRegistryError { sql_error("delete environment provider", error) } -#[allow(dead_code)] -async fn _transaction_marker(_: &mut Transaction<'_, Postgres>) {} +/// Decode every row that still decodes. A row that does not is left out +/// and logged, so one bad row never hides the rest of a listing or stalls +/// a reconciler pass over them. +pub(crate) fn decode_rows( + rows: &[PgRow], + decode: impl Fn(&PgRow) -> Result, + kind: &'static str, +) -> Vec { + rows.iter() + .filter_map(|row| decoded_or_logged(decode(row), kind)) + .collect() +} + +fn decoded_or_logged( + result: Result, + kind: &'static str, +) -> Option { + result + .inspect_err(|error| tracing::error!(%error, kind, "skipping undecodable registry row")) + .ok() +} -/// A string map as the jsonb object the `@>` containment filter compares -/// against; building the value directly cannot fail. -fn metadata_filter_json( - metadata: &std::collections::BTreeMap, -) -> serde_json::Value { +/// A string map as a jsonb object: the stored caller metadata and the `@>` +/// containment filter compared against it. Building the value directly +/// cannot fail. +fn metadata_filter_json(metadata: &BTreeMap) -> serde_json::Value { serde_json::Value::Object( metadata .iter() diff --git a/crates/store-pg/src/environment_registration.rs b/crates/store-pg/src/environment_registration.rs index 0aa704097..8ab5aba87 100644 --- a/crates/store-pg/src/environment_registration.rs +++ b/crates/store-pg/src/environment_registration.rs @@ -1,27 +1,40 @@ -//! Registration keys for outbound `envd` registration, plus the two -//! deployment-wide lookups the environment gateway needs before it knows a -//! universe: a presented secret selects the universe of its key, and a -//! known daemon public key selects the universe of its environment. +//! Registration keys for outbound `envd` registration and their secrets, +//! plus the two deployment-wide lookups the environment gateway needs +//! before it knows a universe: a presented secret selects the universe of +//! its key, and a known daemon public key selects the universe of its +//! environment. // Formatted SQL uses internal schema fragments; request values use bind parameters. use async_trait::async_trait; use environments::{ CreateEnvironmentRegistrationKey, EnvironmentRegistrationKeyId, EnvironmentRegistrationKeyRecord, EnvironmentRegistrationKeyStore, EnvironmentRegistryError, - RegistrationKeyUsage, RevokeEnvironmentRegistrationKey, + RegistrationKeyUsage, RevokeEnvironmentRegistrationKey, RotateEnvironmentRegistrationKey, + UpdateEnvironmentRegistrationKey, }; use sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow}; use uuid::Uuid; use crate::{ PgStore, PgStoreError, - environment::{identity_mode_from_str, invalid, not_found, sql_error, store_message}, + environment::{ + constraint_of, decode_rows, identity_mode_from_str, invalid, not_found, sql_error, + store_message, + }, }; +/// Columns of `environment_registration_keys k`. The display prefix is the +/// live secret's (the latest one when a key has none live). const KEY_COLUMNS: &str = r#" - registration_key_id, display_name, key_prefix, identity_mode, - max_active_environments, ephemeral_disconnect_grace_ms, expires_at_ms, - created_at_ms, revoked_at_ms + k.registration_key_id, k.display_name, k.identity_mode, + k.max_active_environments, k.ephemeral_disconnect_grace_ms, k.offline_close_after_ms, + k.expires_at_ms, k.created_at_ms, k.updated_at_ms, k.revoked_at_ms, + ( + SELECT s.key_prefix FROM environment_registration_key_secrets s + WHERE s.universe_id = k.universe_id AND s.registration_key_id = k.registration_key_id + ORDER BY s.retired_at_ms IS NOT NULL, s.created_at_ms DESC + LIMIT 1 + ) AS key_prefix "#; #[async_trait] @@ -31,44 +44,58 @@ impl EnvironmentRegistrationKeyStore for PgStore { request: CreateEnvironmentRegistrationKey, ) -> Result { request.record.validate()?; + validate_secret_hash(&request.secret_hash)?; self.ensure_universe() .await .map_err(|error| store_message(format!("ensure universe: {error}")))?; let record = request.record; - let query = format!( + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin registration key create", error))?; + sqlx::query( r#" INSERT INTO environment_registration_keys ( - universe_id, registration_key_id, display_name, key_prefix, secret_hash, - identity_mode, max_active_environments, ephemeral_disconnect_grace_ms, - expires_at_ms, created_at_ms, revoked_at_ms + universe_id, registration_key_id, display_name, identity_mode, + max_active_environments, ephemeral_disconnect_grace_ms, offline_close_after_ms, + expires_at_ms, revoked_at_ms, created_at_ms, updated_at_ms ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) - RETURNING {KEY_COLUMNS} - "# - ); - let row = sqlx::query(sqlx::AssertSqlSafe(query)) - .bind(self.config.universe_id) - .bind(record.registration_key_id.as_str()) - .bind(&record.display_name) - .bind(&record.key_prefix) - .bind(&request.secret_hash) - .bind(record.identity_mode.as_str()) - .bind( - record - .max_active_environments - .map(|value| i32::try_from(value).unwrap_or(i32::MAX)), - ) - .bind( - record - .ephemeral_disconnect_grace_ms - .map(|value| i64::try_from(value).unwrap_or(i64::MAX)), - ) - .bind(record.expires_at_ms) - .bind(record.created_at_ms) - .bind(record.revoked_at_ms) - .fetch_one(&self.pool) + "#, + ) + .bind(self.config.universe_id) + .bind(record.registration_key_id.as_str()) + .bind(&record.display_name) + .bind(record.identity_mode.as_str()) + .bind(max_active_column(record.max_active_environments)) + .bind(duration_column(record.ephemeral_disconnect_grace_ms)) + .bind(duration_column(record.offline_close_after_ms)) + .bind(record.expires_at_ms) + .bind(record.revoked_at_ms) + .bind(record.created_at_ms) + .bind(record.updated_at_ms) + .execute(&mut *tx) + .await + .map_err(|error| map_key_write_error(error, &record.registration_key_id))?; + insert_secret( + &mut tx, + self.config.universe_id, + &record.registration_key_id, + &request.secret_hash, + &record.key_prefix, + record.created_at_ms, + ) + .await?; + let record = lock_registration_key( + &mut tx, + self.config.universe_id, + &record.registration_key_id, + ) + .await?; + tx.commit() .await - .map_err(|error| map_key_insert_error(error, &record.registration_key_id))?; - key_from_row(&row) + .map_err(|error| sql_error("commit registration key create", error))?; + Ok(record) } async fn read_registration_key( @@ -76,7 +103,7 @@ impl EnvironmentRegistrationKeyStore for PgStore { registration_key_id: &EnvironmentRegistrationKeyId, ) -> Result { let query = format!( - "SELECT {KEY_COLUMNS} FROM environment_registration_keys WHERE universe_id = $1 AND registration_key_id = $2" + "SELECT {KEY_COLUMNS} FROM environment_registration_keys k WHERE k.universe_id = $1 AND k.registration_key_id = $2" ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(self.config.universe_id) @@ -92,14 +119,114 @@ impl EnvironmentRegistrationKeyStore for PgStore { &self, ) -> Result, EnvironmentRegistryError> { let query = format!( - "SELECT {KEY_COLUMNS} FROM environment_registration_keys WHERE universe_id = $1 ORDER BY created_at_ms, registration_key_id" + "SELECT {KEY_COLUMNS} FROM environment_registration_keys k WHERE k.universe_id = $1 ORDER BY k.created_at_ms, k.registration_key_id" ); let rows = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(self.config.universe_id) .fetch_all(&self.pool) .await .map_err(|error| sql_error("list registration keys", error))?; - rows.iter().map(key_from_row).collect() + Ok(decode_rows( + &rows, + key_from_row, + "environment registration key", + )) + } + + async fn update_registration_key( + &self, + request: UpdateEnvironmentRegistrationKey, + ) -> Result { + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin registration key update", error))?; + let mut record = lock_registration_key( + &mut tx, + self.config.universe_id, + &request.registration_key_id, + ) + .await?; + request.apply(&mut record)?; + sqlx::query( + r#" + UPDATE environment_registration_keys SET + display_name = $3, max_active_environments = $4, + ephemeral_disconnect_grace_ms = $5, offline_close_after_ms = $6, + expires_at_ms = $7, updated_at_ms = $8 + WHERE universe_id = $1 AND registration_key_id = $2 + "#, + ) + .bind(self.config.universe_id) + .bind(record.registration_key_id.as_str()) + .bind(&record.display_name) + .bind(max_active_column(record.max_active_environments)) + .bind(duration_column(record.ephemeral_disconnect_grace_ms)) + .bind(duration_column(record.offline_close_after_ms)) + .bind(record.expires_at_ms) + .bind(record.updated_at_ms) + .execute(&mut *tx) + .await + .map_err(|error| sql_error("update registration key", error))?; + tx.commit() + .await + .map_err(|error| sql_error("commit registration key update", error))?; + Ok(record) + } + + async fn rotate_registration_key( + &self, + request: RotateEnvironmentRegistrationKey, + ) -> Result { + let mut tx = self + .pool + .begin() + .await + .map_err(|error| sql_error("begin registration key rotation", error))?; + let mut record = lock_registration_key( + &mut tx, + self.config.universe_id, + &request.registration_key_id, + ) + .await?; + request.apply(&mut record)?; + // Retire before inserting: a key has at most one live secret. + sqlx::query( + r#" + UPDATE environment_registration_key_secrets + SET retired_at_ms = GREATEST(created_at_ms, $3) + WHERE universe_id = $1 AND registration_key_id = $2 AND retired_at_ms IS NULL + "#, + ) + .bind(self.config.universe_id) + .bind(record.registration_key_id.as_str()) + .bind(request.rotated_at_ms) + .execute(&mut *tx) + .await + .map_err(|error| sql_error("retire registration key secret", error))?; + insert_secret( + &mut tx, + self.config.universe_id, + &record.registration_key_id, + &request.secret_hash, + &request.key_prefix, + request.rotated_at_ms, + ) + .await?; + sqlx::query( + "UPDATE environment_registration_keys SET updated_at_ms = $3 WHERE universe_id = $1 AND registration_key_id = $2", + ) + .bind(self.config.universe_id) + .bind(record.registration_key_id.as_str()) + .bind(record.updated_at_ms) + .execute(&mut *tx) + .await + .map_err(|error| sql_error("touch rotated registration key", error))?; + tx.commit() + .await + .map_err(|error| sql_error("commit registration key rotation", error))?; + Ok(record) } async fn revoke_registration_key( @@ -109,25 +236,30 @@ impl EnvironmentRegistrationKeyStore for PgStore { if request.revoked_at_ms < 0 { return invalid("revoked_at_ms must be nonnegative"); } - let query = format!( + let revoked = sqlx::query( r#" - UPDATE environment_registration_keys - SET revoked_at_ms = COALESCE(revoked_at_ms, GREATEST($3, created_at_ms)) + UPDATE environment_registration_keys SET + revoked_at_ms = COALESCE(revoked_at_ms, GREATEST($3, created_at_ms)), + updated_at_ms = CASE WHEN revoked_at_ms IS NULL + THEN GREATEST(updated_at_ms, $3, created_at_ms) + ELSE updated_at_ms END WHERE universe_id = $1 AND registration_key_id = $2 - RETURNING {KEY_COLUMNS} - "# - ); - let row = sqlx::query(sqlx::AssertSqlSafe(query)) - .bind(self.config.universe_id) - .bind(request.registration_key_id.as_str()) - .bind(request.revoked_at_ms) - .fetch_optional(&self.pool) + "#, + ) + .bind(self.config.universe_id) + .bind(request.registration_key_id.as_str()) + .bind(request.revoked_at_ms) + .execute(&self.pool) + .await + .map_err(|error| sql_error("revoke registration key", error))?; + if revoked.rows_affected() == 0 { + return Err(not_found( + "environment_registration_key", + &request.registration_key_id, + )); + } + self.read_registration_key(&request.registration_key_id) .await - .map_err(|error| sql_error("revoke registration key", error))? - .ok_or_else(|| { - not_found("environment_registration_key", &request.registration_key_id) - })?; - key_from_row(&row) } async fn resolve_registration_key( @@ -135,7 +267,14 @@ impl EnvironmentRegistrationKeyStore for PgStore { secret_hash: &str, ) -> Result, EnvironmentRegistryError> { let query = format!( - "SELECT {KEY_COLUMNS} FROM environment_registration_keys WHERE universe_id = $1 AND secret_hash = $2" + r#" + SELECT {KEY_COLUMNS} + FROM environment_registration_key_secrets live + JOIN environment_registration_keys k + ON k.universe_id = live.universe_id + AND k.registration_key_id = live.registration_key_id + WHERE live.universe_id = $1 AND live.secret_hash = $2 AND live.retired_at_ms IS NULL + "# ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(self.config.universe_id) @@ -154,7 +293,7 @@ impl EnvironmentRegistrationKeyStore for PgStore { let row = sqlx::query( r#" SELECT count(*) AS registered, - count(*) FILTER (WHERE status <> 'closed') AS active, + count(*) FILTER (WHERE state <> 'closed') AS active, max(created_at_ms) AS last_registered_at_ms FROM environments WHERE universe_id = $1 AND registration_key_id = $2 @@ -181,14 +320,14 @@ impl EnvironmentRegistrationKeyStore for PgStore { } } -/// Lock one key row for the duration of an admission transaction. +/// Lock one key row for the duration of a transaction. pub(crate) async fn lock_registration_key( tx: &mut Transaction<'_, Postgres>, universe_id: Uuid, registration_key_id: &EnvironmentRegistrationKeyId, ) -> Result { let query = format!( - "SELECT {KEY_COLUMNS} FROM environment_registration_keys WHERE universe_id = $1 AND registration_key_id = $2 FOR UPDATE" + "SELECT {KEY_COLUMNS} FROM environment_registration_keys k WHERE k.universe_id = $1 AND k.registration_key_id = $2 FOR UPDATE" ); let row = sqlx::query(sqlx::AssertSqlSafe(query)) .bind(universe_id) @@ -200,15 +339,48 @@ pub(crate) async fn lock_registration_key( key_from_row(&row) } -/// The universe whose registration key hashes to `secret_hash`. Key rows are -/// universe-scoped; this is the one lookup that intentionally searches -/// across universes, because the key is what selects the universe. +async fn insert_secret( + tx: &mut Transaction<'_, Postgres>, + universe_id: Uuid, + registration_key_id: &EnvironmentRegistrationKeyId, + secret_hash: &str, + key_prefix: &str, + created_at_ms: i64, +) -> Result<(), EnvironmentRegistryError> { + sqlx::query( + r#" + INSERT INTO environment_registration_key_secrets ( + secret_hash, universe_id, registration_key_id, key_prefix, created_at_ms + ) VALUES ($1,$2,$3,$4,$5) + "#, + ) + .bind(secret_hash) + .bind(universe_id) + .bind(registration_key_id.as_str()) + .bind(key_prefix) + .bind(created_at_ms) + .execute(&mut **tx) + .await + .map_err(|error| map_key_write_error(error, registration_key_id))?; + Ok(()) +} + +/// The universe whose live registration secret hashes to `secret_hash`. +/// Key rows are universe-scoped; this is the one lookup that intentionally +/// searches across universes, because the key is what selects the universe. +/// A retired secret selects nothing. pub async fn find_registration_key_universe( pool: &PgPool, secret_hash: &str, ) -> Result, PgStoreError> { let row: Option<(Uuid,)> = sqlx::query_as( - "SELECT universe_id FROM environment_registration_keys WHERE secret_hash = $1", + r#" + SELECT k.universe_id + FROM environment_registration_key_secrets s + JOIN environment_registration_keys k + ON k.universe_id = s.universe_id AND k.registration_key_id = s.registration_key_id + WHERE s.secret_hash = $1 AND s.retired_at_ms IS NULL + "#, ) .bind(secret_hash) .fetch_optional(pool) @@ -238,9 +410,6 @@ fn key_from_row(row: &PgRow) -> Result = row .try_get("max_active_environments") .map_err(|error| sql_error("decode max active", error))?; - let grace: Option = row - .try_get("ephemeral_disconnect_grace_ms") - .map_err(|error| sql_error("decode disconnect grace", error))?; let record = EnvironmentRegistrationKeyRecord { registration_key_id: EnvironmentRegistrationKeyId::try_new( row.try_get::("registration_key_id") @@ -251,21 +420,24 @@ fn key_from_row(row: &PgRow) -> Result, _>("key_prefix") + .map_err(|error| sql_error("decode key prefix", error))? + .ok_or_else(|| store_message("registration key has no secret"))?, identity_mode: identity_mode_from_str(&identity_mode)?, max_active_environments: max_active .map(|value| u32::try_from(value).map_err(|_| store_message("max active out of range"))) .transpose()?, - ephemeral_disconnect_grace_ms: grace - .map(|value| u64::try_from(value).map_err(|_| store_message("grace out of range"))) - .transpose()?, + ephemeral_disconnect_grace_ms: duration_from_row(row, "ephemeral_disconnect_grace_ms")?, + offline_close_after_ms: duration_from_row(row, "offline_close_after_ms")?, expires_at_ms: row .try_get("expires_at_ms") .map_err(|error| sql_error("decode expires", error))?, created_at_ms: row .try_get("created_at_ms") .map_err(|error| sql_error("decode created", error))?, + updated_at_ms: row + .try_get("updated_at_ms") + .map_err(|error| sql_error("decode updated", error))?, revoked_at_ms: row .try_get("revoked_at_ms") .map_err(|error| sql_error("decode revoked", error))?, @@ -274,18 +446,50 @@ fn key_from_row(row: &PgRow) -> Result Result, EnvironmentRegistryError> { + row.try_get::, _>(name) + .map_err(|error| sql_error(&format!("decode {name}"), error))? + .map(|value| { + u64::try_from(value).map_err(|_| store_message(format!("{name} out of range"))) + }) + .transpose() +} + +fn max_active_column(value: Option) -> Option { + value.map(|value| i32::try_from(value).unwrap_or(i32::MAX)) +} + +fn duration_column(value: Option) -> Option { + value.map(|value| i64::try_from(value).unwrap_or(i64::MAX)) +} + +/// A registration secret hash is lowercase hex SHA-256. +fn validate_secret_hash(value: &str) -> Result<(), EnvironmentRegistryError> { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return invalid("registration secret hash must be 64 lowercase hex characters"); + } + Ok(()) +} + +fn map_key_write_error( error: sqlx::Error, registration_key_id: &EnvironmentRegistrationKeyId, ) -> EnvironmentRegistryError { - if error - .as_database_error() - .is_some_and(|db| db.code().as_deref() == Some("23505")) - { - return EnvironmentRegistryError::AlreadyExists { + match constraint_of(&error) { + Some("environment_registration_keys_pkey") => EnvironmentRegistryError::AlreadyExists { kind: "environment_registration_key", id: registration_key_id.to_string(), - }; + }, + Some("environment_registration_key_secrets_pkey") => { + EnvironmentRegistryError::AlreadyExists { + kind: "environment_registration_key_secret", + id: registration_key_id.to_string(), + } + } + _ => sql_error("write registration key", error), } - sql_error("insert registration key", error) } diff --git a/crates/store-pg/src/lib.rs b/crates/store-pg/src/lib.rs index 8a2b44a43..c666ec378 100644 --- a/crates/store-pg/src/lib.rs +++ b/crates/store-pg/src/lib.rs @@ -368,6 +368,11 @@ pub use deployment::{ list_universe_object_keys, list_universe_session_ids, list_universe_stats, put_universe_slug, read_universe_stats, }; +pub use environment::{ + list_revoked_environment_routes, list_universes_with_environment_work, + list_universes_with_open_provisioned_environments, list_universes_with_registered_environments, + repair_registered_connections, +}; pub use environment_registration::{ find_registered_environment_universe, find_registration_key_universe, }; @@ -385,40 +390,6 @@ pub async fn list_universes(pool: &PgPool) -> Result)> Ok(rows) } -/// Universes with lifecycle intents requiring provider reconciliation. -pub async fn list_universes_with_pending_environments( - pool: &PgPool, -) -> Result, PgStoreError> { - // Include pending provider work and registered-environment heartbeat repair. - let rows: Vec<(Uuid,)> = sqlx::query_as( - "SELECT DISTINCT e.universe_id FROM environments e \ - WHERE e.status IN ('provisioning','booting','closing','unknown') \ - OR (e.source_kind = 'registered' AND e.status NOT IN ('closing','closed')) \ - OR (e.source_kind = 'provisioned' AND ( \ - (e.status = 'ready' AND e.desired_power <> 'running') \ - OR (e.status = 'paused' AND e.desired_power <> 'paused') \ - OR (e.status = 'suspended' AND e.desired_power <> 'suspended') \ - OR (e.status = 'offline' AND e.desired_power <> 'stopped'))) \ - ORDER BY e.universe_id", - ) - .fetch_all(pool) - .await?; - Ok(rows.into_iter().map(|(universe_id,)| universe_id).collect()) -} - -/// Universes holding a ready provisioned environment with an idle policy: -/// the power reaper's scan set. -pub async fn list_universes_with_idle_policies(pool: &PgPool) -> Result, PgStoreError> { - let rows: Vec<(Uuid,)> = sqlx::query_as( - "SELECT DISTINCT e.universe_id FROM environments e \ - WHERE e.status = 'ready' AND e.idle_policy_json IS NOT NULL \ - ORDER BY e.universe_id", - ) - .fetch_all(pool) - .await?; - Ok(rows.into_iter().map(|(universe_id,)| universe_id).collect()) -} - /// Deployment-level check whether a universe exists. Runs above the /// per-universe `PgStore` boundary: multi-universe deployments consult it /// before lazily constructing a universe's store. diff --git a/crates/store-pg/src/migrations.rs b/crates/store-pg/src/migrations.rs index d0f3bf499..52f3a1550 100644 --- a/crates/store-pg/src/migrations.rs +++ b/crates/store-pg/src/migrations.rs @@ -34,9 +34,9 @@ const LIGHTSPEED_TABLES: &[&str] = &[ "channel_accounts", "channel_pairings", "environment_credentials", - "environment_incarnations", "environment_provider_bindings", "environment_providers", + "environment_registration_key_secrets", "environment_registration_keys", "environments", "mcp_servers", diff --git a/crates/store-pg/tests/migrations_live.rs b/crates/store-pg/tests/migrations_live.rs index e382666b2..8db959364 100644 --- a/crates/store-pg/tests/migrations_live.rs +++ b/crates/store-pg/tests/migrations_live.rs @@ -109,8 +109,16 @@ async fn exercise_migrator(pool: &sqlx::PgPool) -> Result<(), String> { return Err(format!("unexpected migrated status: {status:?}")); } + // The environment baseline is folded into one migration; the ledger + // ends at revision 11. + if REQUIRED_SCHEMA_REVISION != 11 { + return Err(format!( + "unexpected required schema revision {REQUIRED_SCHEMA_REVISION}" + )); + } + // The public schema can have these same names. Every newly migrated schema - // must still receive its own foreign keys, including the deferred cycle. + // must still receive its own foreign keys. let foreign_keys: Vec<(String, bool, bool, bool)> = sqlx::query_as( "SELECT constraint_row.conname::text, constraint_row.condeferrable, constraint_row.condeferred, @@ -120,7 +128,6 @@ async fn exercise_migrator(pool: &sqlx::PgPool) -> Result<(), String> { JOIN pg_class AS target ON target.oid = constraint_row.confrelid WHERE source.relnamespace = current_schema()::regnamespace AND constraint_row.conname IN ( - 'environments_current_incarnation_fk', 'environments_registration_key_fk', 'mcp_servers_auth_grant_fk' ) @@ -131,15 +138,12 @@ async fn exercise_migrator(pool: &sqlx::PgPool) -> Result<(), String> { .map_err(|error| error.to_string())?; if foreign_keys != vec![ - ( - "environments_current_incarnation_fk".to_owned(), - true, - true, - true, - ), + // Deferrable but checked immediately, so only a universe delete + // (which defers its constraints) can remove keys and + // environments in either cascade order. ( "environments_registration_key_fk".to_owned(), - false, + true, false, true, ), @@ -151,6 +155,99 @@ async fn exercise_migrator(pool: &sqlx::PgPool) -> Result<(), String> { )); } + // The environment baseline's named constraints and indexes. + let mut expected_constraints = vec![ + "environment_providers_controller_endpoint_format", + "environment_providers_credential_all_or_none", + "environment_providers_data_route_base_format", + "environment_providers_display_name_not_empty", + "environment_providers_provider_id_format", + "environment_providers_times_valid", + "environment_provider_bindings_times_valid", + "environment_registration_keys_display_name_bounded", + "environment_registration_keys_id_format", + "environment_registration_keys_identity_mode_known", + "environment_registration_keys_limits_positive", + "environment_registration_keys_offline_close_persistent", + "environment_registration_keys_times_valid", + "environment_registration_key_secrets_hash_format", + "environment_registration_key_secrets_prefix_not_empty", + "environment_registration_key_secrets_times_valid", + "environments_close_reason", + "environments_closed_at", + "environments_daemon_build", + "environments_ids_format", + "environments_json_objects", + "environments_observation", + "environments_registration_key_fk", + "environments_source_fields", + "environments_source_known", + "environments_state_known", + "environments_state_message_bounded", + "environments_state_per_kind", + "environments_times_valid", + "environments_work_schedule", + ]; + expected_constraints.sort_unstable(); + let constraints: Vec = sqlx::query_scalar( + "SELECT constraint_row.conname::text + FROM pg_constraint AS constraint_row + JOIN pg_class AS source ON source.oid = constraint_row.conrelid + WHERE source.relnamespace = current_schema()::regnamespace + AND source.relname IN ( + 'environment_providers', 'environment_provider_bindings', + 'environment_registration_keys', 'environment_registration_key_secrets', + 'environments' + ) + AND constraint_row.contype IN ('c', 'f') + AND constraint_row.conname = ANY($1::text[]) + ORDER BY constraint_row.conname", + ) + .bind(&expected_constraints) + .fetch_all(pool) + .await + .map_err(|error| error.to_string())?; + let mut constraints: Vec<&str> = constraints.iter().map(String::as_str).collect(); + constraints.sort_unstable(); + if constraints != expected_constraints { + return Err(format!( + "environment baseline constraints differ: {constraints:?}" + )); + } + let mut expected_indexes = vec![ + "environment_registration_key_secrets_live_idx", + "environments_daemon_public_key_idx", + "environments_metadata_idx", + "environments_provider_state_idx", + "environments_provider_target_idx", + "environments_provisioned_open_idx", + "environments_registered_open_idx", + "environments_registration_key_state_idx", + "environments_request_id_idx", + "environments_work_idx", + ]; + expected_indexes.sort_unstable(); + let mut indexes: Vec = sqlx::query_scalar( + "SELECT indexname::text FROM pg_indexes + WHERE schemaname = current_schema() AND indexname = ANY($1::text[])", + ) + .bind(&expected_indexes) + .fetch_all(pool) + .await + .map_err(|error| error.to_string())?; + indexes.sort_unstable(); + if indexes != expected_indexes { + return Err(format!("environment baseline indexes differ: {indexes:?}")); + } + let incarnations: Option = + sqlx::query_scalar("SELECT to_regclass('environment_incarnations')::text") + .fetch_one(pool) + .await + .map_err(|error| error.to_string())?; + if incarnations.is_some() { + return Err("the environment baseline must not create incarnations".to_owned()); + } + let rows: Vec<(i64, String, String)> = sqlx::query_as("SELECT version, name, checksum FROM schema_migrations ORDER BY version") .fetch_all(pool) diff --git a/crates/store-pg/tests/store_pg_live.rs b/crates/store-pg/tests/store_pg_live.rs index 763041077..0bd14559f 100644 --- a/crates/store-pg/tests/store_pg_live.rs +++ b/crates/store-pg/tests/store_pg_live.rs @@ -7,14 +7,14 @@ use auth::{ OAuthClientStore, PutSecretRecord, SECRET_KIND_STATIC_BEARER, SecretId, SecretStore, SecretValue, TokenEndpointAuthMethod, state_hash, }; -use environment_protocol::shared::{EnvironmentTransport, ProviderTargetId}; +use environment_protocol::{ids::ProviderTargetId, routes::DataRouteBase}; use environments::{ - BeginCloseEnvironment, CreateEnvironment, EnvironmentConnectionSpec, EnvironmentId, - EnvironmentIncarnationId, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, - EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, - EnvironmentProvisionRequestId, EnvironmentStatus, EnvironmentStore, EnvironmentTemplateId, - ListEnvironmentProviders, ListEnvironments, ObserveProvisionedEnvironment, - PutEnvironmentProvider, PutEnvironmentProviderBinding, + BeginCloseEnvironment, CreateEnvironment, CreateExternalEnvironment, DaemonBuild, + EnvironmentCloseReason, EnvironmentEndpoint, EnvironmentId, EnvironmentProviderBindingStore, + EnvironmentProviderId, EnvironmentProviderStore, EnvironmentProvisionRequestId, + EnvironmentRegistryError, EnvironmentState, EnvironmentStore, EnvironmentTemplateId, + FinishCloseEnvironment, ListEnvironments, ProviderCredentialChange, PutEnvironmentProvider, + PutEnvironmentProviderBinding, }; use harness::{ BlobRef, CORE_AGENT_LIFECYCLE_CLOSED_EVENT_KIND, @@ -1993,242 +1993,6 @@ async fn pg_live_mcp_crud_and_universe_isolation() { )); } -#[tokio::test(flavor = "current_thread")] -#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] -async fn pg_live_universe_environments_are_independent_of_sessions() { - let store = live_store("environments", 1024).await; - for table in ["environment_jobs", "environment_job_groups"] { - let relation: Option = sqlx::query_scalar("SELECT to_regclass($1)::text") - .bind(format!("public.{table}")) - .fetch_one(store.pool()) - .await - .expect("inspect removed environment job table"); - assert!( - relation.is_none(), - "legacy job table must be removed: {table}" - ); - } - let provider_id = EnvironmentProviderId::new("bridge-local"); - let target_id = ProviderTargetId::new("local-host"); - let environment_id = EnvironmentId::new("instance-local"); - let session_id = SessionId::new("session-env"); - - let provider = store - .put_provider(PutEnvironmentProvider { - provider_id: provider_id.clone(), - display_name: Some("Local bridge".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "ws://127.0.0.1:9000/controller", - EnvironmentTransport::WebSocket, - ), - metadata: Default::default(), - updated_at_ms: 10, - }) - .await - .expect("register provider"); - // Providers are global rows shared by every test universe on this - // database, so assert membership rather than equality. - assert!( - store - .list_providers(ListEnvironmentProviders::default()) - .await - .expect("list providers") - .contains(&provider), - "registered provider must be listed" - ); - - store - .put_provider_binding(PutEnvironmentProviderBinding { - universe_id: store.config().universe_id, - binding_id: EnvironmentProviderBindingId::new("primary"), - provider_id: provider_id.clone(), - status: EnvironmentProviderBindingStatus::Enabled, - metadata: Default::default(), - expected_revision: None, - updated_at_ms: 25, - }) - .await - .expect("provider binding"); - store - .create_environment(CreateEnvironment { - request_id: EnvironmentProvisionRequestId::new("request-local"), - environment_id: environment_id.clone(), - incarnation_id: EnvironmentIncarnationId::new("incarnation-local"), - binding_id: EnvironmentProviderBindingId::new("primary"), - template_id: EnvironmentTemplateId::new("rust-v1"), - display_name: Some("Local host".to_owned()), - metadata: Default::default(), - - idle_policy: Some(environments::EnvironmentIdlePolicy { - pause_after_ms: Some(60_000), - suspend_after_ms: None, - stop_after_ms: Some(3_600_000), - close_after_ms: None, - }), - created_at_ms: 29, - }) - .await - .expect("create environment"); - let instance = store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: target_id.clone(), - status: EnvironmentStatus::Offline, - power_states: vec![ - environments::PowerState::Running, - environments::PowerState::Paused, - environments::PowerState::Stopped, - ], - observed_at_ms: 30, - }) - .await - .expect("upsert target"); - // Power intent, observed power states, and idle policy round-trip - // and feed the reconcile/reaper queries. - assert_eq!(instance.desired_power, environments::PowerState::Running); - assert_eq!( - instance.incarnation.power_states, - vec![ - environments::PowerState::Running, - environments::PowerState::Paused, - environments::PowerState::Stopped, - ] - ); - assert_eq!( - instance - .idle_policy - .as_ref() - .and_then(|policy| policy.pause_after_ms), - Some(60_000) - ); - // Offline while desired running: the reconciler must pick it up. - assert!( - store - .list_environments_needing_reconcile() - .await - .expect("needing reconcile") - .iter() - .any(|record| record.environment_id == environment_id) - ); - // Not ready: the reaper must not. - assert!( - store - .list_environments_with_idle_policy() - .await - .expect("idle policy candidates") - .is_empty() - ); - let stopped_intent = store - .set_environment_power(environments::SetEnvironmentPower { - environment_id: environment_id.clone(), - desired_power: environments::PowerState::Stopped, - updated_at_ms: 31, - }) - .await - .expect("set power"); - assert_eq!( - stopped_intent.desired_power, - environments::PowerState::Stopped - ); - assert!( - !store - .list_environments_needing_reconcile() - .await - .expect("needing reconcile") - .iter() - .any(|record| record.environment_id == environment_id) - ); - let ready = store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: target_id.clone(), - status: EnvironmentStatus::Ready, - power_states: vec![environments::PowerState::Running], - observed_at_ms: 32, - }) - .await - .expect("observe ready"); - assert_eq!( - store - .list_environments_with_idle_policy() - .await - .expect("idle policy candidates"), - vec![ready.clone()] - ); - let cleared = store - .set_environment_idle_policy(environments::SetEnvironmentIdlePolicy { - environment_id: environment_id.clone(), - idle_policy: None, - updated_at_ms: 33, - }) - .await - .expect("clear policy"); - assert!(cleared.idle_policy.is_none()); - store - .set_environment_power(environments::SetEnvironmentPower { - environment_id: environment_id.clone(), - desired_power: environments::PowerState::Running, - updated_at_ms: 34, - }) - .await - .expect("restore power"); - let instance = store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: target_id.clone(), - status: EnvironmentStatus::Offline, - power_states: vec![environments::PowerState::Running], - observed_at_ms: 35, - }) - .await - .expect("observe offline again"); - assert_eq!( - store - .list_environments(ListEnvironments { - metadata: Default::default(), - provider_id: Some(provider_id.clone()), - binding_id: Some(EnvironmentProviderBindingId::new("primary")), - status: Some(EnvironmentStatus::Offline), - ..ListEnvironments::default() - }) - .await - .expect("list instances"), - vec![instance.clone()] - ); - - store - .create_session(CreateSession { - metadata: Default::default(), - session_id: session_id.clone(), - display_name: None, - origin: None, - delete_after_close_ms: None, - created_at_ms: 35, - }) - .await - .expect("create session"); - - let closing = store - .begin_close_environment(BeginCloseEnvironment { - environment_id: environment_id.clone(), - updated_at_ms: 50, - }) - .await - .expect("close environment while a session exists"); - assert_eq!(closing.status, EnvironmentStatus::Closing); - - // Leave nothing behind for a running dev reconciler to chase: the - // provider endpoint is fictional and its closing environments would be - // retried forever. - store_pg::delete_universe(store.pool(), store.config().universe_id) - .await - .expect("delete test universe"); - store - .delete_provider(&provider_id) - .await - .expect("delete test provider"); -} - #[tokio::test(flavor = "current_thread")] #[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] async fn pg_live_auth_secrets_are_encrypted_and_universe_scoped() { @@ -2794,46 +2558,15 @@ async fn pg_live_environment_credentials_round_trip() { }; let store = live_store("environment-credentials", 1024).await; - let provider_id = EnvironmentProviderId::new("cred-provider"); + let provider_id = bound_test_provider(&store, "cred-provider").await; let environment_id = EnvironmentId::new("cred-env"); store - .put_provider(PutEnvironmentProvider { - provider_id: provider_id.clone(), - display_name: None, - controller_connection: EnvironmentConnectionSpec::new( - "ws://127.0.0.1:9000/controller", - EnvironmentTransport::WebSocket, - ), - metadata: Default::default(), - updated_at_ms: 1, - }) - .await - .expect("register provider"); - store - .put_provider_binding(PutEnvironmentProviderBinding { - universe_id: store.config().universe_id, - binding_id: EnvironmentProviderBindingId::new("primary"), - provider_id, - status: EnvironmentProviderBindingStatus::Enabled, - metadata: Default::default(), - expected_revision: None, - updated_at_ms: 2, - }) - .await - .expect("provider binding"); - store - .create_environment(CreateEnvironment { - request_id: EnvironmentProvisionRequestId::new("cred-request"), - environment_id: environment_id.clone(), - incarnation_id: EnvironmentIncarnationId::new("cred-incarnation"), - binding_id: EnvironmentProviderBindingId::new("primary"), - template_id: EnvironmentTemplateId::new("rust-v1"), - display_name: None, - metadata: Default::default(), - - idle_policy: None, - created_at_ms: 3, - }) + .create_environment(create_environment( + &provider_id, + "cred-request", + "cred-env", + 3, + )) .await .expect("create environment"); @@ -2946,6 +2679,115 @@ async fn pg_live_environment_credentials_round_trip() { ); } +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres env"] +async fn pg_live_universe_delete_removes_registered_environments_and_credential_bindings() { + use environments::{ + CreateEnvironmentRegistrationKey, CreateRegisteredEnvironment, EnvironmentCredentialSource, + EnvironmentCredentialStore, EnvironmentRegistrationKeyId, EnvironmentRegistrationKeyStore, + PutEnvironmentCredential, RegisteredIdentityMode, RegistrationKeyPolicy, + mint_registration_key, + }; + + let store = live_store("environment-universe-delete", 1024).await; + let universe_id = store.config().universe_id; + let minted = mint_registration_key( + EnvironmentRegistrationKeyId::new("fleet"), + RegistrationKeyPolicy { + display_name: "Fleet".to_owned(), + identity_mode: RegisteredIdentityMode::Persistent, + max_active_environments: None, + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, + expires_at_ms: None, + }, + 1, + ) + .expect("mint key"); + store + .create_registration_key(CreateEnvironmentRegistrationKey { + secret_hash: minted.secret.secret_hash.clone(), + record: minted.record.clone(), + }) + .await + .expect("create key"); + let environment_id = EnvironmentId::new("pod-0"); + store + .create_registered_environment(CreateRegisteredEnvironment { + registration_key_id: minted.record.registration_key_id.clone(), + secret_hash: minted.secret.secret_hash.clone(), + environment_id: environment_id.clone(), + daemon_public_key: test_daemon_public_key(), + display_name: None, + metadata: Default::default(), + daemon_build: test_build("1.0.0"), + created_at_ms: 2, + }) + .await + .expect("admit registered environment"); + let secret_id = SecretId::new("authsec_pod_token"); + store + .put_secret(PutSecretRecord { + secret_id: secret_id.clone(), + secret_kind: SECRET_KIND_STATIC_BEARER.to_owned(), + value: SecretValue::new("token"), + created_at_ms: 3, + }) + .await + .expect("put secret"); + store + .bind_credential(PutEnvironmentCredential { + environment_id: environment_id.clone(), + env_name: "POD_TOKEN".to_owned(), + source: EnvironmentCredentialSource::DirectSecret { + secret_id: secret_id.clone(), + }, + created_at_ms: 4, + }) + .await + .expect("bind credential"); + + // Outside a universe delete the references still hold. + assert!(matches!( + store.delete_secret(&secret_id).await, + Err(AuthRegistryError::Store { .. }) + )); + assert_eq!( + violated_constraint( + sqlx::query( + "DELETE FROM environment_registration_keys WHERE universe_id = $1 AND registration_key_id = 'fleet'", + ) + .bind(universe_id) + .execute(store.pool()) + .await + ) + .as_deref(), + Some("environments_registration_key_fk") + ); + + assert!( + store_pg::delete_universe(store.pool(), universe_id) + .await + .expect("delete universe with registered environment and credential binding") + ); + for table in [ + "environments", + "environment_registration_keys", + "environment_registration_key_secrets", + "environment_credentials", + "auth_secrets", + ] { + let remaining: i64 = sqlx::query_scalar(sqlx::AssertSqlSafe(format!( + "SELECT count(*) FROM {table} WHERE universe_id = $1" + ))) + .bind(universe_id) + .fetch_one(store.pool()) + .await + .expect("count remaining rows"); + assert_eq!(remaining, 0, "{table} cascades with the universe"); + } +} + #[tokio::test(flavor = "current_thread")] #[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] async fn pg_live_session_metadata_filters_by_containment_and_put_replaces() { @@ -3600,3 +3442,1680 @@ async fn pg_live_concurrent_puts_leave_one_readable_object() { ); assert_eq!(cold.read_bytes(&expected).await.expect("cold read"), bytes); } + +// --------------------------------------------------------------------------- +// Environments +// --------------------------------------------------------------------------- + +/// Register a provider with a unique id (providers are deployment-wide) and +/// bind it, enabled, to the store's universe. +async fn bound_test_provider(store: &PgStore, name: &str) -> EnvironmentProviderId { + let provider_id = EnvironmentProviderId::new(format!("{name}-{}", Uuid::new_v4().simple())); + store + .put_provider(PutEnvironmentProvider { + provider_id: provider_id.clone(), + display_name: None, + controller_endpoint: EnvironmentEndpoint::parse("ws://127.0.0.1:9100/control") + .expect("endpoint"), + data_route_base: None, + credential: ProviderCredentialChange::Keep, + updated_at_ms: 1, + }) + .await + .expect("register provider"); + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id: store.config().universe_id, + provider_id: provider_id.clone(), + enabled: true, + updated_at_ms: 2, + }) + .await + .expect("bind provider"); + provider_id +} + +fn create_environment( + provider_id: &EnvironmentProviderId, + request_id: &str, + environment_id: &str, + created_at_ms: i64, +) -> CreateEnvironment { + CreateEnvironment { + request_id: EnvironmentProvisionRequestId::new(request_id), + environment_id: EnvironmentId::new(environment_id), + provider_id: provider_id.clone(), + template_id: EnvironmentTemplateId::new("rust-v1"), + idle: None, + display_name: None, + metadata: Default::default(), + created_at_ms, + } +} + +fn test_build(version: &str) -> DaemonBuild { + DaemonBuild { + version: Some(version.to_owned()), + git_sha: None, + protocol_version: environment_protocol::ENVD_PROTOCOL_VERSION, + } +} + +/// A fresh lowercase-hex daemon public key; unique deployment-wide. +fn test_daemon_public_key() -> String { + format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()) +} + +/// The constraint a refused statement violated. +fn violated_constraint(result: Result) -> Option { + result.err().and_then(|error| { + error + .as_database_error() + .and_then(|database| database.constraint().map(str::to_owned)) + }) +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] +async fn pg_live_environment_provider_credentials_are_optional_and_sealed() { + let store = live_store("environment-providers", 1024).await; + let universe_id = store.config().universe_id; + let provider_id = EnvironmentProviderId::new(format!("provider-{}", Uuid::new_v4().simple())); + let endpoint = EnvironmentEndpoint::parse("ws://127.0.0.1:9100/control").expect("endpoint"); + let put = |display_name: Option<&str>, + data_route_base: Option, + credential: ProviderCredentialChange, + updated_at_ms: i64| PutEnvironmentProvider { + provider_id: provider_id.clone(), + display_name: display_name.map(str::to_owned), + controller_endpoint: endpoint.clone(), + data_route_base, + credential, + updated_at_ms, + }; + + // A new provider needs no credential, and its data route base derives + // from the controller endpoint. + let bare = store + .put_provider(put(None, None, ProviderCredentialChange::Keep, 1)) + .await + .expect("register provider without a credential"); + assert!(!bare.has_credential); + assert!( + store + .read_provider_credential(&provider_id) + .await + .expect("read absent credential") + .is_none() + ); + assert_eq!( + bare.effective_data_route_base() + .expect("derived data route base") + .as_str(), + "ws://127.0.0.1:9100/routes" + ); + + // A set credential is sealed at rest. + let base = DataRouteBase::new("ws://127.0.0.1:9100/data").expect("base"); + let sealed = store + .put_provider(put( + Some("Guard"), + Some(base.clone()), + ProviderCredentialChange::Set(SecretValue::new("guard-bearer-secret")), + 2, + )) + .await + .expect("set credential"); + assert!(sealed.has_credential); + assert_eq!(sealed.data_route_base, Some(base.clone())); + assert_eq!((sealed.created_at_ms, sealed.updated_at_ms), (1, 2)); + let ciphertext: Vec = sqlx::query_scalar( + "SELECT credential_ciphertext FROM environment_providers WHERE provider_id = $1", + ) + .bind(provider_id.as_str()) + .fetch_one(store.pool()) + .await + .expect("read sealed credential"); + assert!( + !ciphertext + .windows(b"guard-bearer-secret".len()) + .any(|window| window == b"guard-bearer-secret") + ); + + // Keep leaves the stored credential; Clear removes all three columns. + let renamed = store + .put_provider(put( + Some("Renamed"), + Some(base), + ProviderCredentialChange::Keep, + 3, + )) + .await + .expect("rename provider"); + assert!(renamed.has_credential); + assert_eq!( + store + .read_provider_credential(&provider_id) + .await + .expect("read credential") + .expect("credential kept") + .expose(), + "guard-bearer-secret" + ); + let cleared = store + .put_provider(put(None, None, ProviderCredentialChange::Clear, 4)) + .await + .expect("clear credential"); + assert!(!cleared.has_credential); + assert!(cleared.data_route_base.is_none()); + let columns: (Option, Option>, Option>) = sqlx::query_as( + "SELECT credential_key_id, credential_nonce, credential_ciphertext FROM environment_providers WHERE provider_id = $1", + ) + .bind(provider_id.as_str()) + .fetch_one(store.pool()) + .await + .expect("read credential columns"); + assert_eq!(columns, (None, None, None)); + assert!( + store + .list_providers() + .await + .expect("list providers") + .contains(&cleared) + ); + + // Bindings are keyed by provider; a missing provider is NotFound. + let binding = store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id, + provider_id: provider_id.clone(), + enabled: true, + updated_at_ms: 5, + }) + .await + .expect("bind provider"); + assert!(binding.enabled); + let disabled = store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id, + provider_id: provider_id.clone(), + enabled: false, + updated_at_ms: 6, + }) + .await + .expect("disable binding"); + assert!(!disabled.enabled); + assert_eq!((disabled.created_at_ms, disabled.updated_at_ms), (5, 6)); + assert_eq!( + store + .read_provider_binding(universe_id, &provider_id) + .await + .expect("read binding"), + disabled + ); + assert_eq!( + store + .list_provider_bindings(universe_id) + .await + .expect("list bindings"), + vec![disabled] + ); + assert!(matches!( + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id, + provider_id: EnvironmentProviderId::new(format!( + "missing-{}", + Uuid::new_v4().simple() + )), + enabled: true, + updated_at_ms: 6, + }) + .await, + Err(EnvironmentRegistryError::NotFound { + kind: "environment_provider", + .. + }) + )); + + // Creates need an enabled binding; deletes are guarded. + assert!(matches!( + store + .create_environment(create_environment( + &provider_id, + "provider-request", + "provider-env", + 10, + )) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id, + provider_id: provider_id.clone(), + enabled: true, + updated_at_ms: 7, + }) + .await + .expect("enable binding"); + let environment = store + .create_environment(create_environment( + &provider_id, + "provider-request", + "provider-env", + 10, + )) + .await + .expect("create environment"); + assert!(matches!( + store.delete_provider(&provider_id).await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + assert!(matches!( + store + .delete_provider_binding(universe_id, &provider_id) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: environment.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 11, + }) + .await + .expect("begin close"); + let closed = store + .finish_close_environment(FinishCloseEnvironment { + environment_id: environment.environment_id.clone(), + at_ms: 12, + }) + .await + .expect("finish close"); + assert_eq!(closed.state, EnvironmentState::Closed); + store + .delete_provider_binding(universe_id, &provider_id) + .await + .expect("delete binding once its environments are closed"); + store + .delete_provider(&provider_id) + .await + .expect("closed environments do not hold a provider"); + + // The all-or-none credential CHECK names its NULL case explicitly. + let orphan_id = format!("check-{}", Uuid::new_v4().simple()); + let refused = sqlx::query( + "INSERT INTO environment_providers (provider_id, controller_endpoint, credential_nonce, credential_ciphertext, created_at_ms, updated_at_ms) VALUES ($1, 'ws://127.0.0.1:1/control', $2, $3, 1, 1)", + ) + .bind(&orphan_id) + .bind(vec![0_u8; 12]) + .bind(vec![1_u8; 16]) + .execute(store.pool()) + .await; + assert_eq!( + violated_constraint(refused).as_deref(), + Some("environment_providers_credential_all_or_none") + ); +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] +async fn pg_live_provisioned_environment_lifecycle_follows_the_domain_rules() { + use environments::{ + EndProvisionedEnvironment, EnvironmentDisplayStatus, FailEnvironment, IdlePolicy, + ObservedTarget, OpenEnvironment, PowerState, RecordTargetObservation, + ScheduleEnvironmentAttempt, TargetPhase, UpdateEnvironment, + }; + + let store = live_store("environment-lifecycle", 1024).await; + let provider_id = bound_test_provider(&store, "lifecycle").await; + let idle = IdlePolicy { + pause_after_ms: Some(60_000), + ..IdlePolicy::default() + }; + let request = CreateEnvironment { + idle: Some(idle.clone()), + display_name: Some("Sandbox".to_owned()), + ..create_environment(&provider_id, "lifecycle-a", "lifecycle-a", 100) + }; + + // Creates are idempotent per request id with identical inputs. + let created = store + .create_environment(request.clone()) + .await + .expect("create"); + assert_eq!(created.state, EnvironmentState::Provisioning); + let source = created.provisioned().expect("provisioned").clone(); + assert_eq!(source.idle, Some(idle)); + assert_eq!((source.next_attempt_at_ms, source.attempts), (Some(100), 0)); + assert_eq!( + store + .create_environment(CreateEnvironment { + created_at_ms: 200, + ..request.clone() + }) + .await + .expect("retry create"), + created + ); + assert!(matches!( + store + .create_environment(CreateEnvironment { + template_id: EnvironmentTemplateId::new("other-template"), + ..request.clone() + }) + .await, + Err(EnvironmentRegistryError::AlreadyExists { + kind: "environment_request", + .. + }) + )); + assert_eq!( + store + .read_environment(&created.environment_id) + .await + .expect("read back"), + created + ); + assert!(matches!( + store + .create_environment(create_environment( + &EnvironmentProviderId::new(format!("unbound-{}", Uuid::new_v4().simple())), + "lifecycle-unbound", + "lifecycle-unbound", + 100, + )) + .await, + Err(EnvironmentRegistryError::NotFound { + kind: "environment_provider_binding", + .. + }) + )); + + // The work queue holds due provisioning rows only. + let due = |rows: Vec| { + rows.iter() + .any(|record| record.environment_id == created.environment_id) + }; + assert!(!due(store + .list_environment_work(99, 10) + .await + .expect("work"))); + assert!(due(store + .list_environment_work(100, 10) + .await + .expect("work"))); + let scheduled = store + .schedule_environment_attempt(ScheduleEnvironmentAttempt { + environment_id: created.environment_id.clone(), + expected_state: EnvironmentState::Provisioning, + next_attempt_at_ms: 2_100, + attempts: 1, + state_message: Some("waiting for the provider: booting".to_owned()), + at_ms: 101, + }) + .await + .expect("schedule"); + let source = scheduled.provisioned().expect("provisioned"); + assert_eq!( + (source.next_attempt_at_ms, source.attempts), + (Some(2_100), 1) + ); + assert_eq!( + scheduled.state_message.as_deref(), + Some("waiting for the provider: booting") + ); + assert!(!due(store + .list_environment_work(2_099, 10) + .await + .expect("work"))); + + // Observations record the target id and keep the freshest report. + let target = ProviderTargetId::new("pod-a").expect("target id"); + let observe = |target_id: Option, phase, power, at| RecordTargetObservation { + environment_id: created.environment_id.clone(), + provider_target_id: target_id, + observed: ObservedTarget { + phase, + power, + message: None, + observed_at_ms: 0, + }, + public_endpoint: Some("https://a.example.test".to_owned()), + observed_at_ms: at, + }; + let observed = store + .record_target_observation(observe( + Some(target.clone()), + TargetPhase::Active, + Some(PowerState::Running), + 102, + )) + .await + .expect("observe"); + let source = observed.provisioned().expect("provisioned"); + assert_eq!(source.provider_target_id, Some(target.clone())); + assert_eq!( + source + .observed + .as_ref() + .map(|observed| observed.observed_at_ms), + Some(102) + ); + assert_eq!( + source.public_endpoint.as_deref(), + Some("https://a.example.test") + ); + let stale = store + .record_target_observation(observe(None, TargetPhase::Provisioning, None, 90)) + .await + .expect("stale observation"); + assert_eq!(stale, observed); + assert!(matches!( + store + .record_target_observation(observe( + Some(ProviderTargetId::new("pod-other").expect("target id")), + TargetPhase::Active, + None, + 103, + )) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + let second = store + .create_environment(create_environment( + &provider_id, + "lifecycle-b", + "lifecycle-b", + 100, + )) + .await + .expect("create second"); + assert!(matches!( + store + .record_target_observation(RecordTargetObservation { + environment_id: second.environment_id.clone(), + ..observe(Some(target.clone()), TargetPhase::Active, None, 103) + }) + .await, + Err(EnvironmentRegistryError::AlreadyExists { + kind: "provider_target", + .. + }) + )); + + // Open clears the schedule and records the build; a second open skips. + let opened = store + .open_environment(OpenEnvironment { + environment_id: created.environment_id.clone(), + daemon_build: test_build("1.2.3"), + at_ms: 104, + }) + .await + .expect("open"); + assert_eq!(opened.state, EnvironmentState::Open); + assert_eq!(opened.state_message, None); + assert_eq!(opened.daemon_build, Some(test_build("1.2.3"))); + let source = opened.provisioned().expect("provisioned"); + assert_eq!((source.next_attempt_at_ms, source.attempts), (None, 0)); + assert_eq!(opened.display_status(), EnvironmentDisplayStatus::Ready); + assert!( + store + .list_open_provisioned_environments() + .await + .expect("open provisioned") + .contains(&opened) + ); + assert_eq!( + store + .open_environment(OpenEnvironment { + environment_id: created.environment_id.clone(), + daemon_build: test_build("9.9.9"), + at_ms: 105, + }) + .await + .expect("open again"), + opened + ); + + // Fail keeps the build; the provider ending the target closes it. + let failed = store + .fail_environment(FailEnvironment { + environment_id: created.environment_id.clone(), + message: "broken".to_owned(), + daemon_build: None, + at_ms: 106, + }) + .await + .expect("fail"); + assert_eq!(failed.state, EnvironmentState::Failed); + assert_eq!(failed.state_message.as_deref(), Some("broken")); + assert_eq!(failed.daemon_build, Some(test_build("1.2.3"))); + let ended = store + .end_provisioned_environment(EndProvisionedEnvironment { + environment_id: created.environment_id.clone(), + at_ms: 107, + }) + .await + .expect("end"); + assert_eq!(ended.state, EnvironmentState::Closed); + assert_eq!( + ended.close_reason, + Some(EnvironmentCloseReason::ProviderEnded) + ); + assert_eq!( + (ended.closed_at_ms, ended.state_message.as_deref()), + (Some(107), None) + ); + // A closed row keeps no public endpoint, even after a late observation. + assert_eq!( + ended.provisioned().expect("provisioned").public_endpoint, + None + ); + let late = store + .record_target_observation(observe(None, TargetPhase::Closed, None, 108)) + .await + .expect("late observation"); + assert_eq!( + late.provisioned().expect("provisioned").public_endpoint, + None + ); + assert!(matches!( + store + .update_environment(UpdateEnvironment { + environment_id: created.environment_id.clone(), + display_name: None, + metadata: Default::default(), + updated_at_ms: 108, + }) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + + // A close accepted while a create was in flight wins over its answer. + let closing = store + .begin_close_environment(BeginCloseEnvironment { + environment_id: second.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 110, + }) + .await + .expect("begin close"); + assert_eq!(closing.state, EnvironmentState::Closing); + assert_eq!( + closing + .provisioned() + .and_then(|source| source.next_attempt_at_ms), + Some(110) + ); + assert_eq!( + store + .open_environment(OpenEnvironment { + environment_id: second.environment_id.clone(), + daemon_build: test_build("1.2.3"), + at_ms: 111, + }) + .await + .expect("late open"), + closing + ); + assert_eq!( + store + .fail_environment(FailEnvironment { + environment_id: second.environment_id.clone(), + message: "late failure".to_owned(), + daemon_build: None, + at_ms: 111, + }) + .await + .expect("late failure"), + closing + ); + let finished = store + .finish_close_environment(FinishCloseEnvironment { + environment_id: second.environment_id.clone(), + at_ms: 112, + }) + .await + .expect("finish close"); + assert_eq!( + finished.close_reason, + Some(EnvironmentCloseReason::Requested) + ); + assert_eq!(finished.closed_at_ms, Some(112)); + + // A lagging clock is clamped to the row's own times. + let late = store + .create_environment(create_environment( + &provider_id, + "lifecycle-c", + "lifecycle-c", + 500, + )) + .await + .expect("create third"); + let clamped = store + .begin_close_environment(BeginCloseEnvironment { + environment_id: late.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 10, + }) + .await + .expect("close with a lagging clock"); + assert_eq!(clamped.state_changed_at_ms, 500); + assert_eq!( + clamped + .provisioned() + .and_then(|source| source.next_attempt_at_ms), + Some(500) + ); + + // Updates replace the display name and caller metadata. + let renamed = store + .create_environment(create_environment( + &provider_id, + "lifecycle-d", + "lifecycle-d", + 600, + )) + .await + .expect("create fourth"); + let updated = store + .update_environment(UpdateEnvironment { + environment_id: renamed.environment_id.clone(), + display_name: Some("Renamed".to_owned()), + metadata: [("team".to_owned(), "infra".to_owned())].into(), + updated_at_ms: 601, + }) + .await + .expect("update"); + assert_eq!(updated.display_name.as_deref(), Some("Renamed")); + assert_eq!( + updated.metadata.get("team").map(String::as_str), + Some("infra") + ); + assert!(matches!( + store + .update_environment(UpdateEnvironment { + environment_id: renamed.environment_id.clone(), + display_name: None, + metadata: [("lightspeed.reserved".to_owned(), "x".to_owned())].into(), + updated_at_ms: 602, + }) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + + // External environments are open at once and close in one step. + let external = CreateExternalEnvironment { + request_id: EnvironmentProvisionRequestId::new("lifecycle-external"), + environment_id: EnvironmentId::new("lifecycle-external"), + endpoint: EnvironmentEndpoint::parse("ws://127.0.0.1:9200/envd").expect("endpoint"), + display_name: None, + metadata: Default::default(), + created_at_ms: 700, + }; + let opened_external = store + .create_external_environment(external.clone()) + .await + .expect("create external"); + assert_eq!(opened_external.state, EnvironmentState::Open); + assert_eq!( + store + .create_external_environment(external.clone()) + .await + .expect("retry external"), + opened_external + ); + assert!(matches!( + store + .fail_environment(FailEnvironment { + environment_id: external.environment_id.clone(), + message: "external".to_owned(), + daemon_build: None, + at_ms: 701, + }) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + let closed_external = store + .begin_close_environment(BeginCloseEnvironment { + environment_id: external.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 702, + }) + .await + .expect("close external"); + assert_eq!(closed_external.state, EnvironmentState::Closed); + assert_eq!(closed_external.closed_at_ms, Some(702)); + + // Leave nothing for a running dev reconciler to chase. + store_pg::delete_universe(store.pool(), store.config().universe_id) + .await + .expect("delete test universe"); + store + .delete_provider(&provider_id) + .await + .expect("delete test provider"); +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] +async fn pg_live_environment_listing_pages_filters_and_skips_bad_rows() { + let store = live_store("environment-listing", 1024).await; + let provider_id = bound_test_provider(&store, "listing").await; + for index in 1..=5 { + let team = if index % 2 == 1 { "a" } else { "b" }; + store + .create_environment(CreateEnvironment { + metadata: [("team".to_owned(), team.to_owned())].into(), + ..create_environment( + &provider_id, + &format!("list-request-{index}"), + &format!("list-{index}"), + 10, + ) + }) + .await + .expect("create listed environment"); + } + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: EnvironmentId::new("list-2"), + reason: EnvironmentCloseReason::Requested, + at_ms: 11, + }) + .await + .expect("begin close"); + store + .finish_close_environment(FinishCloseEnvironment { + environment_id: EnvironmentId::new("list-2"), + at_ms: 12, + }) + .await + .expect("finish close"); + let ids = |page: &environments::EnvironmentPage| { + page.environments + .iter() + .map(|record| record.environment_id.as_str().to_owned()) + .collect::>() + }; + + // Closed rows are hidden by default; pages continue after an id. + let first = store + .list_environments(ListEnvironments { + limit: 2, + ..ListEnvironments::default() + }) + .await + .expect("first page"); + assert_eq!(ids(&first), ["list-1", "list-3"]); + assert_eq!(first.next_after, Some(EnvironmentId::new("list-3"))); + let second = store + .list_environments(ListEnvironments { + limit: 2, + after: first.next_after.clone(), + ..ListEnvironments::default() + }) + .await + .expect("second page"); + assert_eq!(ids(&second), ["list-4", "list-5"]); + assert_eq!(second.next_after, None); + let all = store + .list_environments(ListEnvironments { + include_closed: true, + ..ListEnvironments::default() + }) + .await + .expect("include closed"); + assert_eq!(all.environments.len(), 5); + let closed = store + .list_environments(ListEnvironments { + state: Some(EnvironmentState::Closed), + ..ListEnvironments::default() + }) + .await + .expect("closed only"); + assert_eq!(ids(&closed), ["list-2"]); + let team_a = store + .list_environments(ListEnvironments { + metadata: [("team".to_owned(), "a".to_owned())].into(), + ..ListEnvironments::default() + }) + .await + .expect("by metadata"); + assert_eq!(ids(&team_a), ["list-1", "list-3", "list-5"]); + let by_provider = store + .list_environments(ListEnvironments { + provider_id: Some(provider_id.clone()), + ..ListEnvironments::default() + }) + .await + .expect("by provider"); + assert_eq!(ids(&by_provider), ["list-1", "list-3", "list-4", "list-5"]); + let other_provider = store + .list_environments(ListEnvironments { + provider_id: Some(EnvironmentProviderId::new("listing-other")), + ..ListEnvironments::default() + }) + .await + .expect("by another provider"); + assert!(other_provider.environments.is_empty()); + for limit in [0, 501] { + assert!(matches!( + store + .list_environments(ListEnvironments { + limit, + ..ListEnvironments::default() + }) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + } + + // A row that no longer decodes is skipped by listings instead of + // failing them. + let oversized: serde_json::Map = (0..40) + .map(|index| (format!("key{index}"), serde_json::json!("v"))) + .collect(); + sqlx::query( + "UPDATE environments SET metadata_json = $3 WHERE universe_id = $1 AND environment_id = $2", + ) + .bind(store.config().universe_id) + .bind("list-4") + .bind(serde_json::Value::Object(oversized)) + .execute(store.pool()) + .await + .expect("poison row"); + let listed = store + .list_environments(ListEnvironments::default()) + .await + .expect("list despite a bad row"); + assert_eq!(ids(&listed), ["list-1", "list-3", "list-5"]); + + store_pg::delete_universe(store.pool(), store.config().universe_id) + .await + .expect("delete test universe"); + store + .delete_provider(&provider_id) + .await + .expect("delete test provider"); +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] +async fn pg_live_environment_checks_refuse_null_positive_disjuncts() { + use environments::{ + CreateEnvironmentRegistrationKey, EnvironmentRegistrationKeyId, + EnvironmentRegistrationKeyStore, RegisteredIdentityMode, RegistrationKeyPolicy, + mint_registration_key, + }; + + let store = live_store("environment-checks", 1024).await; + let universe_id = store.config().universe_id; + let minted = mint_registration_key( + EnvironmentRegistrationKeyId::new("checks"), + RegistrationKeyPolicy { + display_name: "Checks".to_owned(), + identity_mode: RegisteredIdentityMode::Persistent, + max_active_environments: None, + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, + expires_at_ms: None, + }, + 1, + ) + .expect("mint key"); + store + .create_registration_key(CreateEnvironmentRegistrationKey { + secret_hash: minted.secret.secret_hash.clone(), + record: minted.record.clone(), + }) + .await + .expect("create key"); + + let external = sqlx::query( + "INSERT INTO environments (universe_id, environment_id, source_kind, state, state_changed_at_ms, request_id, created_at_ms, updated_at_ms) VALUES ($1, 'check-external', 'external', 'open', 1, 'check-external', 1, 1)", + ) + .bind(universe_id) + .execute(store.pool()) + .await; + assert_eq!( + violated_constraint(external).as_deref(), + Some("environments_source_fields") + ); + + let public_key = test_daemon_public_key(); + for (missing, daemon_public_key, connection_changed_at_ms, last_seen_at_ms) in [ + ("daemon_public_key", None, Some(1_i64), Some(1_i64)), + ( + "connection_changed_at_ms", + Some(public_key.as_str()), + None, + Some(1), + ), + ("last_seen_at_ms", Some(public_key.as_str()), Some(1), None), + ] { + let registered = sqlx::query( + "INSERT INTO environments (universe_id, environment_id, source_kind, state, state_changed_at_ms, registration_key_id, daemon_public_key, connected, connection_changed_at_ms, last_seen_at_ms, created_at_ms, updated_at_ms) VALUES ($1, $2, 'registered', 'open', 1, 'checks', $3, true, $4, $5, 1, 1)", + ) + .bind(universe_id) + .bind(format!("check-registered-{missing}").replace('_', "-")) + .bind(daemon_public_key) + .bind(connection_changed_at_ms) + .bind(last_seen_at_ms) + .execute(store.pool()) + .await; + assert_eq!( + violated_constraint(registered).as_deref(), + Some("environments_source_fields"), + "a registered row without {missing} must be refused" + ); + } + + let observed = sqlx::query( + "INSERT INTO environments (universe_id, environment_id, source_kind, state, state_changed_at_ms, request_id, provider_id, template_id, next_attempt_at_ms, observed_power, created_at_ms, updated_at_ms) VALUES ($1, 'check-observed', 'provisioned', 'provisioning', 1, 'check-observed', 'check-provider', 'rust-v1', 1, 'running', 1, 1)", + ) + .bind(universe_id) + .execute(store.pool()) + .await; + assert_eq!( + violated_constraint(observed).as_deref(), + Some("environments_observation") + ); +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] +async fn pg_live_registered_environments_track_connections_and_sweep() { + use environments::{ + CreateEnvironmentRegistrationKey, CreateRegisteredEnvironment, + EnvironmentRegistrationKeyId, EnvironmentRegistrationKeyStore, EnvironmentUse, + EnvironmentWaitReason, MarkRegisteredConnected, MarkRegisteredDisconnected, + REGISTERED_RECONNECT_WAIT_MS, RegisteredIdentityMode, RegistrationKeyPolicy, + StampRegisteredSeen, mint_registration_key, + }; + + let store = live_store("environment-registered", 1024).await; + let universe_id = store.config().universe_id; + let create_key = |id: &str, identity_mode, grace, offline_close| { + mint_registration_key( + EnvironmentRegistrationKeyId::new(id), + RegistrationKeyPolicy { + display_name: id.to_owned(), + identity_mode, + max_active_environments: Some(2), + ephemeral_disconnect_grace_ms: grace, + offline_close_after_ms: offline_close, + expires_at_ms: None, + }, + 1, + ) + .expect("mint key") + }; + let mut secrets = std::collections::BTreeMap::new(); + for minted in [ + create_key( + "fleet", + RegisteredIdentityMode::Persistent, + None, + Some(3_600_000), + ), + create_key( + "sandboxes", + RegisteredIdentityMode::Ephemeral, + Some(60_000), + None, + ), + ] { + store + .create_registration_key(CreateEnvironmentRegistrationKey { + secret_hash: minted.secret.secret_hash.clone(), + record: minted.record.clone(), + }) + .await + .expect("create key"); + secrets.insert( + minted.record.registration_key_id.to_string(), + minted.secret.secret_hash, + ); + } + let admit = + |key: &str, environment_id: &str, public_key: &str, at: i64| CreateRegisteredEnvironment { + registration_key_id: EnvironmentRegistrationKeyId::new(key), + secret_hash: secrets[key].clone(), + environment_id: EnvironmentId::new(environment_id), + daemon_public_key: public_key.to_owned(), + display_name: Some(environment_id.to_owned()), + metadata: Default::default(), + daemon_build: test_build("1.0.0"), + created_at_ms: at, + }; + + // Admission inserts an open, connected row; a known identity converges. + let key_0 = test_daemon_public_key(); + let pod_0 = store + .create_registered_environment(admit("fleet", "pod-0", &key_0, 100)) + .await + .expect("admit pod-0"); + assert_eq!(pod_0.state, EnvironmentState::Open); + assert_eq!(pod_0.request_id, None); + let source = pod_0.registered().expect("registered").clone(); + assert!(source.connected); + assert_eq!(source.identity_mode, RegisteredIdentityMode::Persistent); + assert_eq!( + (source.connection_changed_at_ms, source.last_seen_at_ms), + (100, 100) + ); + assert_eq!( + store + .create_registered_environment(admit("fleet", "pod-0-retry", &key_0, 200)) + .await + .expect("retry admission"), + pod_0 + ); + assert_eq!( + store + .read_environment_by_daemon_public_key(&key_0) + .await + .expect("read by daemon key"), + Some(pod_0.clone()) + ); + assert_eq!( + store_pg::find_registered_environment_universe(store.pool(), &key_0) + .await + .expect("find universe by daemon key"), + Some(universe_id) + ); + store + .create_registered_environment(admit("fleet", "pod-1", &test_daemon_public_key(), 150)) + .await + .expect("admit pod-1"); + assert!(matches!( + store + .create_registered_environment(admit("fleet", "pod-2", &test_daemon_public_key(), 160,)) + .await, + Err(EnvironmentRegistryError::RegistrationCapacityExhausted { limit: 2, .. }) + )); + let listed_universes = store_pg::list_universes_with_registered_environments(store.pool()) + .await + .expect("universes with registered environments"); + assert!(listed_universes.contains(&universe_id)); + + // The liveness stamp is coarse and conditional. + let stamp = |at_ms| StampRegisteredSeen { + environment_id: EnvironmentId::new("pod-0"), + at_ms, + }; + assert!( + !store + .stamp_registered_seen(stamp(300_099)) + .await + .expect("early stamp") + ); + assert!( + store + .stamp_registered_seen(stamp(300_100)) + .await + .expect("due stamp") + ); + assert_eq!( + store + .read_environment(&EnvironmentId::new("pod-0")) + .await + .expect("read stamped") + .registered() + .map(|source| source.last_seen_at_ms), + Some(300_100) + ); + assert!(matches!( + store + .stamp_registered_seen(StampRegisteredSeen { + environment_id: EnvironmentId::new("missing"), + at_ms: 1, + }) + .await, + Err(EnvironmentRegistryError::NotFound { .. }) + )); + + // Disconnects and reconnects are written on the transition only. + let disconnected = store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: EnvironmentId::new("pod-0"), + at_ms: 400_000, + }) + .await + .expect("disconnect"); + let source = disconnected.registered().expect("registered"); + assert!(!source.connected); + assert_eq!(source.connection_changed_at_ms, 400_000); + assert_eq!( + store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: EnvironmentId::new("pod-0"), + at_ms: 400_500, + }) + .await + .expect("disconnect again"), + disconnected + ); + assert_eq!( + disconnected.use_gate(400_001), + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Disconnected, + retry_after_ms: environments::WAIT_RETRY_AFTER_MS, + until_ms: Some(400_000 + REGISTERED_RECONNECT_WAIT_MS), + } + ); + assert!( + !store + .stamp_registered_seen(stamp(900_000)) + .await + .expect("stamp while away") + ); + let reconnected = store + .mark_registered_connected(MarkRegisteredConnected { + environment_id: EnvironmentId::new("pod-0"), + daemon_build: test_build("1.1.0"), + at_ms: 410_000, + }) + .await + .expect("reconnect"); + let source = reconnected.registered().expect("registered"); + assert!(source.connected); + assert_eq!( + (source.connection_changed_at_ms, source.last_seen_at_ms), + (410_000, 410_000) + ); + assert_eq!(reconnected.daemon_build, Some(test_build("1.1.0"))); + + // A gateway restart disconnects every connected row as of now. + assert_eq!( + store + .repair_registered_connections(500_000) + .await + .expect("repair"), + 2 + ); + let repaired = store + .read_environment(&EnvironmentId::new("pod-0")) + .await + .expect("read repaired"); + let source = repaired.registered().expect("registered"); + assert!(!source.connected); + assert_eq!( + (source.connection_changed_at_ms, source.last_seen_at_ms), + (500_000, 410_000) + ); + + // Sweep rule 2: an ephemeral daemon away past its grace closes. + store + .create_registered_environment(admit( + "sandboxes", + "sbx-0", + &test_daemon_public_key(), + 1_000_000, + )) + .await + .expect("admit sbx-0"); + store + .create_registered_environment(admit( + "sandboxes", + "sbx-1", + &test_daemon_public_key(), + 1_000_000, + )) + .await + .expect("admit sbx-1"); + store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: EnvironmentId::new("sbx-0"), + at_ms: 1_000_000, + }) + .await + .expect("disconnect sbx-0"); + let sweep = store + .sweep_registered_environments(1_060_001) + .await + .expect("first sweep"); + assert!(sweep.disconnected.is_empty()); + assert_eq!( + sweep.closed, + vec![( + EnvironmentId::new("sbx-0"), + EnvironmentCloseReason::EphemeralGrace + )] + ); + + // Rule 1 flips a stale connected row, which rule 2 then closes; rule 3 + // closes persistent rows away past the key's offline close-after. + let sweep = store + .sweep_registered_environments(4_200_000) + .await + .expect("second sweep"); + assert_eq!(sweep.disconnected, vec![EnvironmentId::new("sbx-1")]); + assert_eq!( + sweep.closed, + vec![ + ( + EnvironmentId::new("pod-0"), + EnvironmentCloseReason::OfflineTimeout + ), + ( + EnvironmentId::new("pod-1"), + EnvironmentCloseReason::OfflineTimeout + ), + ( + EnvironmentId::new("sbx-1"), + EnvironmentCloseReason::EphemeralGrace + ), + ] + ); + let closed = store + .read_environment(&EnvironmentId::new("pod-0")) + .await + .expect("read swept"); + assert_eq!(closed.state, EnvironmentState::Closed); + assert_eq!(closed.closed_at_ms, Some(4_200_000)); + assert!( + store + .sweep_registered_environments(4_200_000) + .await + .expect("idle sweep") + .closed + .is_empty() + ); + + // A closed identity is spent. + assert!(matches!( + store + .mark_registered_connected(MarkRegisteredConnected { + environment_id: EnvironmentId::new("pod-0"), + daemon_build: test_build("1.1.0"), + at_ms: 4_300_000, + }) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + assert_eq!( + store + .create_registered_environment(admit("fleet", "pod-0-again", &key_0, 4_300_000)) + .await + .expect("spent identity converges"), + closed + ); + let usage = store + .registration_key_usage(&EnvironmentRegistrationKeyId::new("fleet")) + .await + .expect("usage"); + assert_eq!((usage.registered, usage.active), (2, 0)); +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] +async fn pg_live_registration_keys_rotate_update_and_resolve_live_secrets() { + use environments::{ + CreateEnvironmentRegistrationKey, CreateRegisteredEnvironment, + EnvironmentRegistrationKeyId, EnvironmentRegistrationKeyStore, RegisteredIdentityMode, + RegistrationKeyPolicy, RegistrationKeySettings, RevokeEnvironmentRegistrationKey, + RotateEnvironmentRegistrationKey, UpdateEnvironmentRegistrationKey, mint_registration_key, + mint_registration_secret, + }; + + let store = live_store("environment-keys", 1024).await; + let universe_id = store.config().universe_id; + let key_id = EnvironmentRegistrationKeyId::new("rotating"); + let policy = |identity_mode| RegistrationKeyPolicy { + display_name: "Rotating".to_owned(), + identity_mode, + max_active_environments: None, + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, + expires_at_ms: None, + }; + let minted = mint_registration_key( + key_id.clone(), + policy(RegisteredIdentityMode::Persistent), + 1, + ) + .expect("mint key"); + let created = store + .create_registration_key(CreateEnvironmentRegistrationKey { + secret_hash: minted.secret.secret_hash.clone(), + record: minted.record.clone(), + }) + .await + .expect("create key"); + assert_eq!(created, minted.record); + assert_eq!( + store + .resolve_registration_key(&minted.secret.secret_hash) + .await + .expect("resolve"), + Some(created.clone()) + ); + assert_eq!( + store_pg::find_registration_key_universe(store.pool(), &minted.secret.secret_hash) + .await + .expect("find universe"), + Some(universe_id) + ); + assert!(matches!( + store + .create_registration_key(CreateEnvironmentRegistrationKey { + secret_hash: mint_registration_secret().secret_hash, + record: minted.record.clone(), + }) + .await, + Err(EnvironmentRegistryError::AlreadyExists { + kind: "environment_registration_key", + .. + }) + )); + + // Rotation retires the live secret and keeps the key. + let rotated_secret = mint_registration_secret(); + let rotated = store + .rotate_registration_key(RotateEnvironmentRegistrationKey { + registration_key_id: key_id.clone(), + secret_hash: rotated_secret.secret_hash.clone(), + key_prefix: rotated_secret.key_prefix.clone(), + rotated_at_ms: 10, + }) + .await + .expect("rotate"); + assert_eq!(rotated.key_prefix, rotated_secret.key_prefix); + assert_eq!((rotated.created_at_ms, rotated.updated_at_ms), (1, 10)); + assert_eq!( + store + .read_registration_key(&key_id) + .await + .expect("read rotated"), + rotated + ); + assert_eq!( + store + .resolve_registration_key(&minted.secret.secret_hash) + .await + .expect("resolve retired"), + None + ); + assert_eq!( + store + .resolve_registration_key(&rotated_secret.secret_hash) + .await + .expect("resolve live"), + Some(rotated.clone()) + ); + assert_eq!( + store_pg::find_registration_key_universe(store.pool(), &minted.secret.secret_hash) + .await + .expect("find by retired secret"), + None + ); + // An admission that resolved the key before the rotation committed + // presents the retired secret: refused inside the key-locked + // transaction. The live secret admits. + let admit = |secret_hash: &str| CreateRegisteredEnvironment { + registration_key_id: key_id.clone(), + secret_hash: secret_hash.to_owned(), + environment_id: EnvironmentId::new("rotating-pod"), + daemon_public_key: test_daemon_public_key(), + display_name: None, + metadata: Default::default(), + daemon_build: test_build("1.0.0"), + created_at_ms: 12, + }; + assert!(matches!( + store + .create_registered_environment(admit(&minted.secret.secret_hash)) + .await, + Err(EnvironmentRegistryError::RegistrationKeyUnavailable { + reason: "rotated", + .. + }) + )); + assert_eq!( + store + .create_registered_environment(admit(&rotated_secret.secret_hash)) + .await + .expect("admit with the live secret") + .state, + EnvironmentState::Open + ); + assert!(matches!( + store + .rotate_registration_key(RotateEnvironmentRegistrationKey { + registration_key_id: key_id.clone(), + secret_hash: minted.secret.secret_hash.clone(), + key_prefix: minted.secret.key_prefix.clone(), + rotated_at_ms: 11, + }) + .await, + Err(EnvironmentRegistryError::AlreadyExists { + kind: "environment_registration_key_secret", + .. + }) + )); + + // Updates replace the editable policy; the identity mode bounds it. + let updated = store + .update_registration_key(UpdateEnvironmentRegistrationKey { + registration_key_id: key_id.clone(), + settings: RegistrationKeySettings { + display_name: "Renamed".to_owned(), + max_active_environments: Some(5), + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: Some(60_000), + expires_at_ms: Some(1_000_000), + }, + updated_at_ms: 20, + }) + .await + .expect("update"); + assert_eq!(updated.display_name, "Renamed"); + assert_eq!(updated.offline_close_after_ms, Some(60_000)); + assert_eq!(updated.updated_at_ms, 20); + assert_eq!( + store + .read_registration_key(&key_id) + .await + .expect("read updated"), + updated + ); + let ephemeral_id = EnvironmentRegistrationKeyId::new("ephemeral"); + let ephemeral = mint_registration_key( + ephemeral_id.clone(), + policy(RegisteredIdentityMode::Ephemeral), + 1, + ) + .expect("mint ephemeral key"); + store + .create_registration_key(CreateEnvironmentRegistrationKey { + secret_hash: ephemeral.secret.secret_hash.clone(), + record: ephemeral.record.clone(), + }) + .await + .expect("create ephemeral key"); + assert!(matches!( + store + .update_registration_key(UpdateEnvironmentRegistrationKey { + registration_key_id: ephemeral_id, + settings: RegistrationKeySettings { + display_name: "Ephemeral".to_owned(), + max_active_environments: None, + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: Some(60_000), + expires_at_ms: None, + }, + updated_at_ms: 21, + }) + .await, + Err(EnvironmentRegistryError::InvalidInput { .. }) + )); + assert_eq!( + store + .list_registration_keys() + .await + .expect("list keys") + .len(), + 2 + ); + + // Revocation is idempotent and stops rotation; resolve still answers so + // the caller can refuse with the reason. + let revoked = store + .revoke_registration_key(RevokeEnvironmentRegistrationKey { + registration_key_id: key_id.clone(), + revoked_at_ms: 30, + }) + .await + .expect("revoke"); + assert_eq!(revoked.revoked_at_ms, Some(30)); + assert_eq!( + store + .revoke_registration_key(RevokeEnvironmentRegistrationKey { + registration_key_id: key_id.clone(), + revoked_at_ms: 40, + }) + .await + .expect("revoke again") + .revoked_at_ms, + Some(30) + ); + assert!(matches!( + store + .rotate_registration_key(RotateEnvironmentRegistrationKey { + registration_key_id: key_id.clone(), + secret_hash: mint_registration_secret().secret_hash, + key_prefix: "lsrk_revoked".to_owned(), + rotated_at_ms: 50, + }) + .await, + Err(EnvironmentRegistryError::RegistrationKeyUnavailable { + reason: "revoked", + .. + }) + )); + assert!( + store + .resolve_registration_key(&rotated_secret.secret_hash) + .await + .expect("resolve revoked key") + .is_some_and(|key| key.revoked_at_ms == Some(30)) + ); +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Postgres + MinIO env"] +async fn pg_live_environment_deployment_queries_find_work_and_revoked_routes() { + use environments::{ObservedTarget, OpenEnvironment, RecordTargetObservation, TargetPhase}; + + let store = live_store("environment-deployment-queries", 1024).await; + let universe_id = store.config().universe_id; + let provider_id = bound_test_provider(&store, "deployment-queries").await; + let opened = store + .create_environment(create_environment( + &provider_id, + "deployment-opened", + "deployment-opened", + 100, + )) + .await + .expect("create opened environment"); + let closing = store + .create_environment(create_environment( + &provider_id, + "deployment-closing", + "deployment-closing", + 100, + )) + .await + .expect("create closing environment"); + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: closing.environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 100, + }) + .await + .expect("begin close"); + assert!( + store_pg::list_universes_with_environment_work(store.pool(), 10_000_000) + .await + .expect("universes with work") + .contains(&universe_id) + ); + + // Revoked routes: gone or not open rows, and rows whose binding is + // disabled. + store + .record_target_observation(RecordTargetObservation { + environment_id: opened.environment_id.clone(), + provider_target_id: Some( + ProviderTargetId::new("deployment-target").expect("target id"), + ), + observed: ObservedTarget { + phase: TargetPhase::Active, + power: None, + message: None, + observed_at_ms: 0, + }, + public_endpoint: None, + observed_at_ms: 101, + }) + .await + .expect("observe opened"); + store + .open_environment(OpenEnvironment { + environment_id: opened.environment_id.clone(), + daemon_build: test_build("1.0.0"), + at_ms: 102, + }) + .await + .expect("open opened"); + assert!( + store_pg::list_universes_with_open_provisioned_environments(store.pool()) + .await + .expect("universes with open provisioned environments") + .contains(&universe_id) + ); + let routes = vec![ + (universe_id, opened.environment_id.as_str().to_owned()), + (universe_id, closing.environment_id.as_str().to_owned()), + (universe_id, "deployment-missing".to_owned()), + ]; + assert_eq!( + store_pg::list_revoked_environment_routes(store.pool(), &routes) + .await + .expect("revoked routes"), + vec![ + (universe_id, closing.environment_id.as_str().to_owned()), + (universe_id, "deployment-missing".to_owned()), + ] + ); + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id, + provider_id: provider_id.clone(), + enabled: false, + updated_at_ms: 103, + }) + .await + .expect("disable binding"); + assert!( + store_pg::list_revoked_environment_routes(store.pool(), &routes) + .await + .expect("revoked routes after disable") + .contains(&(universe_id, opened.environment_id.as_str().to_owned())) + ); + + store_pg::delete_universe(store.pool(), universe_id) + .await + .expect("delete test universe"); + store + .delete_provider(&provider_id) + .await + .expect("delete test provider"); +} diff --git a/crates/temporal-runtime/Cargo.toml b/crates/temporal-runtime/Cargo.toml index 32c74dd9c..c6ffa87fc 100644 --- a/crates/temporal-runtime/Cargo.toml +++ b/crates/temporal-runtime/Cargo.toml @@ -63,6 +63,7 @@ url = "2" [dev-dependencies] test-support = { path = "../test-support" } +environment-client = { path = "../environment-client", features = ["conformance"] } dotenvy = "0.15" environment-daemon = { path = "../environment-daemon" } futures-util = "0.3" diff --git a/crates/temporal-runtime/src/bots/fires.rs b/crates/temporal-runtime/src/bots/fires.rs index 48d5aa120..e34cd051b 100644 --- a/crates/temporal-runtime/src/bots/fires.rs +++ b/crates/temporal-runtime/src/bots/fires.rs @@ -363,10 +363,7 @@ pub(crate) fn exec_request_id(trigger_id: &BotTriggerId, scheduled_at_ms: i64) - pub(crate) fn is_terminal_job_status(status: SessionJobStatusView) -> bool { !matches!( status, - SessionJobStatusView::Accepted - | SessionJobStatusView::Queued - | SessionJobStatusView::Running - | SessionJobStatusView::CancelRequested + SessionJobStatusView::Queued | SessionJobStatusView::Running ) } @@ -808,16 +805,16 @@ async fn fetch_exec_payload( }; let deadline = Instant::now() + Duration::from_millis(budget_ms.saturating_add(EXEC_DEADLINE_SLACK_MS)); - let mut after_seq = None; + let mut after = None; let mut stdout: Vec = Vec::new(); let mut stderr_tail: Vec = Vec::new(); loop { let read = api .read_environment_jobs(EnvironmentJobReadParams { jobs: vec![handle.clone()], - output_bytes: None, - after_seq, - include_artifacts: false, + max_bytes: None, + after, + tail_bytes: None, }) .await .map_err(|error| map_job_api_error("read poll command", error))?; @@ -842,11 +839,13 @@ async fn fetch_exec_payload( } } } - if stdout.len() > MAX_POLL_PAYLOAD_BYTES { + // A gap means the environment dropped output it could not retain, + // so stdout can no longer be a complete payload. + if stdout.len() > MAX_POLL_PAYLOAD_BYTES || !entry.output_gaps.is_empty() { cancel_job(api, &handle).await; return Err(payload_too_large("stdout")); } - after_seq = Some(entry.output_next_seq); + after = Some(entry.output_next); if let Some(summary) = entry.summary && is_terminal_job_status(summary.status) { @@ -1172,12 +1171,7 @@ mod tests { #[test] fn job_statuses_split_into_running_and_terminal() { - for status in [ - SessionJobStatusView::Accepted, - SessionJobStatusView::Queued, - SessionJobStatusView::Running, - SessionJobStatusView::CancelRequested, - ] { + for status in [SessionJobStatusView::Queued, SessionJobStatusView::Running] { assert!(!is_terminal_job_status(status), "{status:?}"); } for status in [ @@ -1187,7 +1181,6 @@ mod tests { SessionJobStatusView::TimedOut, SessionJobStatusView::DependencyFailed, SessionJobStatusView::Interrupted, - SessionJobStatusView::Lost, ] { assert!(is_terminal_job_status(status), "{status:?}"); } diff --git a/crates/temporal-runtime/src/config.rs b/crates/temporal-runtime/src/config.rs index 930632c73..9ee9f059f 100644 --- a/crates/temporal-runtime/src/config.rs +++ b/crates/temporal-runtime/src/config.rs @@ -41,24 +41,6 @@ pub enum GatewayAuthMode { Authenticated, } -/// Optional base URL outbound daemons are told to dial for data connections -/// when it differs from the general public base URL (for example a dedicated -/// hostname in front of the environment gateway). -pub fn environment_public_url_from_env() -> anyhow::Result> { - let Some(value) = optional_env("LIGHTSPEED_ENVIRONMENT_PUBLIC_URL") else { - return Ok(None); - }; - if !["https://", "http://", "wss://", "ws://"] - .iter() - .any(|scheme| value.starts_with(scheme)) - { - anyhow::bail!( - "LIGHTSPEED_ENVIRONMENT_PUBLIC_URL must be an http(s) or ws(s) URL, got {value:?}" - ); - } - Ok(Some(value.trim_end_matches('/').to_owned())) -} - pub fn gateway_auth_mode_from_env() -> anyhow::Result { if optional_env("LIGHTSPEED_UNIVERSE_AUTO_CREATE").is_some() { anyhow::bail!( diff --git a/crates/temporal-runtime/src/credential_injection.rs b/crates/temporal-runtime/src/credential_injection.rs index a12c20ad4..593b11513 100644 --- a/crates/temporal-runtime/src/credential_injection.rs +++ b/crates/temporal-runtime/src/credential_injection.rs @@ -8,8 +8,14 @@ use auth::{ RegistryTokenBroker, SecretStore, TokenAudience, }; use environment_protocol::{ - data::jobs::{StartJobsParams, StartJobsResponse}, - shared::SecretString, + bytes::SecretString, + data::{ + handshake::JobLimits, + job::{ + CancelJobsParams, CancelJobsResponse, ReadJobsParams, ReadJobsResponse, + StartJobsParams, StartJobsResponse, + }, + }, }; use environments::{ EnvironmentCredentialSource, EnvironmentCredentialStore, EnvironmentId, @@ -245,33 +251,29 @@ struct CredentialInjectingJobExecutor { #[async_trait] impl JobExecutor for CredentialInjectingJobExecutor { + fn limits(&self) -> Option { + self.inner.limits() + } + async fn start_jobs(&self, mut request: StartJobsParams) -> JobExecResult { for job in &mut request.jobs { let secret_env = self .resolver - .resolve_secret_env(&self.environment_id, &job.env) + .resolve_secret_env(&self.environment_id, &job.command.env) .await .map_err(|error| JobError::InvalidRequest { message: error.to_string(), })?; - for (name, value) in secret_env { - job.secret_env.insert(name, value); - } + job.command.secret_env.extend(secret_env); } self.inner.start_jobs(request).await } - async fn read_jobs( - &self, - request: environment_protocol::data::jobs::ReadJobsParams, - ) -> JobExecResult { + async fn read_jobs(&self, request: ReadJobsParams) -> JobExecResult { self.inner.read_jobs(request).await } - async fn cancel_jobs( - &self, - request: environment_protocol::data::jobs::CancelJobsParams, - ) -> JobExecResult { + async fn cancel_jobs(&self, request: CancelJobsParams) -> JobExecResult { self.inner.cancel_jobs(request).await } } @@ -411,6 +413,114 @@ mod tests { assert_eq!(resolved["GH_TOKEN"].expose(), "environment-token"); } + #[derive(Default)] + struct RecordingJobExecutor { + started: std::sync::Mutex>, + } + + #[async_trait] + impl JobExecutor for RecordingJobExecutor { + fn limits(&self) -> Option { + Some(JobLimits { + max_read_bytes: 1024, + max_request_jobs: 4, + max_retained_output_bytes: 2048, + record_retention_ms: 1, + cancel_grace_ms: 1, + }) + } + + async fn start_jobs(&self, request: StartJobsParams) -> JobExecResult { + self.started.lock().unwrap().push(request); + Err(JobError::Failed { + message: "recorded".to_owned(), + }) + } + + async fn read_jobs(&self, _request: ReadJobsParams) -> JobExecResult { + unreachable!("not read") + } + + async fn cancel_jobs( + &self, + _request: CancelJobsParams, + ) -> JobExecResult { + unreachable!("not cancelled") + } + } + + #[tokio::test(flavor = "current_thread")] + async fn job_wrapper_injects_credentials_into_each_command_and_keeps_limits() { + let environment_id = EnvironmentId::new("environment_1"); + let secret_id = SecretId::new("secret_1"); + let secrets = Arc::new(InMemorySecretStore::new()); + secrets + .put_secret(PutSecretRecord { + secret_id: secret_id.clone(), + secret_kind: SECRET_KIND_STATIC_BEARER.to_owned(), + value: SecretValue::new("environment-token"), + created_at_ms: 1, + }) + .await + .expect("store secret"); + let resolver = EnvironmentCredentialResolver::new( + Arc::new(FixedCredentialStore { + credentials: vec![EnvironmentCredentialRecord { + environment_id: environment_id.clone(), + env_name: "GH_TOKEN".to_owned(), + source: EnvironmentCredentialSource::DirectSecret { secret_id }, + created_at_ms: 1, + updated_at_ms: 1, + }], + }), + Arc::new(InMemoryAuthGrantStore::new()), + Arc::new(InMemoryAuthProviderStore::new()), + secrets, + None, + ); + let inner = Arc::new(RecordingJobExecutor::default()); + let wrapper = CredentialInjectingJobExecutor { + inner: inner.clone(), + resolver, + environment_id, + }; + assert_eq!( + wrapper.limits().map(|limits| limits.max_request_jobs), + Some(4) + ); + + let job = |job_id: &str| environment_protocol::data::job::JobSpec { + job_id: environment_protocol::ids::JobId::new(job_id).unwrap(), + name: None, + command: environment_protocol::data::exec::CommandSpec { + argv: vec!["true".to_owned()], + cwd: None, + env: BTreeMap::from([("CI".to_owned(), "1".to_owned())]), + secret_env: BTreeMap::new(), + input: Default::default(), + timeout_ms: None, + }, + depends_on: Vec::new(), + dependency_policy: Default::default(), + queue_key: None, + }; + let _ = wrapper + .start_jobs(StartJobsParams { + request_id: environment_protocol::ids::JobRequestId::new("req-1").unwrap(), + jobs: vec![job("a"), job("b")], + }) + .await; + + let started = inner.started.lock().unwrap(); + for job in &started[0].jobs { + assert_eq!( + job.command.secret_env["GH_TOKEN"].expose(), + "environment-token" + ); + assert_eq!(job.command.env["CI"], "1"); + } + } + #[test] fn default_grant_audience_is_environment_scoped() { let grant = AuthGrantRecord { diff --git a/crates/temporal-runtime/src/environments/fake_controller.rs b/crates/temporal-runtime/src/environments/fake_controller.rs new file mode 100644 index 000000000..6dbe4b763 --- /dev/null +++ b/crates/temporal-runtime/src/environments/fake_controller.rs @@ -0,0 +1,932 @@ +//! An in-memory provider controller for tests. +//! +//! It serves the controller plane on `/control` and a minimal data plane on +//! `/routes/{universeId}/{environmentId}/{targetId}` over a loopback +//! WebSocket listener, so Lightspeed's derived data route base +//! (`{controller origin}/routes`) reaches it. When started with a +//! credential it checks it as a bearer token on both, exactly as a real +//! provider must; without one it checks nothing. Nothing registers it: a +//! test starts one and registers its controller URL (and credential, if +//! any) like any provider's. +//! +//! Targets come up at once: `target/create` answers an `active`, `running` +//! target, and power changes and closes complete before they answer. A data +//! route dial wakes a powered-down target and refuses `waking` once, and a +//! new target may refuse `booting` for a configured number of dials; after +//! that the route answers `initialize` only, as an envd with no capability +//! families, which is enough for the runtime's open handshake. Tests change +//! targets out of band the way a provider's own idle policy or TTL would. + +use std::{ + collections::BTreeMap, + hash::{DefaultHasher, Hash, Hasher}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use axum::{ + Router, + extract::{ + Path, State, + ws::{Message, WebSocket, WebSocketUpgrade}, + }, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, + routing::get, +}; +use environment_protocol::{ + CONTROLLER_PROTOCOL_VERSION, ENVD_PROTOCOL_VERSION, + control::{ + handshake::{ + ControllerCapabilities, ControllerInitializeParams, ControllerInitializeResponse, + }, + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + CloseTargetParams, CreateTargetParams, GetTargetParams, IngressObservation, PowerState, + SetTargetPowerParams, TargetObservation, TargetOwner, TargetPhase, TargetRef, + TargetResponse, + }, + template::{EnvironmentTemplate, ListTemplatesResponse, TemplateResources}, + }, + data::handshake::{ + DataCapabilities, DataLimits, FilesystemLimits, GlobCeilings, InitializeParams, + InitializeResponse, JobLimits, ProcessLimits, ScanCeilings, SearchCeilings, + TransferCeilings, + }, + error::{Capability, PreconditionReason, ProtocolError, ProtocolErrorKind}, + handshake::{ImplementationInfo, Plane}, + ids::{DaemonInstanceId, EnvironmentId, ProviderTargetId, TemplateId, UniverseId}, + jsonrpc::{self, Request, Response as RpcResponse}, + limits::{MAX_MESSAGE_BYTES, MIN_CONCURRENT_REQUESTS}, + manifest::{ + ControllerInitialize, IngressEnsure, IngressRemove, RequestMethod, TargetClose, + TargetCreate, TargetGet, TargetSetPower, TemplateList, + }, + metadata::{DescriptiveText, DisplayName, Metadata}, + paths::EnvironmentPath, + routes::{DataRouteRefusal, PublicUrl}, +}; +use futures_util::{SinkExt, StreamExt}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; +use tokio::{net::TcpListener, sync::Mutex, task::JoinHandle}; + +/// The one template the fake controller offers. +pub const FAKE_TEMPLATE_ID: &str = "rust-v1"; + +/// Public endpoint the fake controller reports for enabled ingress. +pub const FAKE_INGRESS_ENDPOINT: &str = "https://fake.env.test"; + +/// Power states the fake controller supports when it offers power control. +/// Suspended is included so the suspend path has coverage. +pub const FAKE_POWER_STATES: [PowerState; 4] = [ + PowerState::Running, + PowerState::Paused, + PowerState::Suspended, + PowerState::Stopped, +]; + +/// `Retry-After` of the fake data route's 503 refusals, in seconds. +pub const FAKE_RETRY_AFTER_SECS: u64 = 1; + +/// What the fake controller advertises and how its data routes behave. +#[derive(Clone, Debug)] +pub struct FakeControllerOptions { + /// `capabilities.power`. + pub power: bool, + /// `capabilities.ingress`. + pub ingress: bool, + /// Require a bearer credential on the controller and its data routes. + pub credential: bool, + /// Data route dials of a new target that are refused `booting` before + /// its envd answers. + pub booting_dials: u32, + /// The protocol version the fake envd behind every data route speaks. + pub daemon_protocol_version: u32, +} + +impl Default for FakeControllerOptions { + fn default() -> Self { + Self { + power: true, + ingress: true, + credential: true, + booting_dials: 0, + daemon_protocol_version: ENVD_PROTOCOL_VERSION, + } + } +} + +/// A running fake controller. Dropping it stops the listener. +pub struct FakeController { + controller_url: String, + credential: Option, + app: Arc, + task: JoinHandle<()>, +} + +impl FakeController { + /// Serve a fake controller with the default options on a loopback port. + pub async fn start() -> std::io::Result { + Self::start_with(FakeControllerOptions::default()).await + } + + pub async fn start_with(options: FakeControllerOptions) -> std::io::Result { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let credential = options + .credential + .then(|| format!("fake-credential-{}", uuid::Uuid::new_v4().simple())); + let initialize = ControllerInitializeResponse { + protocol_version: CONTROLLER_PROTOCOL_VERSION, + plane: Plane::Controller, + server_info: implementation("lightspeed-fake-controller"), + capabilities: ControllerCapabilities { + power: options.power, + ingress: options.ingress, + }, + }; + let app = Arc::new(App { + credential: credential.clone(), + options, + initialize, + reachable: AtomicBool::new(true), + targets: Mutex::new(Targets::default()), + }); + let router = Router::new() + .route("/control", get(control_upgrade)) + .route( + "/routes/{universe}/{environment}/{target}", + get(data_upgrade), + ) + .with_state(app.clone()); + let task = tokio::spawn(async move { + let _ = axum::serve(listener, router).await; + }); + Ok(Self { + controller_url: format!("ws://{address}/control"), + credential, + app, + task, + }) + } + + /// The controller URL to register the provider with. + pub fn controller_url(&self) -> &str { + &self.controller_url + } + + /// The provider credential the controller and its data routes accept; + /// `None` when they check none. + pub fn credential(&self) -> Option<&str> { + self.credential.as_deref() + } + + /// While `false`, the controller and every data route answer 503 + /// without upgrading, as a provider that is down would. + pub fn set_reachable(&self, reachable: bool) { + self.app.reachable.store(reachable, Ordering::SeqCst); + } + + /// Move the environment's target to `power` out of band, as the + /// provider's own idle policy would. False when it holds no target. + pub async fn set_power_out_of_band(&self, environment_id: &str, power: PowerState) -> bool { + let mut targets = self.app.targets.lock().await; + match targets.find_mut(environment_id) { + Some(target) => { + target.power = power; + true + } + None => false, + } + } + + /// Remove the environment's target out of band, as a provider TTL + /// would. False when it holds no target. + pub async fn end_target(&self, environment_id: &str) -> bool { + let mut targets = self.app.targets.lock().await; + let Some(target_id) = targets + .find_mut(environment_id) + .map(|target| target.target_id.clone()) + else { + return false; + }; + targets.remove(&target_id); + true + } + + /// How many targets the controller holds. + pub async fn target_count(&self) -> usize { + self.app.targets.lock().await.by_id.len() + } +} + +impl Drop for FakeController { + fn drop(&mut self) { + self.task.abort(); + } +} + +struct App { + credential: Option, + options: FakeControllerOptions, + initialize: ControllerInitializeResponse, + reachable: AtomicBool, + targets: Mutex, +} + +#[derive(Default)] +struct Targets { + by_id: BTreeMap, + by_owner: BTreeMap, +} + +#[derive(Clone)] +struct FakeTarget { + target_id: ProviderTargetId, + owner: TargetOwner, + power: PowerState, + ingress: bool, + booting_dials: u32, +} + +impl FakeTarget { + fn observation(&self) -> TargetObservation { + TargetObservation { + target_id: Some(self.target_id.clone()), + owner: self.owner.clone(), + phase: TargetPhase::Active, + power: Some(self.power), + message: None, + ingress: self.ingress.then(|| IngressObservation { + endpoint: PublicUrl::new(FAKE_INGRESS_ENDPOINT).expect("valid fake endpoint"), + }), + } + } +} + +fn implementation(name: &str) -> ImplementationInfo { + ImplementationInfo::new(name) + .and_then(|info| info.with_version(env!("CARGO_PKG_VERSION"))) + .expect("fake implementation info is within bounds") +} + +fn authorized(app: &App, headers: &HeaderMap) -> bool { + let Some(credential) = &app.credential else { + return true; + }; + headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .is_some_and(|token| constant_time_eq(token.as_bytes(), credential.as_bytes())) +} + +fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + left.len() == right.len() + && left + .iter() + .zip(right) + .fold(0_u8, |difference, (left, right)| { + difference | (left ^ right) + }) + == 0 +} + +/// The answer before any upgrade: 503 while unreachable, 401 without the +/// credential. +fn gate(app: &App, headers: &HeaderMap) -> Option { + if !app.reachable.load(Ordering::SeqCst) { + return Some(StatusCode::SERVICE_UNAVAILABLE.into_response()); + } + if !authorized(app, headers) { + return Some( + ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, "Bearer")], + ) + .into_response(), + ); + } + None +} + +/// A typed data route refusal: its status, a JSON body and, for a 503, +/// `Retry-After`. +fn refuse(refusal: DataRouteRefusal) -> Response { + let status = StatusCode::from_u16(refusal.status()).expect("refusal statuses are valid"); + let body = serde_json::to_string(&refusal).expect("refusals encode"); + let mut response = ( + status, + [(header::CONTENT_TYPE, "application/json".to_owned())], + body, + ) + .into_response(); + if refusal.is_retryable() { + response.headers_mut().insert( + header::RETRY_AFTER, + FAKE_RETRY_AFTER_SECS + .to_string() + .parse() + .expect("valid header"), + ); + } + response +} + +fn with_limits(upgrade: WebSocketUpgrade) -> WebSocketUpgrade { + upgrade + .max_message_size(MAX_MESSAGE_BYTES as usize) + .max_frame_size(MAX_MESSAGE_BYTES as usize) +} + +async fn control_upgrade( + State(app): State>, + headers: HeaderMap, + upgrade: WebSocketUpgrade, +) -> Response { + if let Some(refused) = gate(&app, &headers) { + return refused; + } + with_limits(upgrade) + .on_upgrade(move |socket| serve(socket, Plane::Controller, app)) + .into_response() +} + +async fn data_upgrade( + State(app): State>, + Path((universe, environment, target_id)): Path<(String, String, String)>, + headers: HeaderMap, + upgrade: WebSocketUpgrade, +) -> Response { + if let Some(refused) = gate(&app, &headers) { + return refused; + } + let (Ok(universe_id), Ok(environment_id), Ok(target_id)) = ( + UniverseId::new(universe), + EnvironmentId::new(environment), + ProviderTargetId::new(target_id), + ) else { + return refuse(DataRouteRefusal::Gone {}); + }; + let owner = TargetOwner { + universe_id, + environment_id, + }; + { + let mut targets = app.targets.lock().await; + let Some(target) = targets.by_id.get_mut(&target_id) else { + return refuse(DataRouteRefusal::Gone {}); + }; + if target.owner != owner { + return refuse(DataRouteRefusal::Gone {}); + } + // The dial is the wake signal. + if target.power != PowerState::Running { + target.power = PowerState::Running; + return refuse(DataRouteRefusal::Waking {}); + } + if target.booting_dials > 0 { + target.booting_dials -= 1; + return refuse(DataRouteRefusal::Booting {}); + } + } + with_limits(upgrade) + .on_upgrade(move |socket| serve(socket, Plane::Data, app)) + .into_response() +} + +/// One connection: the envelope and the `initialize`-first handshake, +/// requests answered in order. +async fn serve(socket: WebSocket, plane: Plane, app: Arc) { + let (mut sink, mut stream) = socket.split(); + let mut initialized = false; + while let Some(Ok(frame)) = stream.next().await { + let reply = match frame { + Message::Text(text) => match jsonrpc::parse(text.as_str()) { + Err(rejection) => rejection.into_response(), + Ok(jsonrpc::Message::Request(request)) => { + Some(if request.method == ControllerInitialize::NAME { + initialize(&app, plane, &mut initialized, &request) + } else if !initialized { + RpcResponse::failure(Some(request.id), ProtocolError::not_initialized()) + } else { + let result = match plane { + Plane::Controller => controller_call(&app, &request).await, + _ => Err(ProtocolError::method_not_found(&request.method)), + }; + RpcResponse { + id: Some(request.id), + result, + } + }) + } + Ok(jsonrpc::Message::Notification(_) | jsonrpc::Message::Response(_)) => None, + }, + Message::Binary(_) => Some(RpcResponse::failure( + None, + ProtocolError::invalid_request("binary frames are not part of any plane"), + )), + Message::Ping(payload) => { + if sink.send(Message::Pong(payload)).await.is_err() { + break; + } + None + } + Message::Pong(_) => None, + Message::Close(_) => break, + }; + if let Some(reply) = reply + && sink + .send(Message::Text(reply.to_text().into())) + .await + .is_err() + { + break; + } + } + let _ = sink.close().await; +} + +fn initialize(app: &App, plane: Plane, initialized: &mut bool, request: &Request) -> RpcResponse { + let id = Some(request.id.clone()); + if *initialized { + return RpcResponse::failure( + id, + ProtocolError::invalid_request("initialize was already called on this connection"), + ); + } + let expected = match plane { + Plane::Controller => CONTROLLER_PROTOCOL_VERSION, + _ => app.options.daemon_protocol_version, + }; + if let Some(version) = request + .params + .get("protocolVersion") + .and_then(Value::as_u64) + && version != u64::from(expected) + { + return RpcResponse::failure( + id, + ProtocolError::version_mismatch(expected, u32::try_from(version).unwrap_or(u32::MAX)), + ); + } + let result = match plane { + Plane::Controller => request + .params::() + .and_then(|_| encode(&app.initialize)), + _ => request + .params::() + .and_then(|_| encode(&data_initialize(expected))), + }; + if result.is_ok() { + *initialized = true; + } + RpcResponse { id, result } +} + +/// The fake envd's handshake: no capability families, so a client that +/// needs one is refused before calling anything. +fn data_initialize(protocol_version: u32) -> InitializeResponse { + InitializeResponse { + protocol_version, + plane: Plane::Data, + daemon_instance_id: DaemonInstanceId::new(format!( + "envd_{}", + uuid::Uuid::new_v4().simple() + )) + .expect("valid daemon instance id"), + server_info: implementation("lightspeed-fake-envd"), + capabilities: DataCapabilities::default(), + limits: DataLimits { + max_concurrent_requests: MIN_CONCURRENT_REQUESTS, + max_wait_ms: 60_000, + process: ProcessLimits { + max_read_bytes: 1 << 20, + max_retained_output_bytes: 1 << 20, + retention_after_exit_ms: 60_000, + unread_retention_ms: 60_000, + max_exited_retained_bytes: 1 << 20, + terminate_grace_ms: 1_000, + max_input_queue_bytes: 1 << 20, + }, + job: JobLimits { + max_read_bytes: 1 << 20, + max_request_jobs: 1, + max_retained_output_bytes: 1 << 20, + record_retention_ms: 60_000, + cancel_grace_ms: 1_000, + }, + filesystem: FilesystemLimits { + max_read_file_bytes: 1 << 20, + max_write_file_bytes: 1 << 20, + max_directory_entries: 1, + search: SearchCeilings { + max_matches: 1, + max_files: 1, + max_bytes: 1, + max_duration_ms: 1_000, + max_line_bytes: 1, + }, + glob: GlobCeilings { + max_matches: 1, + max_files: 1, + max_duration_ms: 1_000, + }, + scan: ScanCeilings { + max_roots: 1, + max_patterns: 1, + max_entries: 1, + max_file_bytes: 1, + max_total_bytes: 1, + max_duration_ms: 1_000, + }, + transfer: TransferCeilings { + max_active_operations: 1, + max_content_chunk_bytes: 1, + max_inventory_page_entries: 1, + receipt_retention_ms: 60_000, + max_entries: 1, + max_depth: 1, + max_file_bytes: 1, + max_total_bytes: 1, + max_manifest_bytes: 1, + max_duration_ms: 1_000, + }, + }, + }, + default_cwd: EnvironmentPath::new("/").expect("root path"), + home_directory: None, + } +} + +fn encode(value: &T) -> Result { + serde_json::to_value(value).map_err(|error| ProtocolError::internal(error.to_string())) +} + +fn params(request: &Request) -> Result +where + M::Params: DeserializeOwned, +{ + request.params::() +} + +async fn controller_call(app: &App, request: &Request) -> Result { + let method = request.method.as_str(); + if method == TemplateList::NAME { + params::(request)?; + return encode(&list_templates(&app.options)); + } + if (method == IngressEnsure::NAME || method == IngressRemove::NAME) && !app.options.ingress { + return Err(ProtocolError::capability_unavailable( + Capability::Ingress, + "the fake controller was started without ingress", + )); + } + if method == TargetSetPower::NAME && !app.options.power { + return Err(ProtocolError::capability_unavailable( + Capability::Power, + "the fake controller was started without power control", + )); + } + let mut targets = app.targets.lock().await; + let observation = match method { + TargetCreate::NAME => { + targets.create(params::(request)?, app.options.booting_dials)? + } + TargetGet::NAME => { + let GetTargetParams { target } = params::(request)?; + targets.resolve(&target)?.observation() + } + TargetSetPower::NAME => targets.set_power(params::(request)?)?, + TargetClose::NAME => targets.close(params::(request)?)?, + IngressEnsure::NAME => { + let EnsureIngressParams { target } = params::(request)?; + targets.set_ingress(&target, true)? + } + IngressRemove::NAME => { + let RemoveIngressParams { target } = params::(request)?; + targets.set_ingress(&target, false)? + } + other => return Err(ProtocolError::method_not_found(other)), + }; + encode(&TargetResponse { + target: observation, + }) +} + +fn list_templates(options: &FakeControllerOptions) -> ListTemplatesResponse { + ListTemplatesResponse { + templates: vec![EnvironmentTemplate { + template_id: TemplateId::new(FAKE_TEMPLATE_ID).expect("valid template id"), + display_name: DisplayName::new("Rust development").expect("valid display name"), + description: DescriptiveText::new("Deterministic fake provider template").ok(), + power_states: if options.power { + FAKE_POWER_STATES.to_vec() + } else { + Vec::new() + }, + public_ingress: options.ingress, + idle: None, + deprecated: false, + resources: Some(TemplateResources { + vcpus: Some(2), + memory_bytes: Some(4 << 30), + disk_bytes: Some(20 << 30), + }), + metadata: Metadata::default(), + }], + } +} + +fn owner_mismatch() -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::OwnerMismatch, + "the target belongs to another owner", + ) +} + +impl Targets { + /// The owner's target, or the named one after the ownership fence. + fn resolve(&self, target: &TargetRef) -> Result<&FakeTarget, ProtocolError> { + let target_id = match &target.target_id { + Some(target_id) => target_id, + None => self + .by_owner + .get(&target.owner) + .ok_or_else(|| ProtocolError::not_found("the owner holds no target"))?, + }; + let found = self + .by_id + .get(target_id) + .ok_or_else(|| ProtocolError::not_found(format!("unknown target: {target_id}")))?; + if found.owner != target.owner { + return Err(owner_mismatch()); + } + Ok(found) + } + + fn find_mut(&mut self, environment_id: &str) -> Option<&mut FakeTarget> { + self.by_id + .values_mut() + .find(|target| target.owner.environment_id.as_str() == environment_id) + } + + fn remove(&mut self, target_id: &ProviderTargetId) { + if let Some(target) = self.by_id.remove(target_id) { + self.by_owner.remove(&target.owner); + } + } + + fn create( + &mut self, + params: CreateTargetParams, + booting_dials: u32, + ) -> Result { + if let Some(idle) = ¶ms.idle { + idle.validate() + .map_err(|error| ProtocolError::invalid_field("idle", error.to_string()))?; + } + if let Some(target_id) = self.by_owner.get(¶ms.owner) { + return Ok(self.by_id[target_id].observation()); + } + if params.template_id.as_str() != FAKE_TEMPLATE_ID { + return Err(ProtocolError::not_found(format!( + "unknown template: {}", + params.template_id + ))); + } + let mut identity = DefaultHasher::new(); + params.owner.hash(&mut identity); + let target_id = ProviderTargetId::new(format!("fake-{:016x}", identity.finish())) + .expect("valid fake target id"); + let target = FakeTarget { + target_id: target_id.clone(), + owner: params.owner.clone(), + power: PowerState::Running, + ingress: false, + booting_dials, + }; + let observation = target.observation(); + self.by_owner.insert(params.owner, target_id.clone()); + self.by_id.insert(target_id, target); + Ok(observation) + } + + fn set_power( + &mut self, + params: SetTargetPowerParams, + ) -> Result { + if !FAKE_POWER_STATES.contains(¶ms.power) { + return Err(ProtocolError::invalid_field( + "power", + format!("unsupported power state {}", params.power), + )); + } + let target_id = self.resolve(¶ms.target)?.target_id.clone(); + let target = self.by_id.get_mut(&target_id).expect("resolved target"); + target.power = params.power; + Ok(target.observation()) + } + + fn close(&mut self, params: CloseTargetParams) -> Result { + let closed = |target_id| TargetObservation { + target_id, + owner: params.target.owner.clone(), + phase: TargetPhase::Closed, + power: None, + message: None, + ingress: None, + }; + let target_id = match self.resolve(¶ms.target) { + Ok(target) => target.target_id.clone(), + Err(error) if error.kind() == ProtocolErrorKind::NotFound => { + return Ok(closed(params.target.target_id.clone())); + } + Err(error) => return Err(error), + }; + self.remove(&target_id); + Ok(closed(Some(target_id))) + } + + fn set_ingress( + &mut self, + target: &TargetRef, + enabled: bool, + ) -> Result { + let target_id = self.resolve(target)?.target_id.clone(); + let target = self.by_id.get_mut(&target_id).expect("resolved target"); + target.ingress = enabled; + Ok(target.observation()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::environments::gateway::runtime_client_info; + use environment_client::{ + ConnectOptions, ControllerClient, DataClient, + conformance::{ + controller::{ControllerConformance, run}, + data_route::expect_refusal, + }, + }; + use environment_protocol::routes::{DataRouteBase, provider_data_route}; + use std::{ + sync::atomic::{AtomicU64, Ordering}, + time::Duration, + }; + + fn owner_factory() -> Arc TargetOwner + Send + Sync> { + let next = Arc::new(AtomicU64::new(0)); + let universe = uuid::Uuid::new_v4().to_string(); + Arc::new(move || { + let n = next.fetch_add(1, Ordering::SeqCst); + TargetOwner { + universe_id: UniverseId::new(universe.clone()).unwrap(), + environment_id: EnvironmentId::new(format!("environment_{n}")).unwrap(), + } + }) + } + + fn options(controller: &FakeController) -> ConnectOptions { + let options = ConnectOptions::new(runtime_client_info()); + match controller.credential() { + Some(credential) => options.with_bearer_token(credential), + None => options, + } + } + + async fn conformance(options_for: FakeControllerOptions) { + let controller = FakeController::start_with(options_for).await.unwrap(); + let mut suite = ControllerConformance::new( + controller.controller_url(), + options(&controller), + TemplateId::new(FAKE_TEMPLATE_ID).unwrap(), + owner_factory(), + ); + suite.prompt_bound = Duration::from_secs(5); + suite.settle_timeout = Duration::from_secs(10); + suite.poll_interval = Duration::from_millis(50); + suite.data_route_base = + Some(DataRouteBase::derived_from_controller(controller.controller_url()).unwrap()); + run(&suite) + .await + .into_result() + .expect("the fake controller conforms"); + } + + #[tokio::test(flavor = "current_thread")] + async fn the_fake_controller_conforms() { + conformance(FakeControllerOptions::default()).await; + } + + #[tokio::test(flavor = "current_thread")] + async fn the_fake_controller_without_optional_capabilities_conforms() { + conformance(FakeControllerOptions { + power: false, + ingress: false, + ..FakeControllerOptions::default() + }) + .await; + } + + #[tokio::test(flavor = "current_thread")] + async fn the_fake_controller_without_a_credential_conforms() { + conformance(FakeControllerOptions { + credential: false, + ..FakeControllerOptions::default() + }) + .await; + } + + /// A target created through the controller, and its data route under + /// the base Lightspeed derives from the controller URL. + async fn created(controller: &FakeController) -> (TargetOwner, String) { + let (client, _) = + ControllerClient::connect_initialized(controller.controller_url(), options(controller)) + .await + .expect("controller handshake"); + let owner = owner_factory()(); + let target_id = client + .create_target(&CreateTargetParams { + owner: owner.clone(), + template_id: TemplateId::new(FAKE_TEMPLATE_ID).unwrap(), + display_name: None, + idle: None, + }) + .await + .expect("create") + .target + .target_id + .expect("target id"); + let _ = client.close().await; + let base = DataRouteBase::derived_from_controller(controller.controller_url()).unwrap(); + let route = provider_data_route(&base, &owner, &target_id); + (owner, route) + } + + #[tokio::test(flavor = "current_thread")] + async fn a_dial_wakes_a_powered_down_target_and_refuses_waking_once() { + let controller = FakeController::start().await.unwrap(); + let (owner, route) = created(&controller).await; + assert!( + controller + .set_power_out_of_band(owner.environment_id.as_str(), PowerState::Paused) + .await + ); + let refusal = expect_refusal(&route, options(&controller)) + .await + .expect("a conforming refusal"); + assert_eq!(refusal, DataRouteRefusal::Waking {}); + let (client, _) = DataClient::connect_initialized(&route, options(&controller)) + .await + .expect("the woken target upgrades"); + let _ = client.close().await; + } + + #[tokio::test(flavor = "current_thread")] + async fn a_new_target_refuses_booting_for_its_configured_dials() { + let controller = FakeController::start_with(FakeControllerOptions { + booting_dials: 1, + ..FakeControllerOptions::default() + }) + .await + .unwrap(); + let (_, route) = created(&controller).await; + assert_eq!( + expect_refusal(&route, options(&controller)).await.unwrap(), + DataRouteRefusal::Booting {} + ); + let (client, _) = DataClient::connect_initialized(&route, options(&controller)) + .await + .expect("envd answers after its booting dials"); + let _ = client.close().await; + } + + #[tokio::test(flavor = "current_thread")] + async fn an_ended_target_refuses_gone_and_an_unreachable_controller_refuses_all() { + let controller = FakeController::start().await.unwrap(); + let (owner, route) = created(&controller).await; + assert!(controller.end_target(owner.environment_id.as_str()).await); + assert_eq!(controller.target_count().await, 0); + assert_eq!( + expect_refusal(&route, options(&controller)).await.unwrap(), + DataRouteRefusal::Gone {} + ); + controller.set_reachable(false); + let error = ControllerClient::connect_initialized( + controller.controller_url(), + options(&controller), + ) + .await + .expect_err("an unreachable controller refuses the upgrade"); + assert!(matches!( + error, + environment_client::ClientError::UpgradeRejected { status: 503, .. } + )); + } +} diff --git a/crates/temporal-runtime/src/environments/gateway.rs b/crates/temporal-runtime/src/environments/gateway.rs index 21980c4e2..cbb8984dc 100644 --- a/crates/temporal-runtime/src/environments/gateway.rs +++ b/crates/temporal-runtime/src/environments/gateway.rs @@ -1,16 +1,34 @@ -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; -use environment_protocol::shared::{ - EnvironmentCapabilities, EnvironmentDataConnection, EnvironmentScope, EnvironmentTransport, - ProviderTargetId, -}; +use environment_client::{ClientError, ConnectOptions, DataClient}; +use environment_protocol::{data::handshake::InitializeResponse, handshake::ImplementationInfo}; use uuid::Uuid; +/// Path prefix of the worker routes the environment gateway serves: +/// `{prefix}/{universe}/{environment}`. Runtime-internal: only Lightspeed's +/// own workers dial it, with the deployment token. +pub const ROUTE_PATH_PREFIX: &str = "/environment-gateway/routes"; + +/// Connect bound (TCP, TLS and upgrade) of a worker route. The gateway +/// answers a registered environment's upgrade only once the reverse +/// stream is paired, which takes up to its stream-open bound; the worker +/// waits a little longer so that the gateway's typed refusal, not the +/// client's own timeout, reaches the caller. +pub const WORKER_ROUTE_CONNECT_TIMEOUT: Duration = + Duration::from_secs(crate::gateway::registration::STREAM_OPEN_TIMEOUT.as_secs() + 5); + +/// Runtime-internal header of a worker route dial that must not wake a +/// powered-down target: environment source discovery. The gateway asks the +/// provider for a provisioned target's current power first and refuses the +/// dial (bodiless 412) unless the target is active and running; the +/// recorded observation cannot decide, since providers power targets down +/// on their own clock. Other sources have no power to keep. +pub const NO_WAKE_HEADER: &str = "lightspeed-no-wake"; + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct RouteKey { pub universe_id: Uuid, pub environment_id: String, - pub incarnation_id: String, } #[derive(Clone)] @@ -57,45 +75,69 @@ impl EnvironmentGatewayClientConfig { RouteKey { universe_id, environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), } } - pub fn connection(&self, key: &RouteKey) -> EnvironmentDataConnection { - let base = websocket_base(&self.base_url); - EnvironmentDataConnection { - target_id: ProviderTargetId::new(key.environment_id.clone()), - endpoint: format!( - "{base}/environment-gateway/routes/{}/{}/{}", - key.universe_id, key.environment_id, key.incarnation_id - ), - transport: EnvironmentTransport::WebSocket, - scope: EnvironmentScope::Default, - default_cwd: None, - capabilities: EnvironmentCapabilities::default(), - } + /// The worker route of one environment: a data-plane endpoint whatever + /// the environment's source. Instead of upgrading, the gateway may + /// answer a typed `DataRouteRefusal`. + pub fn endpoint(&self, key: &RouteKey) -> String { + format!( + "{}{ROUTE_PATH_PREFIX}/{}/{}", + websocket_base(&self.base_url), + key.universe_id, + environment_protocol::routes::encode_path_segment(&key.environment_id), + ) } - pub fn connection_for( + pub fn endpoint_for( &self, universe_id: Uuid, environment: &environments::EnvironmentRecord, - ) -> EnvironmentDataConnection { - self.connection(&self.route_key(universe_id, environment)) + ) -> String { + self.endpoint(&self.route_key(universe_id, environment)) + } + + /// Data-plane connect options for a worker route: `clientInfo` names + /// the runtime build, `user_agent` the caller, and the deployment token + /// is the bearer, and the connect bound is + /// [`WORKER_ROUTE_CONNECT_TIMEOUT`]. Callers on background paths add a + /// request timeout. + pub fn connect_options(&self, user_agent: impl Into) -> ConnectOptions { + let mut options = ConnectOptions::new(runtime_client_info()) + .with_bearer_token((*self.deployment_token).clone()) + .with_user_agent(user_agent); + options.connect_timeout = WORKER_ROUTE_CONNECT_TIMEOUT; + options + } + + /// [`connect_options`](Self::connect_options) for a dial that must not + /// wake its target ([`NO_WAKE_HEADER`]). + pub fn no_wake_connect_options(&self, user_agent: impl Into) -> ConnectOptions { + self.connect_options(user_agent) + .with_header(NO_WAKE_HEADER, "1") } - pub fn connect_options( + /// Dial the worker route of `key` and complete the data-plane + /// handshake. + pub async fn connect( &self, - user_agent: impl Into, - ) -> environment_client::WebSocketConnectOptions { - environment_client::WebSocketConnectOptions { - bearer_token: Some((*self.deployment_token).clone()), - user_agent: Some(user_agent.into()), - headers: Vec::new(), - } + key: &RouteKey, + options: ConnectOptions, + ) -> Result<(DataClient, InitializeResponse), ClientError> { + DataClient::connect_initialized(&self.endpoint(key), options).await } } +/// `clientInfo` of every data and controller connection this runtime +/// opens. +pub fn runtime_client_info() -> ImplementationInfo { + ImplementationInfo::new("lightspeed-runtime") + .and_then(|info| info.with_version(release_info::VERSION)) + .and_then(|info| info.with_git_sha(release_info::GIT_SHA)) + .expect("the runtime's build facts are valid implementation info") +} + pub fn bearer_matches(headers: &axum::http::HeaderMap, expected: &str) -> bool { headers .get(axum::http::header::AUTHORIZATION) @@ -111,7 +153,7 @@ pub fn close_message(reason: &'static str) -> axum::extract::ws::Message { })) } -pub(crate) fn websocket_base(url: &str) -> String { +fn websocket_base(url: &str) -> String { if let Some(rest) = url.strip_prefix("https://") { format!("wss://{rest}") } else if let Some(rest) = url.strip_prefix("http://") { @@ -140,18 +182,47 @@ mod tests { RouteKey { universe_id: Uuid::nil(), environment_id: "environment-a".to_owned(), - incarnation_id: "incarnation-a".to_owned(), } } #[test] - fn gateway_connection_is_stable_and_has_no_cached_capabilities() { + fn worker_routes_are_stable_websocket_urls() { + let config = EnvironmentGatewayClientConfig::new("https://gateway.test/", "secret"); + assert_eq!( + config.endpoint(&key()), + "wss://gateway.test/environment-gateway/routes/00000000-0000-0000-0000-000000000000/environment-a" + ); + let local = EnvironmentGatewayClientConfig::new("http://127.0.0.1:18080", "secret"); + assert!( + local + .endpoint(&key()) + .starts_with("ws://127.0.0.1:18080/environment-gateway/routes/") + ); + } + + #[test] + fn connect_options_carry_the_deployment_token_and_runtime_build() { let config = EnvironmentGatewayClientConfig::new("https://gateway.test", "secret"); - let connection = config.connection(&key()); + let options = config.connect_options("lightspeed-test"); assert_eq!( - connection.endpoint, - "wss://gateway.test/environment-gateway/routes/00000000-0000-0000-0000-000000000000/environment-a/incarnation-a" + options.bearer_token.as_ref().map(|token| token.expose()), + Some("secret") + ); + assert_eq!(options.user_agent.as_deref(), Some("lightspeed-test")); + assert_eq!(options.client_info.name(), "lightspeed-runtime"); + assert_eq!(options.client_info.version(), Some(release_info::VERSION)); + assert_eq!(options.connect_timeout, WORKER_ROUTE_CONNECT_TIMEOUT); + } + + #[test] + fn bearer_tokens_compare_exactly() { + let mut headers = axum::http::HeaderMap::new(); + headers.insert( + axum::http::header::AUTHORIZATION, + "Bearer secret".parse().unwrap(), ); - assert_eq!(connection.capabilities, EnvironmentCapabilities::default()); + assert!(bearer_matches(&headers, "secret")); + assert!(!bearer_matches(&headers, "secre")); + assert!(!bearer_matches(&axum::http::HeaderMap::new(), "secret")); } } diff --git a/crates/temporal-runtime/src/environments/lifecycle.rs b/crates/temporal-runtime/src/environments/lifecycle.rs index e7f7c8f54..b7a2e678a 100644 --- a/crates/temporal-runtime/src/environments/lifecycle.rs +++ b/crates/temporal-runtime/src/environments/lifecycle.rs @@ -1,163 +1,185 @@ +//! Environment lifecycle: the API handlers and the reconciler. +//! +//! Lightspeed keeps durable retries for the two things it alone decides: +//! create until accepted (and opened by one data-plane handshake), and +//! close until gone. Everything a provider does while a target runs — +//! power, idle, wake, ingress — is the provider's; the slow display sweep +//! only records what it reports, and acts only when it reports the target +//! failed or gone. Power and ingress API calls are synchronous +//! pass-throughs that store nothing but the returned observation. + +use super::provider_controllers::{ + ControllerDisposition, ControllerError, ControllerFeature, PASS_THROUGH_TIMEOUT, + ProviderController, ProviderControllerConnector, call_disposition, connect_provider_controller, + provider_connect_options, +}; use super::providers::{ - binding_context, environment_view, map_environments_error, - parse_environment_provider_binding_id, registry_idle_policy, registry_lifecycle_status, - registry_power_state, + environment_view, map_environments_error, parse_environment_provider_id, registry_idle_policy, + registry_power_state, registry_state, }; +use super::signals::EnvironmentSignals; use super::*; use ::environments::{ - BeginCloseEnvironment, CreateEnvironment, CreateExternalEnvironment, EnvironmentId, - EnvironmentIncarnationId, EnvironmentProviderBindingStore, EnvironmentProvisionRequestId, - EnvironmentRecord, EnvironmentSource, EnvironmentStatus, EnvironmentStore, - EnvironmentTemplateId, FailEnvironmentLifecycle, FinishCloseEnvironment, ListEnvironments, - ObserveProvisionedEnvironment, PowerState, SetEnvironmentIdlePolicy, SetEnvironmentPower, + BeginCloseEnvironment, CreateEnvironment, CreateExternalEnvironment, DaemonBuild, + EndProvisionedEnvironment, EnvironmentCloseReason, EnvironmentEndpoint, EnvironmentId, + EnvironmentProviderId, EnvironmentProviderStore, EnvironmentProvisionRequestId, + EnvironmentRecord, EnvironmentRegistryError, EnvironmentState, EnvironmentStore, + EnvironmentTemplateId, FailEnvironment, FinishCloseEnvironment, ListEnvironments, + ObservedTarget, OpenEnvironment, RecordTargetObservation, ScheduleEnvironmentAttempt, + UpdateEnvironment, WAIT_RETRY_AFTER_MS, }; -use environment_protocol::control::ingress::{ - EnsureIngressParams, ProviderIngressStatus, RemoveIngressParams, -}; -use environment_protocol::control::targets::{ - AdoptTargetParams, CloseTargetParams, CreateTargetParams, ProviderTargetStatus, - ProviderTargetSummary, SetTargetPowerParams, +use environment_client::{ClientError, DataClient}; +use environment_protocol::{ + control::{ + ingress::{EnsureIngressParams, RemoveIngressParams}, + target::{ + CloseTargetParams, CreateTargetParams, GetTargetParams, SetTargetPowerParams, + TargetObservation, TargetOwner, TargetPhase, TargetRef, + }, + }, + ids::{TemplateId, UniverseId}, + manifest::{IngressEnsure, IngressRemove, TargetSetPower}, + metadata::DisplayName, + routes::{DataRouteRefusal, provider_data_route}, }; +use futures_util::{StreamExt, stream}; +use std::{collections::BTreeMap, time::Duration}; -/// Map a provider target observation to the logical lifecycle status. A -/// passive provider reports Ready only after its private envd is reachable; -/// no provider presence has to register separately. -pub(crate) fn lifecycle_status_from_target(status: ProviderTargetStatus) -> EnvironmentStatus { - match status { - ProviderTargetStatus::Ready => EnvironmentStatus::Ready, - ProviderTargetStatus::Creating | ProviderTargetStatus::Starting => { - EnvironmentStatus::Booting - } - ProviderTargetStatus::Paused => EnvironmentStatus::Paused, - ProviderTargetStatus::Suspended => EnvironmentStatus::Suspended, - ProviderTargetStatus::Stopped => EnvironmentStatus::Offline, - ProviderTargetStatus::Closing => EnvironmentStatus::Closing, - ProviderTargetStatus::Closed => EnvironmentStatus::Closed, - ProviderTargetStatus::Failed => EnvironmentStatus::Failed, - ProviderTargetStatus::Unknown => EnvironmentStatus::Unknown, - } +/// How long a provisioning environment may take, from entering +/// `provisioning` to its open handshake, before it is failed. +pub(crate) const PROVISIONING_DEADLINE_MS: i64 = 30 * 60 * 1000; + +/// Re-check interval while the provider reports work in progress. +pub(crate) const POLL_MS: i64 = 2_000; + +/// First backoff after a failed attempt; doubled per attempt. +const BACKOFF_BASE_MS: i64 = 2_000; + +/// Longest backoff between attempts. +const BACKOFF_MAX_MS: i64 = 5 * 60 * 1000; + +/// Rows of one provider in flight at once, over one shared controller +/// connection. +pub(crate) const PROVIDER_CONCURRENCY: usize = 8; + +/// Rows one work pass takes from a universe. +pub(crate) const WORK_BATCH: u32 = 256; + +/// How often the reconciler wakes without a nudge. +pub(crate) const WORK_TICK: Duration = Duration::from_secs(1); + +/// How often the display sweep observes open provisioned environments. +pub(crate) const DISPLAY_SWEEP_INTERVAL: Duration = Duration::from_secs(5 * 60); + +/// The delay before attempt `attempts + 1` after `attempts` consecutive +/// failures: 2 s doubling, at most 5 min. +pub(crate) fn backoff_delay_ms(attempts: u32) -> i64 { + let shift = attempts.saturating_sub(1).min(8); + (BACKOFF_BASE_MS << shift).min(BACKOFF_MAX_MS) } -impl EnvironmentService { - pub(crate) async fn put_environment_ingress_record( - &self, - params: EnvironmentIngressPutParams, - ) -> Result { - let environment_id = parse_registry_environment_id(params.environment_id)?; - let environment = EnvironmentStore::read_environment(self.store.as_ref(), &environment_id) - .await - .map_err(map_environments_error)?; - if !matches!( - environment.status, - EnvironmentStatus::Ready - | EnvironmentStatus::Offline - | EnvironmentStatus::Paused - | EnvironmentStatus::Suspended - ) { - return Err(AgentApiError::rejected( - "public ingress requires a ready or powered-down environment", - )); - } - let EnvironmentSource::Provisioned { - provider_id, - binding_id, - } = &environment.source - else { - return Err(AgentApiError::rejected( - "provider-managed ingress is unavailable for external environments", - )); +/// True once a provisioning row has been provisioning for the deadline. +pub(crate) fn provisioning_deadline_passed(record: &EnvironmentRecord, now_ms: i64) -> bool { + record.state == EnvironmentState::Provisioning + && now_ms.saturating_sub(record.state_changed_at_ms) >= PROVISIONING_DEADLINE_MS +} + +/// The controller-plane owner of a provisioned environment. +pub(crate) fn target_owner( + universe_id: uuid::Uuid, + environment: &EnvironmentRecord, +) -> Result { + let invalid = + |error: environment_protocol::ids::IdError| EnvironmentRegistryError::InvalidInput { + message: format!("environment ids do not fit the controller protocol: {error}"), }; - let target_id = environment - .incarnation - .provider_target_id - .clone() - .ok_or_else(|| AgentApiError::rejected("environment has no provider target"))?; - let provider = self.read_environment_provider(provider_id).await?; - let binding = EnvironmentProviderBindingStore::read_provider_binding( - self.store.as_ref(), - self.store.config().universe_id, - binding_id, + Ok(TargetOwner { + universe_id: UniverseId::new(universe_id.to_string()).map_err(invalid)?, + environment_id: environment_protocol::ids::EnvironmentId::new( + environment.environment_id.as_str(), ) - .await - .map_err(map_environments_error)?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) - .await?; - let public_endpoint = async { - if params.enabled { - let response = controller - .ensure_ingress(&EnsureIngressParams { - request_id: format!("ingress-enable:{}", environment.request_id), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id, - }) - .await?; - if response.status != ProviderIngressStatus::Ready { - return Err(AgentApiError::rejected( - "provider did not realize public ingress", - )); - } - Ok(Some(response.public_endpoint.ok_or_else(|| { - AgentApiError::rejected( - "provider returned ready ingress without a public endpoint", - ) - })?)) - } else { - let response = controller - .remove_ingress(&RemoveIngressParams { - request_id: format!("ingress-disable:{}", environment.request_id), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id, - }) - .await?; - if response.status != ProviderIngressStatus::Disabled { - return Err(AgentApiError::rejected( - "provider did not remove public ingress", - )); - } - Ok(None) - } + .map_err(invalid)?, + }) +} + +/// The wire name of a target phase, for state messages. +fn phase_name(phase: TargetPhase) -> &'static str { + match phase { + TargetPhase::Provisioning => "provisioning", + TargetPhase::Active => "active", + TargetPhase::Closing => "closing", + TargetPhase::Closed => "closed", + TargetPhase::Failed => "failed", + TargetPhase::Unknown => "unknown", + } +} + +/// What a provider observation records: power only on an active target. +fn observed_target(observation: &TargetObservation, now_ms: i64) -> ObservedTarget { + ObservedTarget { + phase: observation.phase, + power: observation + .power + .filter(|_| observation.phase == TargetPhase::Active), + message: observation + .message + .as_ref() + .map(|message| message.as_str().to_owned()), + observed_at_ms: now_ms, + } +} + +fn observation_request( + environment: &EnvironmentRecord, + observation: &TargetObservation, + now_ms: i64, +) -> RecordTargetObservation { + RecordTargetObservation { + environment_id: environment.environment_id.clone(), + provider_target_id: observation.target_id.clone(), + observed: observed_target(observation, now_ms), + public_endpoint: observation + .ingress + .as_ref() + .map(|ingress| ingress.endpoint.as_str().to_owned()), + observed_at_ms: now_ms, + } +} + +/// Wall-clock milliseconds for reconciler writes. +pub(crate) fn clock_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +impl EnvironmentService { + fn reconciler(&self) -> Reconciler<'_> { + Reconciler { + universe_id: self.store.config().universe_id, + environments: self.store.as_ref(), + providers: self.store.as_ref(), + connector: self.provider_controller_connector.as_ref(), + signals: &self.signals, } - .await; - let public_endpoint = finish_provider_controller(controller, public_endpoint).await?; - let environment = EnvironmentStore::set_environment_ingress( - self.store.as_ref(), - ::environments::SetEnvironmentIngress { - environment_id, - enabled: params.enabled, - public_endpoint, - updated_at_ms: now_ms()?, - }, - ) - .await - .map_err(map_environments_error)?; - Ok(EnvironmentIngressPutResponse { - environment: environment_view(&environment), - }) } pub(crate) async fn create_external_environment_record( &self, params: EnvironmentExternalCreateParams, ) -> Result { - let request_id = - EnvironmentProvisionRequestId::try_new(params.request_id).map_err(|error| { - AgentApiError::invalid_request(format!("invalid environment request id: {error}")) - })?; - let connection = external_connection_from_api(params.connection)?; + let request_id = parse_request_id(params.request_id)?; + let endpoint = EnvironmentEndpoint::parse(¶ms.endpoint).map_err(|error| { + AgentApiError::invalid_request(format!("invalid environment endpoint: {error}")) + })?; let environment = EnvironmentStore::create_external_environment( self.store.as_ref(), CreateExternalEnvironment { request_id, environment_id: allocate_environment_id(), - incarnation_id: allocate_incarnation_id(), - connection, + endpoint, display_name: params.display_name, metadata: validated_caller_metadata(params.metadata)?, created_at_ms: now_ms()?, @@ -169,132 +191,199 @@ impl EnvironmentService { environment: environment_view(&environment), }) } + + /// `environments/create`: accept the creation and nudge the + /// reconciler; provider I/O belongs to the reconciler. pub(crate) async fn create_environment_record( &self, params: EnvironmentCreateParams, ) -> Result { - let environment = self.accept_environment_create(params).await?; - Ok(EnvironmentCreateResponse { - environment: environment_view(&environment), - }) - } - - /// Accept environment creation; provider I/O belongs to the reconciler. - pub(crate) async fn accept_environment_create( - &self, - params: EnvironmentCreateParams, - ) -> Result { - let request_id = - EnvironmentProvisionRequestId::try_new(params.request_id).map_err(|error| { - AgentApiError::invalid_request(format!("invalid environment request id: {error}")) - })?; - let binding_id = parse_environment_provider_binding_id(params.binding_id)?; + let request_id = parse_request_id(params.request_id)?; + let provider_id = parse_environment_provider_id(params.provider_id)?; let template_id = EnvironmentTemplateId::try_new(params.template_id).map_err(|error| { AgentApiError::invalid_request(format!("invalid environment template id: {error}")) })?; - let idle_policy = params.idle_policy.as_ref().map(registry_idle_policy); - if let Some(policy) = &idle_policy { - policy.validate().map_err(map_environments_error)?; - } - EnvironmentStore::create_environment( + let idle = params.idle.as_ref().map(registry_idle_policy).transpose()?; + let environment = EnvironmentStore::create_environment( self.store.as_ref(), CreateEnvironment { request_id, environment_id: allocate_environment_id(), - incarnation_id: allocate_incarnation_id(), - binding_id, + provider_id, template_id, + idle, display_name: params.display_name, metadata: validated_caller_metadata(params.metadata)?, - idle_policy, created_at_ms: now_ms()?, }, ) .await - .map_err(map_environments_error) + .map_err(map_environments_error)?; + if environment.state == EnvironmentState::Provisioning { + self.signals.nudge_work(); + } + Ok(EnvironmentCreateResponse { + environment: environment_view(&environment), + }) } - /// `environments/power/put`: record power intent. Provider support is - /// checked against the states observed on the current incarnation; the - /// reconciler converges asynchronously. + /// `environments/power/put`: a synchronous `target/setPower` + /// pass-through. Only the returned observation is stored. pub(crate) async fn put_environment_power_record( &self, params: EnvironmentPowerPutParams, ) -> Result { let environment_id = parse_registry_environment_id(params.environment_id)?; - let desired_power = registry_power_state(params.power); - let environment = EnvironmentStore::read_environment(self.store.as_ref(), &environment_id) - .await - .map_err(map_environments_error)?; - if !matches!(environment.source, EnvironmentSource::Provisioned { .. }) { - return Err(AgentApiError::rejected( - "external environments have no power control", - )); - } - if desired_power != PowerState::Running - && !environment - .incarnation - .power_states - .contains(&desired_power) - { - return Err(AgentApiError::rejected( - if environment.incarnation.power_states.is_empty() { - format!( - "environment power state {desired_power} is unavailable: the provider has not reported power support for this environment yet" - ) - } else { - format!( - "environment power state {desired_power} is not supported by the provider (supported: {})", - environment - .incarnation - .power_states - .iter() - .map(|state| state.as_str()) - .collect::>() - .join(", ") - ) + let power = registry_power_state(params.power); + let environment = self + .open_provisioned(&environment_id, "power control") + .await?; + let environment = self + .pass_through( + &environment, + ControllerFeature::Power, + |controller, target| { + Box::pin(async move { + controller + .client + .call_with_timeout::( + &SetTargetPowerParams { target, power }, + PASS_THROUGH_TIMEOUT, + ) + .await + }) }, - )); - } - let environment = EnvironmentStore::set_environment_power( - self.store.as_ref(), - SetEnvironmentPower { - environment_id, - desired_power, - updated_at_ms: now_ms()?, - }, - ) - .await - .map_err(map_environments_error)?; + ) + .await?; Ok(EnvironmentPowerPutResponse { environment: environment_view(&environment), }) } - /// `environments/idle-policy/put`: replace or clear the staged idle - /// policy of a provisioned environment. - pub(crate) async fn put_environment_idle_policy_record( + /// `environments/ingress/put`: a synchronous `ingress/ensure` or + /// `ingress/remove` pass-through; the reported endpoint is cached. + pub(crate) async fn put_environment_ingress_record( &self, - params: EnvironmentIdlePolicyPutParams, - ) -> Result { + params: EnvironmentIngressPutParams, + ) -> Result { let environment_id = parse_registry_environment_id(params.environment_id)?; - let idle_policy = params.idle_policy.as_ref().map(registry_idle_policy); - if let Some(policy) = &idle_policy { - policy.validate().map_err(map_environments_error)?; + let enabled = params.enabled; + let environment = self + .open_provisioned(&environment_id, "provider-managed ingress") + .await?; + let environment = self + .pass_through( + &environment, + ControllerFeature::Ingress, + |controller, target| { + Box::pin(async move { + if enabled { + controller + .client + .call_with_timeout::( + &EnsureIngressParams { target }, + PASS_THROUGH_TIMEOUT, + ) + .await + } else { + controller + .client + .call_with_timeout::( + &RemoveIngressParams { target }, + PASS_THROUGH_TIMEOUT, + ) + .await + } + }) + }, + ) + .await?; + Ok(EnvironmentIngressPutResponse { + environment: environment_view(&environment), + }) + } + + /// The environment a pass-through acts on: provisioned and open. + async fn open_provisioned( + &self, + environment_id: &EnvironmentId, + what: &str, + ) -> Result { + let environment = EnvironmentStore::read_environment(self.store.as_ref(), environment_id) + .await + .map_err(map_environments_error)?; + if environment.provisioned().is_none() { + return Err(AgentApiError::rejected(format!( + "{what} is available only for provisioned environments" + ))); + } + if environment.state != EnvironmentState::Open { + return Err(AgentApiError::rejected(format!( + "environment {environment_id} is {}; {what} needs an open environment", + environment.state + ))); } - let environment = EnvironmentStore::set_environment_idle_policy( + Ok(environment) + } + + /// Run one synchronous controller call against an open provisioned + /// environment's target and record the observation it answers. + async fn pass_through( + &self, + environment: &EnvironmentRecord, + feature: ControllerFeature, + call: F, + ) -> Result + where + F: for<'a> FnOnce( + &'a ProviderController, + TargetRef, + ) -> futures_util::future::BoxFuture< + 'a, + Result, + >, + { + let source = environment + .provisioned() + .ok_or_else(|| AgentApiError::internal("pass-through on a non-provisioned row"))?; + let provider_id = source.provider_id.clone(); + let target_id = source.provider_target_id.clone().ok_or_else(|| { + AgentApiError::rejected(format!( + "environment {} has no provider target", + environment.environment_id + )) + })?; + let owner = target_owner(self.store.config().universe_id, environment) + .map_err(map_environments_error)?; + let controller = self + .connect_provider_controller(&provider_id) + .await + .map_err(ControllerError::into_api_error)?; + let answered = match controller.require(feature) { + Ok(()) => call( + &controller, + TargetRef { + owner: owner.clone(), + target_id: Some(target_id), + }, + ) + .await + .map_err(|error| controller.call_error(error)), + Err(error) => Err(error), + }; + controller.close().await; + let observation = answered.map_err(ControllerError::into_api_error)?.target; + if observation.owner != owner { + return Err(AgentApiError::internal(format!( + "provider {provider_id} answered with another owner's target" + ))); + } + EnvironmentStore::record_target_observation( self.store.as_ref(), - SetEnvironmentIdlePolicy { - environment_id, - idle_policy, - updated_at_ms: now_ms()?, - }, + observation_request(environment, &observation, now_ms()?), ) .await - .map_err(map_environments_error)?; - Ok(EnvironmentIdlePolicyPutResponse { - environment: environment_view(&environment), - }) + .map_err(map_environments_error) } pub(crate) async fn read_environment_record( @@ -310,360 +399,960 @@ impl EnvironmentService { }) } + /// `environments/update`: replace the display name and caller metadata. + pub(crate) async fn update_environment_record( + &self, + params: EnvironmentUpdateParams, + ) -> Result { + let environment_id = parse_registry_environment_id(params.environment_id)?; + let environment = EnvironmentStore::update_environment( + self.store.as_ref(), + UpdateEnvironment { + environment_id, + display_name: params.display_name, + metadata: validated_caller_metadata(params.metadata)?, + updated_at_ms: now_ms()?, + }, + ) + .await + .map_err(map_environments_error)?; + Ok(EnvironmentUpdateResponse { + environment: environment_view(&environment), + }) + } + pub(crate) async fn list_environment_records( &self, params: EnvironmentListParams, ) -> Result { - let environments = EnvironmentStore::list_environments( + let page = EnvironmentStore::list_environments( self.store.as_ref(), ListEnvironments { - metadata: params.metadata, provider_id: params .provider_id .map(parse_environment_provider_id) .transpose()?, - binding_id: params - .binding_id - .map(parse_environment_provider_binding_id) - .transpose()?, - status: params.status.map(registry_lifecycle_status), registration_key_id: params .registration_key_id .map(parse_registration_key_id) .transpose()?, + state: params.state.map(registry_state), + include_closed: params.include_closed, + metadata: params.metadata, + limit: params + .limit + .unwrap_or(::environments::LIST_ENVIRONMENTS_DEFAULT_LIMIT), + after: params + .after + .map(parse_registry_environment_id) + .transpose()?, }, ) .await .map_err(map_environments_error)?; Ok(EnvironmentListResponse { - environments: environments.iter().map(environment_view).collect(), + environments: page.environments.iter().map(environment_view).collect(), + next_after: page.next_after.map(|id| id.to_string()), }) } + /// `environments/close`: a provisioned environment moves to `closing` + /// and the reconciler is nudged; external and registered ones close at + /// once and their live routes end. pub(crate) async fn close_environment_record( &self, params: EnvironmentCloseParams, ) -> Result { let environment_id = parse_registry_environment_id(params.environment_id)?; + let environment = self + .begin_close(&environment_id, EnvironmentCloseReason::Requested) + .await?; + Ok(EnvironmentCloseResponse { + environment: environment_view(&environment), + }) + } + + /// Begin closing one environment with `reason`, then signal: the + /// reconciler for a provisioned row, live routes for every row. + pub(crate) async fn begin_close( + &self, + environment_id: &EnvironmentId, + reason: EnvironmentCloseReason, + ) -> Result { let environment = EnvironmentStore::begin_close_environment( self.store.as_ref(), BeginCloseEnvironment { - environment_id, - updated_at_ms: now_ms()?, + environment_id: environment_id.clone(), + reason, + at_ms: now_ms()?, }, ) .await .map_err(map_environments_error)?; - Ok(EnvironmentCloseResponse { - environment: environment_view(&environment), - }) + if environment.state == EnvironmentState::Closing { + self.signals.nudge_work(); + } + self.signals + .revoke(self.store.config().universe_id, environment_id.as_str()); + Ok(environment) } - /// Public entry point for one reconciliation pass; used by acceptance - /// tests that drive the reconciler deterministically instead of running - /// the background loop. + /// Start a work pass, as its own task, for every provider of this + /// universe with due provisioning or closing rows and no pass in + /// flight; returns how many it started. The caller never waits on a + /// provider, so a slow or hung controller delays only its own rows. + pub(crate) async fn start_environment_work( + &self, + now_ms: i64, + work: &ProviderWork, + ) -> Result { + let universe_id = self.store.config().universe_id; + let rows = self + .store + .list_environment_work(now_ms, WORK_BATCH) + .await + .map_err(map_environments_error)?; + let mut started = 0; + for (provider_id, rows) in by_provider(rows) { + let Some(claim) = work.claim(universe_id, &provider_id) else { + continue; + }; + let service = self.clone(); + tokio::spawn(async move { + let outcome = service + .reconciler() + .work_provider(provider_id, rows, now_ms) + .await; + claim.finish(outcome.into_result()); + }); + started += 1; + } + Ok(started) + } + + /// One display sweep over this universe's open provisioned rows. + pub(crate) async fn run_display_sweep(&self, now_ms: i64) -> Result { + self.reconciler().run_display_sweep(now_ms).await + } + + /// One work pass that treats every scheduled row as due, then the + /// display sweep of this universe; acceptance tests drive the + /// reconciler with it instead of the background loop. pub async fn reconcile_environments_once(&self) -> Result { - self.reconcile_environment_lifecycle_once().await + let reconciler = self.reconciler(); + let worked = reconciler.run_work(i64::MAX, clock_ms()).await; + let swept = reconciler.run_display_sweep(clock_ms()).await; + Ok(worked? + swept?) + } +} + +/// The reconciler of one universe, over its stores. +pub(crate) struct Reconciler<'a> { + pub(crate) universe_id: uuid::Uuid, + pub(crate) environments: &'a dyn EnvironmentStore, + pub(crate) providers: &'a dyn EnvironmentProviderStore, + pub(crate) connector: &'a dyn ProviderControllerConnector, + pub(crate) signals: &'a EnvironmentSignals, +} + +/// What one pass over a group of rows did. +#[derive(Default)] +struct PassOutcome { + changed: usize, + first_error: Option, +} + +impl PassOutcome { + fn absorb(&mut self, environment_id: &EnvironmentId, result: Result) { + match result { + Ok(changed) => self.changed += usize::from(changed), + Err(error) => { + tracing::debug!( + target: "temporal_runtime", + %environment_id, + %error, + "environment reconcile step failed" + ); + self.first_error.get_or_insert(AgentApiError { + message: format!("environment {environment_id}: {}", error.message), + ..error + }); + } + } + } + + fn merge(&mut self, other: PassOutcome) { + self.changed += other.changed; + if self.first_error.is_none() { + self.first_error = other.first_error; + } + } + + fn into_result(self) -> Result { + match self.first_error { + Some(error) => Err(error), + None => Ok(self.changed), + } + } +} + +/// Group rows by provider; rows without one are not provisioned. +fn by_provider( + rows: Vec, +) -> BTreeMap> { + let mut groups: BTreeMap> = BTreeMap::new(); + for row in rows { + if let Some(provider_id) = row.provider_id().cloned() { + groups.entry(provider_id).or_default().push(row); + } } + groups +} - /// Reconcile every pending environment in this universe once. Calls are - /// stable and idempotent; a crash after controller I/O is recovered by a - /// later pass issuing the same IDs again. - pub(crate) async fn reconcile_environment_lifecycle_once( +/// How a step's registry failure is treated: the row's own records that +/// cannot be written (a contract breach the provider caused) back off like +/// a transient error; a missing row is done; a store failure is the pass's. +fn step_registry_error(error: &EnvironmentRegistryError) -> Option { + match error { + EnvironmentRegistryError::InvalidInput { message } => Some(message.clone()), + EnvironmentRegistryError::AlreadyExists { kind, id } => { + Some(format!("the provider's answer conflicts with {kind} {id}")) + } + _ => None, + } +} + +impl Reconciler<'_> { + /// Work on the rows due at `list_at_ms`; `now_ms` stamps writes and + /// decides deadlines and backoff. Providers run in parallel, each with + /// at most [`PROVIDER_CONCURRENCY`] rows in flight over one controller + /// connection. + pub(crate) async fn run_work( &self, + list_at_ms: i64, + now_ms: i64, ) -> Result { - let mut changed = self.reconcile_registered_once().await?; - let environments = - EnvironmentStore::list_environments_needing_reconcile(self.store.as_ref()) - .await - .map_err(map_environments_error)?; - // One unreachable provider must not stall every other environment in - // the universe: per-environment failures are remembered and the pass - // continues; the first failure is reported once the pass is complete. - let mut first_error: Option = None; - for environment in environments { - let outcome = match environment.status { - EnvironmentStatus::Closing => self.reconcile_environment_close(&environment).await, - EnvironmentStatus::Ready - | EnvironmentStatus::Paused - | EnvironmentStatus::Suspended - | EnvironmentStatus::Offline - if environment.power_diverges() => - { - self.reconcile_environment_power(&environment).await - } - EnvironmentStatus::Provisioning - | EnvironmentStatus::Booting - | EnvironmentStatus::Unknown => { - match self.reconcile_environment_create(&environment).await { - Err(error) if error.kind == api::AgentApiErrorKind::Rejected => { - EnvironmentStore::fail_environment_lifecycle( - self.store.as_ref(), - FailEnvironmentLifecycle { - environment_id: environment.environment_id.clone(), - message: error.message, - observed_at_ms: now_ms()?, - }, - ) - .await - .map_err(map_environments_error) - .map(|_| true) - } - other => other, + let rows = self + .environments + .list_environment_work(list_at_ms, WORK_BATCH) + .await + .map_err(map_environments_error)?; + let groups = by_provider(rows); + let outcomes = futures_util::future::join_all( + groups + .into_iter() + .map(|(provider_id, rows)| self.work_provider(provider_id, rows, now_ms)), + ) + .await; + let mut outcome = PassOutcome::default(); + for provider in outcomes { + outcome.merge(provider); + } + outcome.into_result() + } + + async fn work_provider( + &self, + provider_id: EnvironmentProviderId, + rows: Vec, + now_ms: i64, + ) -> PassOutcome { + let mut outcome = PassOutcome::default(); + let controller = + match connect_provider_controller(self.providers, self.connector, &provider_id).await { + Ok(controller) => Some(controller), + Err(error) => { + let unregistered = matches!( + error, + ControllerError::Registry(EnvironmentRegistryError::NotFound { .. }) + ); + let message = error.to_string(); + for row in &rows { + let result = self + .without_controller(row, unregistered, &message, now_ms) + .await; + outcome.absorb(&row.environment_id, result); } + None } - _ => Ok(false), }; - match outcome { - Ok(did_change) => changed += usize::from(did_change), - Err(error) => { - if first_error.is_none() { - first_error = Some(AgentApiError { - message: format!( - "environment {}: {}", - environment.environment_id, error.message - ), - ..error - }); + let Some(controller) = controller else { + return outcome; + }; + let controller = &controller; + let results = stream::iter(rows) + .map(|row| async move { + let result = self.step(controller, &row, now_ms).await; + (row.environment_id, result) + }) + .buffer_unordered(PROVIDER_CONCURRENCY) + .collect::>() + .await; + controller.close().await; + for (environment_id, result) in results { + outcome.absorb(&environment_id, result); + } + outcome + } + + /// A due row whose provider controller could not be reached. A closing + /// row of a provider that is no longer registered is closed: nothing + /// can reach its target any more. + async fn without_controller( + &self, + row: &EnvironmentRecord, + unregistered: bool, + message: &str, + now_ms: i64, + ) -> Result { + let result = match row.state { + EnvironmentState::Closing if unregistered => { + tracing::warn!( + target: "temporal_runtime", + environment_id = %row.environment_id, + "closing an environment whose provider is no longer registered" + ); + self.finish_close(row, now_ms).await + } + EnvironmentState::Provisioning if provisioning_deadline_passed(row, now_ms) => { + self.deadline(row, now_ms).await + } + EnvironmentState::Provisioning if unregistered => { + self.fail(row, message, None, now_ms).await + } + _ => self.backoff(row, message, now_ms).await, + }; + result.map_err(map_environments_error) + } + + async fn step( + &self, + controller: &ProviderController, + row: &EnvironmentRecord, + now_ms: i64, + ) -> Result { + let attempt = match row.state { + EnvironmentState::Provisioning => self.provision_step(controller, row, now_ms).await, + EnvironmentState::Closing => self.close_step(controller, row, now_ms).await, + _ => Ok(false), + }; + match attempt { + Ok(changed) => Ok(changed), + Err(EnvironmentRegistryError::NotFound { .. }) => Ok(false), + Err(error) => match step_registry_error(&error) { + Some(message) => self + .backoff(row, &message, now_ms) + .await + .map_err(map_environments_error), + None => Err(map_environments_error(error)), + }, + } + } + + /// One step of a provisioning row: create while no target id is + /// recorded, observe once one is, and open after one data-plane + /// handshake with the target's envd. + async fn provision_step( + &self, + controller: &ProviderController, + row: &EnvironmentRecord, + now_ms: i64, + ) -> Result { + if provisioning_deadline_passed(row, now_ms) { + return self.deadline(row, now_ms).await; + } + let Some(source) = row.provisioned() else { + return Ok(false); + }; + let owner = target_owner(self.universe_id, row)?; + let answer = match &source.provider_target_id { + Some(target_id) => { + controller + .client + .get_target(&GetTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: Some(target_id.clone()), + }, + }) + .await + } + None => { + let template_id = + TemplateId::new(source.template_id.as_str()).map_err(|error| { + EnvironmentRegistryError::InvalidInput { + message: format!("template id does not fit the protocol: {error}"), + } + })?; + controller + .client + .create_target(&CreateTargetParams { + owner: owner.clone(), + template_id, + display_name: row + .display_name + .as_deref() + .and_then(|name| DisplayName::new(name).ok()), + idle: source.idle.clone(), + }) + .await + } + }; + let observation = match answer { + Ok(response) => response.target, + Err(error) => { + let error = controller.call_error(error); + return match error.disposition() { + ControllerDisposition::Transient => { + self.backoff(row, &error.to_string(), now_ms).await } - } + ControllerDisposition::Refused => { + self.fail(row, &error.to_string(), None, now_ms).await + } + ControllerDisposition::Gone => { + let message = if source.provider_target_id.is_some() { + format!("the provider target disappeared: {error}") + } else { + error.to_string() + }; + self.fail(row, &message, None, now_ms).await + } + }; } + }; + if observation.owner != owner { + return self + .backoff( + row, + &format!( + "provider {} answered with another owner's target", + controller.provider_id + ), + now_ms, + ) + .await; } - match first_error { - Some(error) => Err(error), - None => Ok(changed), + let recorded = self + .environments + .record_target_observation(observation_request(row, &observation, now_ms)) + .await?; + // A close that landed while this step was in flight owns the row's + // schedule now; its own step picks the target up next. + if recorded.state != EnvironmentState::Provisioning { + return Ok(true); + } + let message = observation.message.as_ref().map(|message| message.as_str()); + match observation.phase { + TargetPhase::Provisioning => { + let note = match message { + Some(message) => format!("waiting for the provider: {message}"), + None => "waiting for the provider".to_owned(), + }; + self.schedule(&recorded, now_ms + POLL_MS, 0, Some(note), now_ms) + .await + } + TargetPhase::Active => { + self.open_handshake(controller, &recorded, &owner, now_ms) + .await + } + TargetPhase::Failed => { + self.fail( + &recorded, + message.unwrap_or("the provider reported the target failed"), + None, + now_ms, + ) + .await + } + TargetPhase::Closing | TargetPhase::Closed => { + self.fail( + &recorded, + "the provider ended the target before it opened", + None, + now_ms, + ) + .await + } + TargetPhase::Unknown => { + self.backoff( + &recorded, + message.unwrap_or("the provider cannot tell the target's state"), + now_ms, + ) + .await + } } } - async fn reconcile_environment_create( + /// The only readiness proof: `initialize` over the target's data route. + /// A typed refusal that may clear is waited out at its `Retry-After`. + async fn open_handshake( &self, - environment: &::environments::EnvironmentRecord, - ) -> Result { - let EnvironmentSource::Provisioned { - provider_id, - binding_id, - } = &environment.source + controller: &ProviderController, + row: &EnvironmentRecord, + owner: &TargetOwner, + now_ms: i64, + ) -> Result { + let Some(target_id) = row + .provisioned() + .and_then(|source| source.provider_target_id.clone()) else { - return Ok(false); + return self + .backoff( + row, + "the provider reported an active target without a targetId", + now_ms, + ) + .await; }; - let provider = self.read_environment_provider(provider_id).await?; - let binding = EnvironmentProviderBindingStore::read_provider_binding( - self.store.as_ref(), - self.store.config().universe_id, - binding_id, + let route = provider_data_route(&controller.data_route_base, owner, &target_id); + match DataClient::connect_initialized( + &route, + provider_connect_options(controller.credential.as_ref()), ) .await - .map_err(map_environments_error)?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) - .await?; - let target = async { - match ( - environment.incarnation.template_id.as_ref(), - environment.incarnation.adoption_source_target.as_ref(), - ) { - (Some(template_id), None) => Ok(controller - .create_target(&CreateTargetParams { - request_id: environment.request_id.to_string(), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - template_id: template_id.to_string(), - }) - .await? - .target), - (None, Some(source_target)) => Ok(controller - .adopt_target(&AdoptTargetParams { - request_id: environment.request_id.to_string(), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - source_target: source_target.clone(), + { + Ok((client, initialized)) => { + let _ = client.close().await; + let written = self + .environments + .open_environment(OpenEnvironment { + environment_id: row.environment_id.clone(), + daemon_build: DaemonBuild::from_implementation( + &initialized.server_info, + initialized.protocol_version, + ), + at_ms: now_ms, }) - .await? - .target), - _ => Err(AgentApiError::internal( - "provisioned incarnation has invalid realization", - )), + .await?; + Ok(written.state == EnvironmentState::Open) + } + Err(ClientError::VersionMismatch { local, remote, .. }) => { + self.fail( + row, + &format!( + "the environment's envd speaks protocol version {remote}; this runtime requires {local}" + ), + Some(DaemonBuild { + version: None, + git_sha: None, + protocol_version: remote, + }), + now_ms, + ) + .await } + Err(error) => match error.data_route_refusal() { + Some(refusal) if refusal.is_retryable() => { + let retry_after = i64::try_from( + error.retry_after_ms().unwrap_or(WAIT_RETRY_AFTER_MS), + ) + .unwrap_or(i64::MAX); + self.schedule( + row, + now_ms.saturating_add(retry_after), + 0, + Some(format!("waiting for envd: {refusal}")), + now_ms, + ) + .await + } + Some(DataRouteRefusal::Gone {}) => { + self.fail(row, "the provider closed the target", None, now_ms) + .await + } + Some(DataRouteRefusal::Failed { message }) => { + self.fail(row, message.as_str(), None, now_ms).await + } + _ => { + self.backoff( + row, + &format!("envd did not complete the data-plane handshake: {error}"), + now_ms, + ) + .await + } + }, } - .await; - let target = finish_provider_controller(controller, target).await?; - self.record_target_observation(environment, target).await } - /// Persist a provider observation when it changes the record: status, - /// target id, or the provider-reported power states. - async fn record_target_observation( + /// One step of a closing row: `target/close` by owner, with the target + /// id when one is recorded, until the provider reports it gone. Closing + /// has no deadline: a provider still closing is polled, and any other + /// phase it answers (a target it cannot remove) backs off like an + /// error. + async fn close_step( &self, - environment: &::environments::EnvironmentRecord, - target: ProviderTargetSummary, - ) -> Result { - let status = lifecycle_status_from_target(target.status); - let changed = environment.status != status - || environment.incarnation.provider_target_id.as_ref() != Some(&target.target_id) - || environment.incarnation.power_states != target.power_states; - if changed { - EnvironmentStore::observe_provisioned_environment( - self.store.as_ref(), - ObserveProvisionedEnvironment { - environment_id: environment.environment_id.clone(), - provider_target_id: target.target_id, - status, - power_states: target.power_states, - observed_at_ms: now_ms()?, - }, - ) + controller: &ProviderController, + row: &EnvironmentRecord, + now_ms: i64, + ) -> Result { + let owner = target_owner(self.universe_id, row)?; + let target_id = row + .provisioned() + .and_then(|source| source.provider_target_id.clone()); + match controller + .client + .close_target(&CloseTargetParams { + target: TargetRef { owner, target_id }, + }) + .await + { + Ok(response) => match response.target.phase { + TargetPhase::Closed => self.finish_close(row, now_ms).await, + TargetPhase::Closing => { + self.schedule( + row, + now_ms + POLL_MS, + 0, + Some("waiting for the provider to close the target".to_owned()), + now_ms, + ) + .await + } + phase => { + let message = match response.target.message.as_ref() { + Some(message) => format!( + "the provider answered close with a {} target: {}", + phase_name(phase), + message.as_str() + ), + None => format!( + "the provider answered close with a {} target", + phase_name(phase) + ), + }; + self.backoff(row, &message, now_ms).await + } + }, + Err(error) if call_disposition(&error) == ControllerDisposition::Gone => { + self.finish_close(row, now_ms).await + } + Err(error) => { + let error = controller.call_error(error); + self.backoff(row, &error.to_string(), now_ms).await + } + } + } + + /// Observe every open provisioned row with `target/get` and record what + /// the provider reports. A failed target fails its row; a closed or + /// missing one closes it (`provider_ended`). Power is never touched. + pub(crate) async fn run_display_sweep(&self, now_ms: i64) -> Result { + let rows = self + .environments + .list_open_provisioned_environments() .await .map_err(map_environments_error)?; + let outcomes = futures_util::future::join_all( + by_provider(rows) + .into_iter() + .map(|(provider_id, rows)| self.sweep_provider(provider_id, rows, now_ms)), + ) + .await; + let mut outcome = PassOutcome::default(); + for provider in outcomes { + outcome.merge(provider); } - Ok(changed) + outcome.into_result() } - /// Converge a steady-state provisioned environment toward its desired - /// power. One idempotent `setTargetPower` per pass; the observed - /// summary is recorded like any other observation, so a transitional - /// answer (`Starting`) flows back through the create/observe path. - async fn reconcile_environment_power( + async fn sweep_provider( &self, - environment: &::environments::EnvironmentRecord, - ) -> Result { - let EnvironmentSource::Provisioned { - provider_id, - binding_id, - } = &environment.source - else { - return Ok(false); - }; - let Some(target_id) = environment.incarnation.provider_target_id.clone() else { - return Ok(false); - }; - let provider = self.read_environment_provider(provider_id).await?; - let binding = EnvironmentProviderBindingStore::read_provider_binding( - self.store.as_ref(), - self.store.config().universe_id, - binding_id, - ) - .await - .map_err(map_environments_error)?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) - .await?; - let response = controller - .set_target_power(&SetTargetPowerParams { - request_id: format!( - "power:{}:{}", - environment.request_id, environment.desired_power - ), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id, - power: environment.desired_power, + provider_id: EnvironmentProviderId, + rows: Vec, + now_ms: i64, + ) -> PassOutcome { + let mut outcome = PassOutcome::default(); + let controller = + match connect_provider_controller(self.providers, self.connector, &provider_id).await { + Ok(controller) => controller, + Err(error) => { + tracing::debug!( + target: "temporal_runtime", + %provider_id, + %error, + "display sweep skipped an unreachable provider" + ); + return outcome; + } + }; + let controller = &controller; + let results = stream::iter(rows) + .map(|row| async move { + let result = self.observe_open(controller, &row, now_ms).await; + (row.environment_id, result) }) + .buffer_unordered(PROVIDER_CONCURRENCY) + .collect::>() .await; - let response = finish_provider_controller(controller, response).await?; - self.record_target_observation(environment, response.target) - .await + controller.close().await; + for (environment_id, result) in results { + outcome.absorb( + &environment_id, + match result { + Err(EnvironmentRegistryError::NotFound { .. }) => Ok(false), + other => other.map_err(map_environments_error), + }, + ); + } + outcome } - async fn reconcile_environment_close( + async fn observe_open( &self, - environment: &::environments::EnvironmentRecord, - ) -> Result { - let Some(target_id) = environment.incarnation.provider_target_id.clone() else { - EnvironmentStore::finish_close_environment( - self.store.as_ref(), - FinishCloseEnvironment { - environment_id: environment.environment_id.clone(), - observed_at_ms: now_ms()?, + controller: &ProviderController, + row: &EnvironmentRecord, + now_ms: i64, + ) -> Result { + let Some(target_id) = row + .provisioned() + .and_then(|source| source.provider_target_id.clone()) + else { + return Ok(false); + }; + let owner = target_owner(self.universe_id, row)?; + let observation = match controller + .client + .get_target(&GetTargetParams { + target: TargetRef { + owner: owner.clone(), + target_id: Some(target_id), }, - ) + }) .await - .map_err(map_environments_error)?; - return Ok(true); + { + Ok(response) => response.target, + Err(error) if call_disposition(&error) == ControllerDisposition::Gone => { + return self.end(row, now_ms).await; + } + Err(error) => { + tracing::debug!( + target: "temporal_runtime", + environment_id = %row.environment_id, + %error, + "display sweep could not observe a target" + ); + return Ok(false); + } }; - let EnvironmentSource::Provisioned { - provider_id, - binding_id, - } = &environment.source - else { + if observation.owner != owner { return Ok(false); - }; - let provider = self.read_environment_provider(provider_id).await?; - let binding = EnvironmentProviderBindingStore::read_provider_binding( - self.store.as_ref(), - self.store.config().universe_id, - binding_id, - ) - .await - .map_err(map_environments_error)?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) + } + let recorded = self + .environments + .record_target_observation(observation_request(row, &observation, now_ms)) .await?; - let response = async { - let ingress_response = controller - .remove_ingress(&RemoveIngressParams { - request_id: format!("ingress-close:{}", environment.request_id), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id: target_id.clone(), - }) - .await?; - if ingress_response.status != ProviderIngressStatus::Disabled { - return Err(AgentApiError::rejected( - "provider did not remove ingress before target close", - )); + match observation.phase { + TargetPhase::Failed => { + let message = observation + .message + .as_ref() + .map(|message| message.as_str()) + .unwrap_or("the provider reported the target failed"); + self.fail(&recorded, message, None, now_ms).await } - controller - .close_target(&CloseTargetParams { - request_id: format!("close:{}", environment.request_id), - environment_id: environment.environment_id.to_string(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - binding: binding_context(&binding), - target_id, - force: false, - }) - .await + TargetPhase::Closed => self.end(&recorded, now_ms).await, + _ => Ok(recorded != *row), } - .await; - let response = finish_provider_controller(controller, response).await?; - if response.status == ProviderTargetStatus::Closed { - EnvironmentStore::finish_close_environment( - self.store.as_ref(), - FinishCloseEnvironment { - environment_id: environment.environment_id.clone(), - observed_at_ms: now_ms()?, - }, - ) - .await - .map_err(map_environments_error)?; + } + + async fn deadline( + &self, + row: &EnvironmentRecord, + now_ms: i64, + ) -> Result { + let message = format!( + "provisioning did not finish within {} minutes: {}", + PROVISIONING_DEADLINE_MS / 60_000, + row.state_message.as_deref().unwrap_or("no progress") + ); + self.fail(row, &message, None, now_ms).await + } + + async fn backoff( + &self, + row: &EnvironmentRecord, + message: &str, + now_ms: i64, + ) -> Result { + let attempts = row + .provisioned() + .map_or(0, |source| source.attempts) + .saturating_add(1); + self.schedule( + row, + now_ms.saturating_add(backoff_delay_ms(attempts)), + attempts, + Some(message.to_owned()), + now_ms, + ) + .await + } + + async fn schedule( + &self, + row: &EnvironmentRecord, + next_attempt_at_ms: i64, + attempts: u32, + state_message: Option, + now_ms: i64, + ) -> Result { + let written = self + .environments + .schedule_environment_attempt(ScheduleEnvironmentAttempt { + environment_id: row.environment_id.clone(), + expected_state: row.state, + next_attempt_at_ms, + attempts, + state_message, + at_ms: now_ms, + }) + .await?; + Ok(written != *row) + } + + async fn fail( + &self, + row: &EnvironmentRecord, + message: &str, + daemon_build: Option, + now_ms: i64, + ) -> Result { + let written = self + .environments + .fail_environment(FailEnvironment { + environment_id: row.environment_id.clone(), + message: message.to_owned(), + daemon_build, + at_ms: now_ms, + }) + .await?; + Ok(written.state == EnvironmentState::Failed) + } + + async fn finish_close( + &self, + row: &EnvironmentRecord, + now_ms: i64, + ) -> Result { + let written = self + .environments + .finish_close_environment(FinishCloseEnvironment { + environment_id: row.environment_id.clone(), + at_ms: now_ms, + }) + .await?; + self.signals + .revoke(self.universe_id, row.environment_id.as_str()); + Ok(written.state == EnvironmentState::Closed) + } + + async fn end( + &self, + row: &EnvironmentRecord, + now_ms: i64, + ) -> Result { + let written = self + .environments + .end_provisioned_environment(EndProvisionedEnvironment { + environment_id: row.environment_id.clone(), + at_ms: now_ms, + }) + .await?; + self.signals + .revoke(self.universe_id, row.environment_id.as_str()); + Ok(written.state == EnvironmentState::Closed) + } +} + +/// The provider work passes in flight, one per universe and provider at a +/// time, and the rate limit of their failure warnings. +#[derive(Clone, Default)] +pub(crate) struct ProviderWork { + inner: std::sync::Arc>, +} + +#[derive(Default)] +struct ProviderWorkState { + running: std::collections::HashSet<(uuid::Uuid, EnvironmentProviderId)>, + failures: std::collections::HashMap< + (uuid::Uuid, EnvironmentProviderId), + (String, std::time::Instant), + >, +} + +impl ProviderWork { + /// Claim the provider's next pass in the universe; `None` while one is + /// in flight. + fn claim( + &self, + universe_id: uuid::Uuid, + provider_id: &EnvironmentProviderId, + ) -> Option { + let key = (universe_id, provider_id.clone()); + let mut state = self.inner.lock().expect("provider work poisoned"); + if !state.running.insert(key.clone()) { + return None; } - Ok(true) + Some(ProviderWorkClaim { + work: self.clone(), + key, + }) + } + + #[cfg(test)] + fn in_flight(&self) -> usize { + self.inner + .lock() + .expect("provider work poisoned") + .running + .len() } } -fn external_connection_from_api( - value: EnvironmentConnectionView, -) -> Result<::environments::EnvironmentConnectionSpec, AgentApiError> { - let transport = match value.transport { - EnvironmentConnectionTransportView::WebSocket => { - environment_protocol::shared::EnvironmentTransport::WebSocket +/// A claimed provider pass; released when dropped, so a pass that panics +/// still frees its provider. +struct ProviderWorkClaim { + work: ProviderWork, + key: (uuid::Uuid, EnvironmentProviderId), +} + +impl ProviderWorkClaim { + /// Record the pass's outcome: a failure warns when it is new or its + /// last warning is older than [`ReconcileFailureLog::REPEAT_INTERVAL`]. + fn finish(self, result: Result) { + let (universe_id, provider_id) = (&self.key.0, &self.key.1); + let mut state = self.work.inner.lock().expect("provider work poisoned"); + match result { + Ok(_) => { + state.failures.remove(&self.key); + } + Err(error) => { + let message = error.to_string(); + let now = std::time::Instant::now(); + let repeat = state.failures.get(&self.key).is_some_and(|(last, at)| { + *last == message + && now.duration_since(*at) < ReconcileFailureLog::REPEAT_INTERVAL + }); + if repeat { + tracing::debug!(target: "temporal_runtime", %universe_id, %provider_id, error = %message, "environment provider work pass still failing"); + } else { + tracing::warn!(target: "temporal_runtime", %universe_id, %provider_id, error = %message, "environment provider work pass failed (repeats suppressed for 60s)"); + state.failures.insert(self.key.clone(), (message, now)); + } + } } - _ => { - return Err(AgentApiError::invalid_request( - "external environment connections currently require webSocket transport", - )); + drop(state); + } +} + +impl Drop for ProviderWorkClaim { + fn drop(&mut self) { + if let Ok(mut state) = self.work.inner.lock() { + state.running.remove(&self.key); } - }; - Ok(::environments::EnvironmentConnectionSpec { - endpoint: value.endpoint, - transport, - }) + } } /// Rate-limits repeated reconcile-pass failures per universe so a provider @@ -712,12 +1401,14 @@ pub(crate) fn parse_registration_key_id( }) } -pub(crate) fn allocate_environment_id() -> EnvironmentId { - EnvironmentId::new(format!("environment_{}", uuid::Uuid::new_v4().simple())) +fn parse_request_id(value: String) -> Result { + EnvironmentProvisionRequestId::try_new(value).map_err(|error| { + AgentApiError::invalid_request(format!("invalid environment request id: {error}")) + }) } -pub(crate) fn allocate_incarnation_id() -> EnvironmentIncarnationId { - EnvironmentIncarnationId::new(format!("incarnation_{}", uuid::Uuid::new_v4().simple())) +pub(crate) fn allocate_environment_id() -> EnvironmentId { + EnvironmentId::new(format!("environment_{}", uuid::Uuid::new_v4().simple())) } /// Caller-supplied environment metadata: the same bounds and reserved-prefix @@ -728,3 +1419,404 @@ fn validated_caller_metadata( super::validate_caller_metadata(&metadata)?; Ok(metadata) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::environments::{ + fake_controller::{FAKE_TEMPLATE_ID, FakeController, FakeControllerOptions}, + provider_controllers::WebSocketProviderControllerConnector, + }; + use ::environments::{ + EnvironmentProviderBindingStore, InMemoryEnvironmentRegistryStore, PowerState, + ProviderCredentialChange, PutEnvironmentProvider, PutEnvironmentProviderBinding, + }; + use auth::SecretValue; + use environment_protocol::ENVD_PROTOCOL_VERSION; + + #[test] + fn backoff_doubles_from_two_seconds_up_to_five_minutes() { + assert_eq!(backoff_delay_ms(1), 2_000); + assert_eq!(backoff_delay_ms(2), 4_000); + assert_eq!(backoff_delay_ms(5), 32_000); + assert_eq!(backoff_delay_ms(8), 256_000); + assert_eq!(backoff_delay_ms(9), BACKOFF_MAX_MS); + assert_eq!(backoff_delay_ms(u32::MAX), BACKOFF_MAX_MS); + } + + #[test] + fn one_pass_per_universe_and_provider_is_in_flight_and_a_dropped_claim_frees_it() { + let work = ProviderWork::default(); + let universe = uuid::Uuid::from_u128(1); + let provider = EnvironmentProviderId::new("incus"); + let other = EnvironmentProviderId::new("sandbox"); + let claim = work.claim(universe, &provider).unwrap(); + assert!(work.claim(universe, &provider).is_none()); + let elsewhere = work.claim(uuid::Uuid::from_u128(2), &provider).unwrap(); + let sibling = work.claim(universe, &other).unwrap(); + assert_eq!(work.in_flight(), 3); + claim.finish(Ok(0)); + drop(elsewhere); + drop(sibling); + assert_eq!(work.in_flight(), 0); + assert!(work.claim(universe, &provider).is_some()); + } + + struct Fixture { + store: InMemoryEnvironmentRegistryStore, + signals: EnvironmentSignals, + provider_id: EnvironmentProviderId, + } + + impl Fixture { + async fn new(controller: &FakeController) -> Self { + Self::with_endpoint(controller.controller_url(), controller.credential()).await + } + + async fn with_endpoint(endpoint: &str, credential: Option<&str>) -> Self { + let store = InMemoryEnvironmentRegistryStore::new(); + let provider_id = EnvironmentProviderId::new("fake"); + store + .put_provider(PutEnvironmentProvider { + provider_id: provider_id.clone(), + display_name: None, + controller_endpoint: EnvironmentEndpoint::parse(endpoint).unwrap(), + data_route_base: None, + credential: match credential { + Some(credential) => { + ProviderCredentialChange::Set(SecretValue::new(credential)) + } + None => ProviderCredentialChange::Keep, + }, + updated_at_ms: 1, + }) + .await + .unwrap(); + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id: store.universe_id(), + provider_id: provider_id.clone(), + enabled: true, + updated_at_ms: 1, + }) + .await + .unwrap(); + Self { + store, + signals: EnvironmentSignals::default(), + provider_id, + } + } + + fn reconciler(&self) -> Reconciler<'_> { + Reconciler { + universe_id: self.store.universe_id(), + environments: &self.store, + providers: &self.store, + connector: &WebSocketProviderControllerConnector, + signals: &self.signals, + } + } + + async fn create(&self, name: &str, created_at_ms: i64) -> EnvironmentId { + let environment_id = EnvironmentId::new(name); + self.store + .create_environment(CreateEnvironment { + request_id: EnvironmentProvisionRequestId::new(format!("request-{name}")), + environment_id: environment_id.clone(), + provider_id: self.provider_id.clone(), + template_id: EnvironmentTemplateId::new(FAKE_TEMPLATE_ID), + idle: None, + display_name: Some("Test environment".to_owned()), + metadata: BTreeMap::new(), + created_at_ms, + }) + .await + .unwrap(); + environment_id + } + + async fn read(&self, environment_id: &EnvironmentId) -> EnvironmentRecord { + self.store.read_environment(environment_id).await.unwrap() + } + } + + #[tokio::test(flavor = "current_thread")] + async fn a_created_target_opens_after_one_handshake_that_records_the_build() { + let controller = FakeController::start().await.unwrap(); + let fixture = Fixture::new(&controller).await; + let environment_id = fixture.create("environment_a", 1_000).await; + fixture + .reconciler() + .run_work(i64::MAX, 2_000) + .await + .unwrap(); + let opened = fixture.read(&environment_id).await; + assert_eq!(opened.state, EnvironmentState::Open); + let source = opened.provisioned().unwrap(); + assert!(source.provider_target_id.is_some()); + assert_eq!(source.next_attempt_at_ms, None); + assert_eq!( + source.observed.as_ref().map(|observed| observed.phase), + Some(TargetPhase::Active) + ); + assert_eq!( + opened.daemon_build.map(|build| build.protocol_version), + Some(ENVD_PROTOCOL_VERSION) + ); + assert_eq!(controller.target_count().await, 1); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_booting_refusal_is_waited_out_at_its_retry_after() { + let controller = FakeController::start_with(FakeControllerOptions { + booting_dials: 1, + ..FakeControllerOptions::default() + }) + .await + .unwrap(); + let fixture = Fixture::new(&controller).await; + let environment_id = fixture.create("environment_a", 1_000).await; + fixture + .reconciler() + .run_work(i64::MAX, 2_000) + .await + .unwrap(); + let waiting = fixture.read(&environment_id).await; + assert_eq!(waiting.state, EnvironmentState::Provisioning); + assert_eq!( + waiting.state_message.as_deref(), + Some("waiting for envd: booting") + ); + let source = waiting.provisioned().unwrap(); + assert_eq!(source.attempts, 0, "a typed wait is not a failure"); + assert_eq!(source.next_attempt_at_ms, Some(3_000)); + // Not due before its Retry-After. + fixture.reconciler().run_work(2_500, 2_500).await.unwrap(); + assert_eq!( + fixture.read(&environment_id).await.state, + EnvironmentState::Provisioning + ); + fixture.reconciler().run_work(3_000, 3_000).await.unwrap(); + assert_eq!( + fixture.read(&environment_id).await.state, + EnvironmentState::Open + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn an_unreachable_provider_backs_off_per_row_until_the_deadline() { + let controller = FakeController::start().await.unwrap(); + controller.set_reachable(false); + let fixture = Fixture::new(&controller).await; + let environment_id = fixture.create("environment_a", 1_000).await; + fixture + .reconciler() + .run_work(i64::MAX, 2_000) + .await + .unwrap(); + let first = fixture.read(&environment_id).await; + let source = first.provisioned().unwrap(); + assert_eq!(first.state, EnvironmentState::Provisioning); + assert_eq!(source.attempts, 1); + assert_eq!(source.next_attempt_at_ms, Some(4_000)); + assert!( + first + .state_message + .as_deref() + .unwrap() + .contains("unreachable") + ); + fixture + .reconciler() + .run_work(i64::MAX, 4_000) + .await + .unwrap(); + let second = fixture.read(&environment_id).await; + assert_eq!(second.provisioned().unwrap().attempts, 2); + assert_eq!( + second.provisioned().unwrap().next_attempt_at_ms, + Some(8_000) + ); + // Past the deadline the row fails with the last error. + fixture + .reconciler() + .run_work(i64::MAX, 1_000 + PROVISIONING_DEADLINE_MS) + .await + .unwrap(); + let failed = fixture.read(&environment_id).await; + assert_eq!(failed.state, EnvironmentState::Failed); + assert!( + failed + .state_message + .as_deref() + .unwrap() + .starts_with("provisioning did not finish within 30 minutes: ") + ); + // The provider back: a new row opens. + controller.set_reachable(true); + let next = fixture.create("environment_b", 2_000).await; + fixture + .reconciler() + .run_work(i64::MAX, 3_000) + .await + .unwrap(); + assert_eq!(fixture.read(&next).await.state, EnvironmentState::Open); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_daemon_of_another_protocol_version_fails_naming_both() { + let controller = FakeController::start_with(FakeControllerOptions { + daemon_protocol_version: ENVD_PROTOCOL_VERSION + 1, + ..FakeControllerOptions::default() + }) + .await + .unwrap(); + let fixture = Fixture::new(&controller).await; + let environment_id = fixture.create("environment_a", 1_000).await; + fixture + .reconciler() + .run_work(i64::MAX, 2_000) + .await + .unwrap(); + let failed = fixture.read(&environment_id).await; + assert_eq!(failed.state, EnvironmentState::Failed); + assert_eq!( + failed.state_message.as_deref(), + Some( + format!( + "the environment's envd speaks protocol version {}; this runtime requires {ENVD_PROTOCOL_VERSION}", + ENVD_PROTOCOL_VERSION + 1 + ) + .as_str() + ) + ); + assert_eq!( + failed.daemon_build.map(|build| build.protocol_version), + Some(ENVD_PROTOCOL_VERSION + 1) + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn an_unknown_template_is_a_refusal_that_fails_the_row() { + let controller = FakeController::start().await.unwrap(); + let fixture = Fixture::new(&controller).await; + let environment_id = EnvironmentId::new("environment_a"); + fixture + .store + .create_environment(CreateEnvironment { + request_id: EnvironmentProvisionRequestId::new("request"), + environment_id: environment_id.clone(), + provider_id: fixture.provider_id.clone(), + template_id: EnvironmentTemplateId::new("missing-template"), + idle: None, + display_name: None, + metadata: BTreeMap::new(), + created_at_ms: 1_000, + }) + .await + .unwrap(); + fixture + .reconciler() + .run_work(i64::MAX, 2_000) + .await + .unwrap(); + assert_eq!( + fixture.read(&environment_id).await.state, + EnvironmentState::Failed + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn close_runs_until_the_target_is_gone_and_revokes_routes() { + let controller = FakeController::start().await.unwrap(); + let fixture = Fixture::new(&controller).await; + let environment_id = fixture.create("environment_a", 1_000).await; + fixture + .reconciler() + .run_work(i64::MAX, 2_000) + .await + .unwrap(); + let mut route = fixture + .signals + .watch(fixture.store.universe_id(), environment_id.as_str()); + fixture + .store + .begin_close_environment(BeginCloseEnvironment { + environment_id: environment_id.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: 3_000, + }) + .await + .unwrap(); + // While the provider is down the close backs off and stays closing. + controller.set_reachable(false); + fixture + .reconciler() + .run_work(i64::MAX, 3_000) + .await + .unwrap(); + let closing = fixture.read(&environment_id).await; + assert_eq!(closing.state, EnvironmentState::Closing); + assert_eq!(closing.provisioned().unwrap().attempts, 1); + controller.set_reachable(true); + fixture + .reconciler() + .run_work(i64::MAX, 5_000) + .await + .unwrap(); + let closed = fixture.read(&environment_id).await; + assert_eq!(closed.state, EnvironmentState::Closed); + assert_eq!(closed.close_reason, Some(EnvironmentCloseReason::Requested)); + assert_eq!(controller.target_count().await, 0); + assert!(route.has_changed().unwrap()); + route.mark_unchanged(); + } + + #[tokio::test(flavor = "current_thread")] + async fn the_display_sweep_records_power_and_ends_a_target_the_provider_removed() { + let controller = FakeController::start().await.unwrap(); + let fixture = Fixture::new(&controller).await; + let paused = fixture.create("environment_a", 1_000).await; + let ended = fixture.create("environment_b", 1_000).await; + fixture + .reconciler() + .run_work(i64::MAX, 2_000) + .await + .unwrap(); + assert!( + controller + .set_power_out_of_band(paused.as_str(), PowerState::Paused) + .await + ); + assert!(controller.end_target(ended.as_str()).await); + fixture + .reconciler() + .run_display_sweep(10_000) + .await + .unwrap(); + let observed = fixture.read(&paused).await; + assert_eq!(observed.state, EnvironmentState::Open); + assert_eq!( + observed.display_status(), + ::environments::EnvironmentDisplayStatus::Paused + ); + let gone = fixture.read(&ended).await; + assert_eq!(gone.state, EnvironmentState::Closed); + assert_eq!( + gone.close_reason, + Some(EnvironmentCloseReason::ProviderEnded) + ); + // The sweep never powers anything: the paused target stays paused. + assert_eq!(controller.target_count().await, 1); + fixture + .reconciler() + .run_display_sweep(20_000) + .await + .unwrap(); + assert_eq!( + fixture.read(&paused).await.display_status(), + ::environments::EnvironmentDisplayStatus::Paused + ); + } +} diff --git a/crates/temporal-runtime/src/environments/mod.rs b/crates/temporal-runtime/src/environments/mod.rs index 6164bdca0..848588cb7 100644 --- a/crates/temporal-runtime/src/environments/mod.rs +++ b/crates/temporal-runtime/src/environments/mod.rs @@ -1,17 +1,18 @@ //! Environment lifecycle management, connectivity, runtime resolution, and source discovery. -use crate::environments::gateway::EnvironmentGatewayClientConfig; use api::*; use lifecycle::parse_registry_environment_id; -use provider_controllers::{ProviderControllerConnector, finish_provider_controller}; -use providers::{map_environments_error, parse_environment_provider_id}; +use provider_controllers::ProviderControllerConnector; +use providers::map_environments_error; +use signals::EnvironmentSignals; use std::{collections::BTreeMap, sync::Arc}; use store_pg::PgStore; pub(crate) mod credentials; +pub mod fake_controller; pub(crate) mod lifecycle; -pub(crate) mod power; pub(crate) mod provider_controllers; pub(crate) mod providers; pub(crate) mod registration; +pub mod signals; pub mod gateway; pub(crate) mod prompts; @@ -23,8 +24,8 @@ pub(crate) mod sources; #[derive(Clone)] pub(crate) struct EnvironmentService { pub(crate) store: Arc, - pub(crate) environment_gateway: EnvironmentGatewayClientConfig, pub(crate) provider_controller_connector: Arc, + pub(crate) signals: Arc, } fn now_ms() -> Result { i64::try_from( @@ -37,6 +38,6 @@ fn now_ms() -> Result { } fn validate_caller_metadata(metadata: &BTreeMap) -> Result<(), AgentApiError> { - environment_protocol::registration::validate_registration_metadata(None, metadata) - .map_err(|message| AgentApiError::invalid_request(format!("invalid metadata: {message}"))) + environment_protocol::metadata::validate_caller_metadata(metadata) + .map_err(|error| AgentApiError::invalid_request(format!("invalid metadata: {error}"))) } diff --git a/crates/temporal-runtime/src/environments/power.rs b/crates/temporal-runtime/src/environments/power.rs deleted file mode 100644 index 1932f8816..000000000 --- a/crates/temporal-runtime/src/environments/power.rs +++ /dev/null @@ -1,275 +0,0 @@ -//! Idle power policy: the power reaper. -//! -//! One pass reads the daemon idle report of every `ready` provisioned -//! environment that carries an idle policy and, when a stage threshold has -//! been crossed and nothing is executing, records the matching power intent -//! (or a close). Activity is never persisted: the daemon owns the clock and -//! reports a monotonic idle duration; Lightspeed only decides. - -use super::providers::map_environments_error; -use super::*; - -use ::environments::{ - BeginCloseEnvironment, EnvironmentRecord, EnvironmentStore, IdleAction, PowerState, - SetEnvironmentPower, -}; -use environment_client::EnvironmentDataClient; -use environment_protocol::{ - data::{ - handshake::{InitializeParams, InitializedParams}, - idle::{IdleParams, IdleResponse}, - }, - shared::CURRENT_PROTOCOL_VERSION, -}; - -/// Outcome of one reaper pass for reporting/tests. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct PowerReaperStats { - pub candidates: usize, - pub unreachable: usize, - pub busy: usize, - pub powered_down: usize, - pub closed: usize, -} - -/// Decide the idle action for one environment from the daemon's report. -/// Pure so it can be tested without a daemon: nothing is due while work -/// runs, while a power change is already pending, or before the first -/// supported stage threshold. -pub(crate) fn decide_idle_action( - environment: &EnvironmentRecord, - report: &IdleResponse, -) -> Option { - if !report.is_quiescent() { - return None; - } - if environment.desired_power != PowerState::Running { - return None; - } - let policy = environment.idle_policy.as_ref()?; - policy.due_action(report.idle_for_ms, &environment.incarnation.power_states) -} - -impl EnvironmentService { - /// Public entry point for one reaper pass; used by acceptance tests that - /// drive the reaper deterministically instead of running the loop. - pub async fn reap_idle_environments_once(&self) -> Result { - self.reconcile_idle_power_once().await - } - - pub(crate) async fn reconcile_idle_power_once( - &self, - ) -> Result { - let candidates = EnvironmentStore::list_environments_with_idle_policy(self.store.as_ref()) - .await - .map_err(map_environments_error)?; - let mut stats = PowerReaperStats { - candidates: candidates.len(), - ..PowerReaperStats::default() - }; - let universe_id = self.store.config().universe_id; - for environment in candidates { - let report = match self.read_idle_report(universe_id, &environment).await { - Some(report) => report, - None => { - stats.unreachable += 1; - continue; - } - }; - let Some(action) = decide_idle_action(&environment, &report) else { - if !report.is_quiescent() { - stats.busy += 1; - } - continue; - }; - let now = now_ms()?; - match action.power_state() { - Some(desired_power) => { - EnvironmentStore::set_environment_power( - self.store.as_ref(), - SetEnvironmentPower { - environment_id: environment.environment_id.clone(), - desired_power, - updated_at_ms: now, - }, - ) - .await - .map_err(map_environments_error)?; - stats.powered_down += 1; - tracing::info!( - target: "temporal_runtime", - environment_id = %environment.environment_id, - idle_for_ms = report.idle_for_ms, - %desired_power, - "idle policy powered environment down" - ); - } - None => { - match EnvironmentStore::begin_close_environment( - self.store.as_ref(), - BeginCloseEnvironment { - environment_id: environment.environment_id.clone(), - updated_at_ms: now, - }, - ) - .await - { - Ok(_) => stats.closed += 1, - // Already closing/closed by someone else: converged. - Err(::environments::EnvironmentRegistryError::InvalidInput { .. }) => {} - Err(error) => return Err(map_environments_error(error)), - } - tracing::info!( - target: "temporal_runtime", - environment_id = %environment.environment_id, - idle_for_ms = report.idle_for_ms, - "idle policy closed environment" - ); - } - } - } - Ok(stats) - } - - /// Ask the environment's daemon for its idle report through the ordinary - /// on-demand route. `None` when the daemon is unreachable or predates - /// `env/idle`; the reaper then leaves the environment alone. - async fn read_idle_report( - &self, - universe_id: uuid::Uuid, - environment: &EnvironmentRecord, - ) -> Option { - let connection = self - .environment_gateway - .connection_for(universe_id, environment); - let mut client = EnvironmentDataClient::connect( - &connection.endpoint, - self.environment_gateway - .connect_options("lightspeed-power-reaper"), - ) - .await - .ok()?; - let response = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-power-reaper".to_owned(), - scope: connection.scope.clone(), - resume_connection_id: None, - }) - .await - .ok()?; - if response.protocol_version != CURRENT_PROTOCOL_VERSION { - return None; - } - client.initialized(&InitializedParams {}).await.ok()?; - client.idle(&IdleParams {}).await.ok() - } -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use ::environments::{ - EnvironmentIdlePolicy, EnvironmentIncarnationId, EnvironmentIncarnationRecord, - EnvironmentProviderBindingId, EnvironmentProviderId, EnvironmentProvisionRequestId, - EnvironmentSource, EnvironmentStatus, EnvironmentTemplateId, - }; - - use super::*; - - fn environment( - policy: EnvironmentIdlePolicy, - power_states: Vec, - ) -> EnvironmentRecord { - EnvironmentRecord { - environment_id: ::environments::EnvironmentId::new("environment-1"), - request_id: EnvironmentProvisionRequestId::new("request-1"), - source: EnvironmentSource::Provisioned { - provider_id: EnvironmentProviderId::new("incus"), - binding_id: EnvironmentProviderBindingId::new("primary"), - }, - display_name: None, - status: EnvironmentStatus::Ready, - desired_power: PowerState::Running, - idle_policy: Some(policy), - incarnation: EnvironmentIncarnationRecord { - incarnation_id: EnvironmentIncarnationId::new("incarnation-1"), - provision_request_id: Some(EnvironmentProvisionRequestId::new("request-1")), - provider_target_id: Some("target-1".into()), - template_id: Some(EnvironmentTemplateId::new("dev")), - adoption_source_target: None, - power_states, - created_at_ms: 1, - updated_at_ms: 1, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: BTreeMap::new(), - last_seen_at_ms: None, - created_at_ms: 1, - updated_at_ms: 1, - } - } - - fn report(idle_for_ms: u64, running_processes: u32, running_jobs: u32) -> IdleResponse { - IdleResponse { - idle_for_ms, - running_processes, - running_jobs, - leftover_process_groups: 0, - } - } - - #[test] - fn idle_decision_respects_work_pending_power_and_provider_support() { - let policy = EnvironmentIdlePolicy { - pause_after_ms: Some(1_000), - suspend_after_ms: Some(2_000), - stop_after_ms: Some(3_000), - close_after_ms: Some(4_000), - }; - let incus = vec![PowerState::Running, PowerState::Paused, PowerState::Stopped]; - let env = environment(policy.clone(), incus.clone()); - assert_eq!(decide_idle_action(&env, &report(500, 0, 0)), None); - assert_eq!( - decide_idle_action(&env, &report(1_500, 0, 0)), - Some(IdleAction::Pause) - ); - // Running work blocks every stage regardless of the clock. - assert_eq!(decide_idle_action(&env, &report(10_000, 1, 0)), None); - assert_eq!(decide_idle_action(&env, &report(10_000, 0, 2)), None); - // Suspend is due but unsupported by Incus: pause remains. - assert_eq!( - decide_idle_action(&env, &report(2_500, 0, 0)), - Some(IdleAction::Pause) - ); - assert_eq!( - decide_idle_action(&env, &report(3_500, 0, 0)), - Some(IdleAction::Stop) - ); - assert_eq!( - decide_idle_action(&env, &report(4_500, 0, 0)), - Some(IdleAction::Close) - ); - // A Firecracker-shaped provider gets suspend. - let firecracker = environment(policy.clone(), PowerState::ALL.to_vec()); - assert_eq!( - decide_idle_action(&firecracker, &report(2_500, 0, 0)), - Some(IdleAction::Suspend) - ); - // A pending power change is left to the reconciler. - let mut pending = environment(policy, incus); - pending.desired_power = PowerState::Paused; - assert_eq!(decide_idle_action(&pending, &report(10_000, 0, 0)), None); - // No power control at all: only close can ever apply. - let bare = environment( - EnvironmentIdlePolicy { - pause_after_ms: Some(1_000), - ..EnvironmentIdlePolicy::default() - }, - Vec::new(), - ); - assert_eq!(decide_idle_action(&bare, &report(10_000, 0, 0)), None); - } -} diff --git a/crates/temporal-runtime/src/environments/prompts.rs b/crates/temporal-runtime/src/environments/prompts.rs index f87168c1c..7dc62bd2a 100644 --- a/crates/temporal-runtime/src/environments/prompts.rs +++ b/crates/temporal-runtime/src/environments/prompts.rs @@ -20,17 +20,17 @@ pub(crate) async fn refresh( let connection = discovery.connection().await?; let query = tools::environment::sources::scan_query( source.roots.as_deref(), - &connection.cwd, - connection.initialized.home_directory.as_deref(), + connection.cwd.as_str(), + connection + .initialized + .home_directory + .as_ref() + .map(|home| home.as_str()), "prompts", )?; let scan = { let _timer = PhaseTimer::new("prompts_scan"); - connection - .client - .scan(&query) - .await - .map_err(|e| e.to_string())? + connection.scan(query).await? }; tools::prompts::environment::assemble(&scan) }; @@ -53,10 +53,23 @@ mod tests { ENVIRONMENT_PROMPT_CONTEXT_KEY, EnvironmentPromptReport, assemble, publication, }; + fn daemon(root: &std::path::Path, cwd: &std::path::Path) -> environment_daemon::DaemonRuntime { + environment_daemon::DaemonRuntime::new(environment_daemon::config::DaemonConfig { + listen: None, + cwd: cwd.to_owned(), + fs_root: root.to_owned(), + state_dir: root.parent().unwrap().join("state"), + read_only_fs: true, + registration: None, + scrubbed_env: Vec::new(), + }) + .unwrap() + } + #[tokio::test(flavor = "current_thread")] async fn local_environment_prompts_scan_in_order_and_replace_on_failure() { let directory = tempfile::tempdir().unwrap(); - let root = directory.path().canonicalize().unwrap(); + let root = directory.path().canonicalize().unwrap().join("root"); let cwd = root.join("project"); let home = root.join("home"); std::fs::create_dir_all(cwd.join(".agents/prompts")).unwrap(); @@ -85,8 +98,7 @@ mod tests { ) .unwrap(); std::fs::write(cwd.join(".agents/prompts/notes.json"), "Must not load").unwrap(); - let fs = - environment_daemon::filesystem::LocalFileSystem::new(root.clone(), cwd.clone(), true); + let runtime = daemon(&root, &cwd); let query = tools::environment::sources::scan_query( None, cwd.to_str().unwrap(), @@ -94,15 +106,23 @@ mod tests { "prompts", ) .unwrap(); - let scan = fs.scan(query).await.unwrap(); - let (text, paths) = assemble(&scan).unwrap(); + let scan = runtime + .filesystem() + .scan( + &environment_daemon::dispatch::RequestContext::detached(), + query, + ) + .await + .unwrap(); + let observation = assemble(&scan).unwrap(); assert_eq!( - text, + observation.text, "First instructions\n\nLast instructions\n\nBase instructions" ); - assert_eq!(paths.len(), 3); + assert_eq!(observation.paths.len(), 3); + assert!(observation.warnings.is_empty()); let blobs = InMemoryBlobStore::new(); - let entries = publication(&blobs, "machine", Ok((text, paths))) + let entries = publication(&blobs, "machine", Ok(observation.clone())) .await .unwrap(); let entry = &entries[&ContextEntryKey::new(ENVIRONMENT_PROMPT_CONTEXT_KEY)]; @@ -114,10 +134,36 @@ mod tests { ) .unwrap(); assert!(report.available); - let mut incomplete = scan.clone(); - incomplete.complete = false; - assert!(assemble(&incomplete).is_err()); - let failed = publication(&blobs, "other-machine", assemble(&incomplete)) + // A root that cannot be scanned is a warning naming it; the other + // roots' prompts still load. + let mut partial = scan.clone(); + let failed_root = environment_protocol::paths::EnvironmentPath::new( + home.join(".agents/prompts").to_str().unwrap(), + ) + .unwrap(); + partial + .roots + .push(environment_protocol::data::scan::ScanRootResult::Failed { + path: failed_root.clone(), + error: environment_protocol::error::ProtocolError::deadline_exceeded("slow"), + }); + let degraded = assemble(&partial).unwrap(); + assert_eq!(degraded.text, observation.text); + assert_eq!(degraded.warnings.len(), 1); + assert!(degraded.warnings[0].starts_with(failed_root.as_str())); + // Prompts need a full observation of every root. + let mut unchanged = scan.clone(); + unchanged.roots.push( + environment_protocol::data::scan::ScanRootResult::Unchanged { + path: failed_root, + fingerprint: environment_protocol::data::scan::ScanFingerprint::new(format!( + "scan:v1:{}", + "0".repeat(64) + )) + .unwrap(), + }, + ); + let failed = publication(&blobs, "other-machine", assemble(&unchanged)) .await .unwrap(); let failed = &failed[&ContextEntryKey::new(ENVIRONMENT_PROMPT_CONTEXT_KEY)]; @@ -130,10 +176,14 @@ mod tests { assert!(text.contains("unavailable")); assert!(!text.contains("Base instructions")); assert!( - publication(&blobs, "machine", Ok((String::new(), vec![]))) - .await - .unwrap() - .is_empty() + publication( + &blobs, + "machine", + Ok(tools::prompts::environment::PromptObservation::default()) + ) + .await + .unwrap() + .is_empty() ); let feature = EnvironmentsFeature { skills: Some(Default::default()), diff --git a/crates/temporal-runtime/src/environments/provider_controllers.rs b/crates/temporal-runtime/src/environments/provider_controllers.rs index dd91f8b39..fa6be3d32 100644 --- a/crates/temporal-runtime/src/environments/provider_controllers.rs +++ b/crates/temporal-runtime/src/environments/provider_controllers.rs @@ -1,701 +1,533 @@ +//! Provider controllers: dialing a provider's controller, with its bearer +//! credential when one is stored, and classifying what goes wrong. +//! +//! Nothing a controller advertises is stored. Every connection runs the +//! controller handshake through [`ControllerClient::connect_initialized`], +//! and its live `capabilities` gate the optional calls on that connection. +//! The data route base is configuration: the provider record's configured +//! base, or the one derived from its controller endpoint +//! ([`EnvironmentProviderRecord::effective_data_route_base`]). + use super::*; -use ::environments::EnvironmentConnectionSpec; -use async_trait::async_trait; -use environment_client::{ - EnvironmentClientError, EnvironmentProviderClient, WebSocketConnectOptions, +use ::environments::{ + EnvironmentEndpoint, EnvironmentProviderId, EnvironmentProviderRecord, + EnvironmentProviderStore, EnvironmentRegistryError, }; +use async_trait::async_trait; +use auth::SecretValue; +use environment_client::{ClientError, ConnectOptions, ControllerClient}; use environment_protocol::{ - control::{ - handshake::{ControllerInitializeParams, ControllerInitializeResponse}, - ingress::{ - EnsureIngressParams, IngressResponse, ProviderIngressStatus, RemoveIngressParams, - }, - targets::{ - AdoptTargetParams, AdoptTargetResponse, CloseTargetParams, CloseTargetResponse, - CreateTargetParams, CreateTargetResponse, EnvironmentTemplate, ListTemplatesParams, - ListTemplatesResponse, PowerState, ProviderTargetStatus, ProviderTargetSummary, - SetTargetPowerParams, SetTargetPowerResponse, - }, - }, - shared::{ - CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentScope, EnvironmentTransport, - ImplementationInfo, ProviderTargetId, - }, + control::handshake::ControllerCapabilities, error::ProtocolErrorKind, routes::DataRouteBase, }; - -use std::{ - collections::{BTreeMap, hash_map::DefaultHasher}, - hash::{Hash, Hasher}, - sync::Arc, -}; - -#[async_trait] -pub(crate) trait ProviderController: Send { - async fn close(&mut self) -> Result<(), AgentApiError> { - Ok(()) +use std::time::Duration; +use thiserror::Error; + +/// Deadline of every reconciler controller call. Controllers answer target +/// methods promptly and continue slow work in the background, so this +/// bounds a hung controller, not a slow create; it is short because a hung +/// call holds one of the provider's in-flight rows until it expires. +pub(crate) const CONTROLLER_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// Deadline of the synchronous power and ingress calls an API request +/// waits on. +pub(crate) const PASS_THROUGH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Options for one controller or provider data-route connection: the +/// provider credential as the bearer token when one is stored (no +/// `Authorization` header otherwise), the default connect and `initialize` +/// bounds, and [`CONTROLLER_REQUEST_TIMEOUT`] on every call. +pub(crate) fn provider_connect_options(credential: Option<&SecretValue>) -> ConnectOptions { + let options = ConnectOptions::new(super::gateway::runtime_client_info()) + .with_request_timeout(CONTROLLER_REQUEST_TIMEOUT); + match credential { + Some(credential) => options.with_bearer_token(credential.expose()), + None => options, } - - async fn initialize( - &mut self, - params: &ControllerInitializeParams, - ) -> Result; - - async fn create_target( - &mut self, - params: &CreateTargetParams, - ) -> Result; - - async fn adopt_target( - &mut self, - params: &AdoptTargetParams, - ) -> Result; - - async fn list_templates( - &mut self, - params: &ListTemplatesParams, - ) -> Result; - - async fn close_target( - &mut self, - params: &CloseTargetParams, - ) -> Result; - async fn set_target_power( - &mut self, - params: &SetTargetPowerParams, - ) -> Result; - async fn ensure_ingress( - &mut self, - params: &EnsureIngressParams, - ) -> Result; - async fn remove_ingress( - &mut self, - params: &RemoveIngressParams, - ) -> Result; } +/// Dials a provider controller and runs its handshake. Tests substitute an +/// in-process client. #[async_trait] pub(crate) trait ProviderControllerConnector: Send + Sync { async fn connect( &self, - connection: &EnvironmentConnectionSpec, - ) -> Result, AgentApiError>; + endpoint: &EnvironmentEndpoint, + options: ConnectOptions, + ) -> Result; } -#[derive(Default)] -pub(crate) struct WebSocketProviderControllerConnector { - fake_backend: Arc>, -} +/// The production connector: `ws://`, `wss://` and `unix://` controller +/// URLs through [`ControllerClient::connect_initialized`]. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct WebSocketProviderControllerConnector; #[async_trait] impl ProviderControllerConnector for WebSocketProviderControllerConnector { async fn connect( &self, - connection: &EnvironmentConnectionSpec, - ) -> Result, AgentApiError> { - let mut controller: Box = match &connection.transport { - EnvironmentTransport::WebSocket => { - let client = EnvironmentProviderClient::connect( - &connection.endpoint, - WebSocketConnectOptions { - bearer_token: None, - user_agent: Some("lightspeed-temporal-runtime".to_owned()), - headers: Vec::new(), - }, - ) - .await - .map_err(map_environment_client_error)?; - Box::new(client) - } - EnvironmentTransport::Http => return Err(unsupported_transport("http")), - EnvironmentTransport::Stdio => return Err(unsupported_transport("stdio")), - EnvironmentTransport::Ssh => return Err(unsupported_transport("ssh")), - EnvironmentTransport::Provider { provider_type } => { - if provider_type == "fake" { - Box::new(FakeProviderController { - backend: self.fake_backend.clone(), - }) - } else { - return Err(unsupported_transport(format!("provider:{provider_type}"))); - } - } - }; - let initialized = controller - .initialize(&ControllerInitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-temporal-runtime".to_owned(), - }) - .await; - let initialized = match initialized { - Ok(initialized) => initialized, - Err(error) => { - let _ = controller.close().await; - return Err(error); - } - }; - if initialized.protocol_version != CURRENT_PROTOCOL_VERSION { - let _ = controller.close().await; - return Err(AgentApiError::rejected(format!( - "environment provider protocol version {} is incompatible with {}", - initialized.protocol_version, CURRENT_PROTOCOL_VERSION, - ))); - } - Ok(controller) + endpoint: &EnvironmentEndpoint, + options: ConnectOptions, + ) -> Result { + ControllerClient::connect_initialized(endpoint.as_str(), options) + .await + .map(|(client, _)| client) } } -/// Power states the in-process fake provider advertises. Suspended is -/// deliberately included so the Firecracker-shaped path has coverage. -const FAKE_POWER_STATES: [PowerState; 4] = [ - PowerState::Running, - PowerState::Paused, - PowerState::Suspended, - PowerState::Stopped, -]; - -#[derive(Default)] -struct FakeBackend { - targets: BTreeMap, - requests: BTreeMap<(String, String, String), String>, +/// An optional controller capability the runtime gates calls on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ControllerFeature { + Power, + Ingress, } -#[derive(Clone)] -struct FakeTarget { - universe_id: String, - binding_id: String, - environment_id: String, - incarnation_id: String, - response: CreateTargetResponse, -} - -struct FakeProviderController { - backend: Arc>, -} - -#[async_trait] -impl ProviderController for FakeProviderController { - async fn initialize( - &mut self, - _params: &ControllerInitializeParams, - ) -> Result { - Ok(ControllerInitializeResponse { - protocol_version: CURRENT_PROTOCOL_VERSION, - capabilities: environment_protocol::control::handshake::ControllerCapabilities { - list_templates: true, - list_targets: true, - create_target: true, - adopt_target: true, - get_target: true, - close_target: true, - set_target_power: true, - ingress: true, - }, - implementation: ImplementationInfo { - name: "lightspeed-fake-environment-provider".to_owned(), - version: Some(env!("CARGO_PKG_VERSION").to_owned()), - ..Default::default() - }, - }) - } - - async fn list_templates( - &mut self, - _params: &ListTemplatesParams, - ) -> Result { - Ok(ListTemplatesResponse { - templates: vec![EnvironmentTemplate { - template_id: "rust-v1".to_owned(), - display_name: "Rust development".to_owned(), - description: Some("Deterministic fake provider template".to_owned()), - public_ingress: false, - deprecated: false, - metadata: BTreeMap::new(), - }], - }) +impl ControllerFeature { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Power => "power control", + Self::Ingress => "public ingress", + } } - async fn create_target( - &mut self, - params: &CreateTargetParams, - ) -> Result { - if params.template_id != "rust-v1" && !params.template_id.starts_with("adopted:") { - return Err(AgentApiError::rejected( - "fake provider policy rejected the template", - )); - } - let key = ( - params.binding.universe_id.clone(), - params.binding.binding_id.clone(), - params.request_id.clone(), - ); - let mut backend = self.backend.lock().await; - if let Some(target_id) = backend.requests.get(&key) { - return Ok(backend - .targets - .get(target_id) - .expect("fake request index target") - .response - .clone()); - } - if backend.targets.values().any(|target| { - target.universe_id == params.binding.universe_id - && target.binding_id == params.binding.binding_id - && target.response.target.status != ProviderTargetStatus::Closed - }) { - return Err(AgentApiError::rejected( - "fake provider policy allows one active target per binding", - )); + pub(crate) fn in_capabilities(self, capabilities: &ControllerCapabilities) -> bool { + match self { + Self::Power => capabilities.power, + Self::Ingress => capabilities.ingress, } - let mut identity = DefaultHasher::new(); - params.binding.universe_id.hash(&mut identity); - params.binding.binding_id.hash(&mut identity); - params.incarnation_id.hash(&mut identity); - let target_id = format!("fake-{:016x}", identity.finish()); - let provider_target_id = ProviderTargetId::new(target_id.clone()); - let capabilities = EnvironmentCapabilities::filesystem(true, true).with_process(); - let response = CreateTargetResponse { - target: ProviderTargetSummary { - target_id: provider_target_id.clone(), - display_name: Some(params.environment_id.clone()), - status: ProviderTargetStatus::Ready, - scope: EnvironmentScope::Default, - capabilities: capabilities.clone(), - default_cwd: None, - power_states: FAKE_POWER_STATES.to_vec(), - metadata: BTreeMap::from([( - "imageFingerprint".to_owned(), - format!("fake:{}", params.template_id), - )]), - }, - }; - backend.requests.insert(key, target_id.clone()); - backend.targets.insert( - target_id, - FakeTarget { - universe_id: params.binding.universe_id.clone(), - binding_id: params.binding.binding_id.clone(), - environment_id: params.environment_id.clone(), - incarnation_id: params.incarnation_id.clone(), - response: response.clone(), - }, - ); - Ok(response) } +} - async fn adopt_target( - &mut self, - params: &AdoptTargetParams, - ) -> Result { - let create = CreateTargetParams { - request_id: params.request_id.clone(), - environment_id: params.environment_id.clone(), - incarnation_id: params.incarnation_id.clone(), - binding: params.binding.clone(), - template_id: format!("adopted:{}", params.source_target), - }; - let response = self.create_target(&create).await?; - Ok(AdoptTargetResponse { - target: response.target, - }) - } +/// One initialized controller connection, what its handshake advertised, +/// and what its data routes need. +#[derive(Clone)] +pub(crate) struct ProviderController { + pub(crate) provider_id: EnvironmentProviderId, + pub(crate) client: ControllerClient, + /// The live `initialize` capabilities of this connection. + pub(crate) capabilities: ControllerCapabilities, + /// The effective data route base. + pub(crate) data_route_base: DataRouteBase, + /// The provider credential, presented on its data routes too. + pub(crate) credential: Option, +} - async fn close_target( - &mut self, - params: &CloseTargetParams, - ) -> Result { - let mut backend = self.backend.lock().await; - let target = backend - .targets - .get_mut(params.target_id.as_str()) - .ok_or_else(|| { - AgentApiError::not_found(format!("fake target not found: {}", params.target_id)) - })?; - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - { - return Err(AgentApiError::rejected( - "fake target ownership metadata does not match close request", - )); +impl ProviderController { + /// Refuse a call that needs an optional capability the controller did + /// not advertise, before anything is sent. + pub(crate) fn require(&self, feature: ControllerFeature) -> Result<(), ControllerError> { + if feature.in_capabilities(&self.capabilities) { + Ok(()) + } else { + Err(ControllerError::FeatureUnavailable { + provider_id: self.provider_id.to_string(), + feature, + }) } - target.response.target.status = ProviderTargetStatus::Closed; - Ok(CloseTargetResponse { - target_id: params.target_id.clone(), - status: ProviderTargetStatus::Closed, - }) } - /// The fake provider converges instantly: the requested steady state is - /// the observed state on return. - async fn set_target_power( - &mut self, - params: &SetTargetPowerParams, - ) -> Result { - if !FAKE_POWER_STATES.contains(¶ms.power) { - return Err(AgentApiError::rejected(format!( - "fake provider does not support power state {}", - params.power - ))); - } - let mut backend = self.backend.lock().await; - let target = backend - .targets - .get_mut(params.target_id.as_str()) - .ok_or_else(|| { - AgentApiError::not_found(format!("fake target not found: {}", params.target_id)) - })?; - if target.universe_id != params.binding.universe_id - || target.binding_id != params.binding.binding_id - || target.environment_id != params.environment_id - || target.incarnation_id != params.incarnation_id - { - return Err(AgentApiError::rejected( - "fake target ownership metadata does not match power request", - )); - } - if target.response.target.status == ProviderTargetStatus::Closed { - return Err(AgentApiError::rejected("fake target is closed")); + /// Wrap a failed call of this controller. + pub(crate) fn call_error(&self, source: ClientError) -> ControllerError { + ControllerError::Call { + provider_id: self.provider_id.to_string(), + source, } - target.response.target.status = match params.power { - PowerState::Running => ProviderTargetStatus::Ready, - PowerState::Paused => ProviderTargetStatus::Paused, - PowerState::Suspended => ProviderTargetStatus::Suspended, - PowerState::Stopped => ProviderTargetStatus::Stopped, - }; - Ok(SetTargetPowerResponse { - target: target.response.target.clone(), - }) } - async fn ensure_ingress( - &mut self, - _params: &EnsureIngressParams, - ) -> Result { - Ok(IngressResponse { - status: ProviderIngressStatus::Ready, - public_endpoint: Some("https://fake.env.test".to_owned()), - }) + pub(crate) async fn close(&self) { + let _ = self.client.close().await; } +} - async fn remove_ingress( - &mut self, - _params: &RemoveIngressParams, - ) -> Result { - Ok(IngressResponse { - status: ProviderIngressStatus::Disabled, - public_endpoint: None, - }) - } +/// Connect to a provider's controller with its stored credential, if any. +pub(crate) async fn connect_provider_controller( + store: &dyn EnvironmentProviderStore, + connector: &dyn ProviderControllerConnector, + provider_id: &EnvironmentProviderId, +) -> Result { + let provider: EnvironmentProviderRecord = store.read_provider(provider_id).await?; + let data_route_base = provider.effective_data_route_base()?; + let credential = store.read_provider_credential(provider_id).await?; + let client = connector + .connect( + &provider.controller_endpoint, + provider_connect_options(credential.as_ref()), + ) + .await + .map_err(|source| ControllerError::Connect { + provider_id: provider_id.to_string(), + source, + })?; + Ok(ProviderController { + provider_id: provider_id.clone(), + capabilities: client.initialize_response().capabilities.clone(), + client, + data_route_base, + credential, + }) } -#[async_trait] -impl ProviderController for EnvironmentProviderClient -where - T: environment_client::JsonRpcTransport + Send, -{ - async fn close(&mut self) -> Result<(), AgentApiError> { - EnvironmentProviderClient::close(self) - .await - .map_err(map_environment_client_error) +/// Check a controller endpoint at provider registration, with the +/// credential the provider will hold. A controller that answers +/// definitively wrong (an HTTP refusal such as 404 for a URL that names no +/// controller or 401 for a wrong or missing credential, or another plane) +/// is refused, so a mis-registered provider never provisions anything. One +/// that cannot be reached right now is accepted. +pub(crate) async fn probe_controller( + connector: &dyn ProviderControllerConnector, + endpoint: &EnvironmentEndpoint, + credential: Option<&SecretValue>, +) -> Result<(), AgentApiError> { + match connector + .connect(endpoint, provider_connect_options(credential)) + .await + { + Ok(client) => { + let _ = client.close().await; + Ok(()) + } + Err(error) if refuses_registration(&error) => Err(AgentApiError::invalid_request(format!( + "the provider controller at {endpoint} refused the handshake: {error}" + ))), + Err(error) => { + tracing::warn!( + target: "temporal_runtime", + %endpoint, + %error, + "provider controller is not reachable at registration; it is registered anyway" + ); + Ok(()) + } } +} - async fn initialize( - &mut self, - params: &ControllerInitializeParams, - ) -> Result { - EnvironmentProviderClient::initialize(self, params) - .await - .map_err(map_environment_client_error) +/// Handshake failures that prove the registration wrong rather than the +/// controller temporarily unavailable. +fn refuses_registration(error: &ClientError) -> bool { + match error { + ClientError::UpgradeRejected { status, .. } => { + (400..500).contains(status) && !matches!(status, 408 | 429) + } + ClientError::WrongPlane { .. } | ClientError::Configuration { .. } => true, + _ => false, } +} - async fn create_target( - &mut self, - params: &CreateTargetParams, - ) -> Result { - EnvironmentProviderClient::create_target(self, params) - .await - .map_err(map_environment_client_error) - } +/// How the reconciler treats a failed controller interaction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ControllerDisposition { + /// May succeed later unchanged: back off and retry, until the + /// provisioning deadline. + Transient, + /// The provider refused the request; repeating it cannot help. + Refused, + /// The provider holds no such target. + Gone, +} - async fn adopt_target( - &mut self, - params: &AdoptTargetParams, - ) -> Result { - EnvironmentProviderClient::adopt_target(self, params) - .await - .map_err(map_environment_client_error) - } +#[derive(Debug, Error)] +pub(crate) enum ControllerError { + /// Dialing the controller or its `initialize` failed, a version or + /// plane mismatch included. Always transient: a provider restart or a + /// rollout that upgrades the runtime first must not fail environments. + #[error("provider controller {provider_id} is unreachable: {source}")] + Connect { + provider_id: String, + #[source] + source: ClientError, + }, - async fn list_templates( - &mut self, - params: &ListTemplatesParams, - ) -> Result { - EnvironmentProviderClient::list_templates(self, params) - .await - .map_err(map_environment_client_error) - } + /// A call on an initialized connection failed. + #[error("provider controller {provider_id}: {source}")] + Call { + provider_id: String, + #[source] + source: ClientError, + }, - async fn close_target( - &mut self, - params: &CloseTargetParams, - ) -> Result { - EnvironmentProviderClient::close_target(self, params) - .await - .map_err(map_environment_client_error) - } + /// The call needs an optional capability the controller did not + /// advertise; nothing was sent. + #[error("provider {provider_id} does not offer {}", feature.as_str())] + FeatureUnavailable { + provider_id: String, + feature: ControllerFeature, + }, - async fn set_target_power( - &mut self, - params: &SetTargetPowerParams, - ) -> Result { - EnvironmentProviderClient::set_target_power(self, params) - .await - .map_err(map_environment_client_error) - } + #[error(transparent)] + Registry(#[from] EnvironmentRegistryError), +} - async fn ensure_ingress( - &mut self, - params: &EnsureIngressParams, - ) -> Result { - EnvironmentProviderClient::ensure_ingress(self, params) - .await - .map_err(map_environment_client_error) +impl ControllerError { + /// Refusals are the controller's typed answers that no retry changes. + /// Connection failures, deadlines, undecodable answers and retryable or + /// `internal` answers are transient; `notFound` means the target is + /// gone. Registry failures are the runtime's own and transient. + pub(crate) fn disposition(&self) -> ControllerDisposition { + match self { + Self::Call { source, .. } => call_disposition(source), + Self::FeatureUnavailable { .. } => ControllerDisposition::Refused, + Self::Connect { .. } | Self::Registry(_) => ControllerDisposition::Transient, + } } - async fn remove_ingress( - &mut self, - params: &RemoveIngressParams, - ) -> Result { - EnvironmentProviderClient::remove_ingress(self, params) - .await - .map_err(map_environment_client_error) + /// The API error for a caller waiting on this controller interaction. + pub(crate) fn into_api_error(self) -> AgentApiError { + match self { + Self::Registry(error) => super::providers::map_environments_error(error), + other => match other.disposition() { + ControllerDisposition::Refused => AgentApiError::rejected(other.to_string()), + ControllerDisposition::Gone => AgentApiError::not_found(other.to_string()), + ControllerDisposition::Transient => AgentApiError::internal(other.to_string()), + }, + } } } -/// Finish one scoped provider-controller operation and close its transport on -/// both success and failure. A close error does not replace the operation's -/// result: controller calls are already complete when this runs. -pub(crate) async fn finish_provider_controller( - mut controller: Box, - result: Result, -) -> Result { - let _ = controller.close().await; - result -} - -pub(crate) fn map_environment_client_error(error: EnvironmentClientError) -> AgentApiError { +/// The disposition of a failed call on an initialized connection. +pub(crate) fn call_disposition(error: &ClientError) -> ControllerDisposition { match error { - EnvironmentClientError::Protocol(error) => { - AgentApiError::rejected(format!("provider controller error: {}", error.message)) - } - EnvironmentClientError::TransportClosed => { - AgentApiError::rejected("provider controller disconnected") - } - EnvironmentClientError::Serialize(error) => { - AgentApiError::rejected(format!("provider controller call failed: {error}")) - } - EnvironmentClientError::Transport(message) - | EnvironmentClientError::InvalidMessage(message) => { - AgentApiError::rejected(format!("provider controller call failed: {message}")) - } + ClientError::Protocol(error) => match error.kind() { + ProtocolErrorKind::NotFound => ControllerDisposition::Gone, + ProtocolErrorKind::Internal => ControllerDisposition::Transient, + kind if kind.is_retryable() => ControllerDisposition::Transient, + _ => ControllerDisposition::Refused, + }, + _ => ControllerDisposition::Transient, } } -fn unsupported_transport(transport: impl std::fmt::Display) -> AgentApiError { - AgentApiError::invalid_request(format!( - "provider controller transport is not supported by this gateway: {transport}" - )) +impl EnvironmentService { + /// A controller connection for one API request; the caller closes it. + pub(crate) async fn connect_provider_controller( + &self, + provider_id: &EnvironmentProviderId, + ) -> Result { + connect_provider_controller( + self.store.as_ref(), + self.provider_controller_connector.as_ref(), + provider_id, + ) + .await + } } #[cfg(test)] mod tests { use super::*; - use ::environments::EnvironmentConnectionSpec; - use environment_client::{EnvironmentClientResult, JsonRpcTransport}; - use environment_protocol::control::targets::ProviderBindingContext; - use serde_json::Value; - use std::sync::atomic::{AtomicBool, Ordering}; - - struct CloseTrackingTransport { - closed: Arc, - } - - #[async_trait] - impl JsonRpcTransport for CloseTrackingTransport { - async fn send(&mut self, _message: Value) -> EnvironmentClientResult<()> { - Ok(()) + use environment_protocol::{ + error::{PreconditionReason, ProtocolError}, + handshake::Plane, + }; + + fn call(source: ClientError) -> ControllerError { + ControllerError::Call { + provider_id: "provider".into(), + source, } + } - async fn recv(&mut self) -> EnvironmentClientResult> { - Ok(None) + #[test] + fn connection_failures_and_contract_breaches_are_transient() { + for source in [ + ClientError::Closed { + code: None, + reason: None, + }, + ClientError::Transport("reset".into()), + ClientError::DeadlineExceeded { + elapsed_ms: 120_000, + }, + ClientError::Decode("missing field".into()), + ClientError::InvalidMessage("binary frame".into()), + ClientError::Protocol(ProtocolError::internal("bug")), + ClientError::Protocol(ProtocolError::unavailable("incus api down")), + ] { + assert_eq!( + call(source.clone()).disposition(), + ControllerDisposition::Transient, + "{source}" + ); } - - async fn close(&mut self) -> EnvironmentClientResult<()> { - self.closed.store(true, Ordering::SeqCst); - Ok(()) + // Every handshake failure is transient, a version mismatch included. + for source in [ + ClientError::VersionMismatch { + plane: Plane::Controller, + local: 4, + remote: 5, + }, + ClientError::UpgradeRejected { + status: 404, + refusal: None, + retry_after_ms: None, + }, + ClientError::ConnectTimeout { + endpoint: "ws://provider".into(), + }, + ] { + let error = ControllerError::Connect { + provider_id: "provider".into(), + source, + }; + assert_eq!(error.disposition(), ControllerDisposition::Transient); } } - fn connection() -> EnvironmentConnectionSpec { - EnvironmentConnectionSpec::new( - "in-process", - EnvironmentTransport::Provider { - provider_type: "fake".to_owned(), - }, - ) + #[test] + fn typed_refusals_and_missing_targets_are_final() { + let refused = call(ClientError::Protocol(ProtocolError::failed_precondition( + PreconditionReason::TemplateDeprecated, + "deprecated", + ))); + assert_eq!(refused.disposition(), ControllerDisposition::Refused); + let gone = call(ClientError::Protocol(ProtocolError::not_found("no target"))); + assert_eq!(gone.disposition(), ControllerDisposition::Gone); + let unadvertised = ControllerError::FeatureUnavailable { + provider_id: "provider".into(), + feature: ControllerFeature::Power, + }; + assert_eq!(unadvertised.disposition(), ControllerDisposition::Refused); + assert_eq!( + unadvertised.into_api_error().kind, + api::AgentApiErrorKind::Rejected + ); } - fn create( - universe: &str, - binding: &str, - request: &str, - environment: &str, - ) -> CreateTargetParams { - CreateTargetParams { - request_id: request.to_owned(), - environment_id: environment.to_owned(), - incarnation_id: format!("incarnation-{environment}"), - binding: ProviderBindingContext { - universe_id: universe.to_owned(), - binding_id: binding.to_owned(), + #[test] + fn registration_refuses_only_definitive_handshake_failures() { + let rejected = |status| ClientError::UpgradeRejected { + status, + refusal: None, + retry_after_ms: None, + }; + for refused in [ + rejected(404), + rejected(401), + ClientError::WrongPlane { + expected: Plane::Controller, + actual: Plane::Data, }, - template_id: "rust-v1".to_owned(), + ] { + assert!(refuses_registration(&refused), "{refused}"); + } + for accepted in [ + rejected(503), + rejected(429), + ClientError::Connect { + endpoint: "ws://provider".into(), + message: "refused".into(), + }, + ClientError::VersionMismatch { + plane: Plane::Controller, + local: 4, + remote: 5, + }, + ] { + assert!(!refuses_registration(&accepted), "{accepted}"); } } - #[tokio::test(flavor = "current_thread")] - async fn finishing_a_provider_operation_closes_its_transport() { - let closed = Arc::new(AtomicBool::new(false)); - let controller: Box = - Box::new(EnvironmentProviderClient::new(CloseTrackingTransport { - closed: closed.clone(), - })); - - let value = finish_provider_controller(controller, Ok::<_, AgentApiError>(42)) - .await - .expect("operation result"); - - assert_eq!(value, 42); - assert!(closed.load(Ordering::SeqCst)); - } - - #[tokio::test(flavor = "current_thread")] - async fn fake_provider_adopts_retries_and_isolates_bindings() { - let connector = WebSocketProviderControllerConnector::default(); - let mut first = connector.connect(&connection()).await.expect("connect"); - let created = first - .create_target(&create( - "universe-a", - "binding-a", - "request-1", - "environment-1", - )) - .await - .expect("create"); - drop(first); - - let mut after_restart = connector.connect(&connection()).await.expect("reconnect"); - let adopted = after_restart - .create_target(&create( - "universe-a", - "binding-a", - "request-1", - "different-environment", - )) - .await - .expect("adopt"); - assert_eq!(adopted.target.target_id, created.target.target_id); - - let isolated = after_restart - .create_target(&create( - "universe-b", - "binding-b", - "request-1", - "environment-2", - )) - .await - .expect("isolated create"); - assert_ne!(isolated.target.target_id, created.target.target_id); - - let imported = after_restart - .adopt_target(&AdoptTargetParams { - request_id: "adopt-request-1".to_owned(), - environment_id: "environment-imported".to_owned(), - incarnation_id: "incarnation-imported".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-c".to_owned(), - binding_id: "binding-c".to_owned(), - }, - source_target: "legacy/hand-built-vm".to_owned(), - }) - .await - .expect("explicit adoption"); + #[test] + fn the_bearer_is_sent_only_with_a_credential() { + assert!(provider_connect_options(None).bearer_token.is_none()); + let credential = SecretValue::new("provider-credential"); assert_eq!( - imported.target.metadata.get("imageFingerprint"), - Some(&"fake:adopted:legacy/hand-built-vm".to_owned()) + provider_connect_options(Some(&credential)) + .bearer_token + .as_ref() + .map(|token| token.expose()), + Some("provider-credential") ); + } - let wrong_owner = after_restart - .close_target(&CloseTargetParams { - request_id: "close-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-environment-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-b".to_owned(), - binding_id: "binding-b".to_owned(), + async fn provider_store( + endpoint: &str, + credential: Option<&str>, + ) -> ::environments::InMemoryEnvironmentRegistryStore { + let store = ::environments::InMemoryEnvironmentRegistryStore::new(); + store + .put_provider(::environments::PutEnvironmentProvider { + provider_id: EnvironmentProviderId::new("provider"), + display_name: None, + controller_endpoint: EnvironmentEndpoint::parse(endpoint).unwrap(), + data_route_base: None, + credential: match credential { + Some(credential) => { + ::environments::ProviderCredentialChange::Set(SecretValue::new(credential)) + } + None => ::environments::ProviderCredentialChange::Keep, }, - target_id: created.target.target_id.clone(), - force: false, + updated_at_ms: 1, }) .await - .expect_err("ownership mismatch"); - assert_eq!(wrong_owner.kind, AgentApiErrorKind::Rejected); - - let power = |power: PowerState| SetTargetPowerParams { - request_id: format!("power-{power}"), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-environment-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-a".to_owned(), - binding_id: "binding-a".to_owned(), + .unwrap(); + store + } + + #[tokio::test(flavor = "current_thread")] + async fn a_connection_carries_live_capabilities_and_the_derived_data_route() { + let controller = crate::environments::fake_controller::FakeController::start_with( + crate::environments::fake_controller::FakeControllerOptions { + power: false, + ..Default::default() }, - target_id: created.target.target_id.clone(), - power, - }; - assert_eq!(created.target.power_states, FAKE_POWER_STATES.to_vec()); - let paused = after_restart - .set_target_power(&power(PowerState::Paused)) - .await - .expect("pause"); - assert_eq!(paused.target.status, ProviderTargetStatus::Paused); - let resumed = after_restart - .set_target_power(&power(PowerState::Running)) - .await - .expect("resume"); - assert_eq!(resumed.target.status, ProviderTargetStatus::Ready); - assert_eq!(resumed.target.target_id, created.target.target_id); - - let closed = after_restart - .close_target(&CloseTargetParams { - request_id: "close-1".to_owned(), - environment_id: "environment-1".to_owned(), - incarnation_id: "incarnation-environment-1".to_owned(), - binding: ProviderBindingContext { - universe_id: "universe-a".to_owned(), - binding_id: "binding-a".to_owned(), - }, - target_id: created.target.target_id.clone(), - force: false, - }) - .await - .expect("close"); - assert_eq!(closed.status, ProviderTargetStatus::Closed); + ) + .await + .unwrap(); + let store = provider_store(controller.controller_url(), controller.credential()).await; + let connected = connect_provider_controller( + &store, + &WebSocketProviderControllerConnector, + &EnvironmentProviderId::new("provider"), + ) + .await + .expect("connect"); + assert!(!connected.capabilities.power && connected.capabilities.ingress); + assert!(matches!( + connected.require(ControllerFeature::Power), + Err(ControllerError::FeatureUnavailable { .. }) + )); assert!( - after_restart - .set_target_power(&power(PowerState::Running)) - .await - .is_err() + connected + .data_route_base + .as_str() + .ends_with(environment_protocol::routes::DEFAULT_DATA_ROUTE_PATH) ); + connected.close().await; + } + + #[tokio::test(flavor = "current_thread")] + async fn a_provider_without_a_credential_connects_without_a_bearer() { + let controller = crate::environments::fake_controller::FakeController::start_with( + crate::environments::fake_controller::FakeControllerOptions { + credential: false, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!(controller.credential().is_none()); + let store = provider_store(controller.controller_url(), None).await; + let connected = connect_provider_controller( + &store, + &WebSocketProviderControllerConnector, + &EnvironmentProviderId::new("provider"), + ) + .await + .expect("connect without a credential"); + assert!(connected.credential.is_none()); + connected.close().await; + // A controller that requires a credential refuses a provider that + // has none, and registration says so. + let guarded = crate::environments::fake_controller::FakeController::start() + .await + .unwrap(); + let refused = probe_controller( + &WebSocketProviderControllerConnector, + &EnvironmentEndpoint::parse(guarded.controller_url()).unwrap(), + None, + ) + .await + .expect_err("a missing credential is a definitive refusal"); + assert_eq!(refused.kind, api::AgentApiErrorKind::InvalidRequest); } } diff --git a/crates/temporal-runtime/src/environments/providers.rs b/crates/temporal-runtime/src/environments/providers.rs index 21204d8c7..1a868fc7c 100644 --- a/crates/temporal-runtime/src/environments/providers.rs +++ b/crates/temporal-runtime/src/environments/providers.rs @@ -1,12 +1,15 @@ +use super::provider_controllers::ControllerError; use super::*; use ::environments::{ - EnvironmentProviderBindingId, EnvironmentProviderBindingRecord, - EnvironmentProviderBindingStatus, EnvironmentProviderId, EnvironmentProviderRecord, - EnvironmentRecord, EnvironmentRegistryError, EnvironmentSource, EnvironmentStatus, + DaemonBuild, EnvironmentCloseReason, EnvironmentDisplayStatus, + EnvironmentProviderBindingRecord, EnvironmentProviderId, EnvironmentProviderRecord, + EnvironmentRecord, EnvironmentRegistryError, EnvironmentSource, EnvironmentState, IdlePolicy, + ObservedTarget, PowerState, TargetPhase, }; -use environment_protocol::control::targets::{ - EnvironmentTemplate, ListTemplatesParams, ProviderBindingContext, +use environment_protocol::{ + control::template::{EnvironmentTemplate, ListTemplatesParams}, + ids::UniverseId, }; impl EnvironmentService { @@ -32,11 +35,11 @@ impl EnvironmentService { &self, params: EnvironmentProviderBindingReadParams, ) -> Result { - let binding_id = parse_environment_provider_binding_id(params.binding_id)?; + let provider_id = parse_environment_provider_id(params.provider_id)?; let record = ::environments::EnvironmentProviderBindingStore::read_provider_binding( self.store.as_ref(), self.store.config().universe_id, - &binding_id, + &provider_id, ) .await .map_err(map_environments_error)?; @@ -45,57 +48,62 @@ impl EnvironmentService { }) } + /// Templates are listed live from every enabled binding's provider; + /// nothing about them is stored. pub(crate) async fn list_environment_template_records( &self, params: EnvironmentTemplateListParams, ) -> Result { - let binding_filter = params - .binding_id - .map(parse_environment_provider_binding_id) - .transpose()?; - let bindings = if let Some(binding_id) = binding_filter { - vec![ + let universe_id = self.store.config().universe_id; + let bindings = match params + .provider_id + .map(parse_environment_provider_id) + .transpose()? + { + Some(provider_id) => vec![ ::environments::EnvironmentProviderBindingStore::read_provider_binding( self.store.as_ref(), - self.store.config().universe_id, - &binding_id, + universe_id, + &provider_id, ) .await .map_err(map_environments_error)?, - ] - } else { - ::environments::EnvironmentProviderBindingStore::list_provider_bindings( + ], + None => ::environments::EnvironmentProviderBindingStore::list_provider_bindings( self.store.as_ref(), - self.store.config().universe_id, + universe_id, ) .await - .map_err(map_environments_error)? + .map_err(map_environments_error)?, }; + let universe = UniverseId::new(universe_id.to_string()).map_err(|error| { + AgentApiError::internal(format!( + "universe id does not fit the controller protocol: {error}" + )) + })?; let mut templates = Vec::new(); - for binding in bindings - .into_iter() - .filter(|binding| binding.status == EnvironmentProviderBindingStatus::Enabled) - { - let provider = self.read_environment_provider(&binding.provider_id).await?; - let mut controller = self - .provider_controller_connector - .connect(&provider.controller_connection) - .await?; + for binding in bindings.into_iter().filter(|binding| binding.enabled) { + let controller = self + .connect_provider_controller(&binding.provider_id) + .await + .map_err(ControllerError::into_api_error)?; let response = controller + .client .list_templates(&ListTemplatesParams { - binding: binding_context(&binding), + universe_id: universe.clone(), }) - .await; - let response = finish_provider_controller(controller, response).await?; + .await + .map_err(|error| controller.call_error(error).into_api_error()); + controller.close().await; templates.extend( - response + response? .templates .iter() - .map(|template| environment_template_view(&binding, template)), + .map(|template| environment_template_view(&binding.provider_id, template)), ); } templates.sort_by(|left, right| { - (&left.binding_id, &left.template_id).cmp(&(&right.binding_id, &right.template_id)) + (&left.provider_id, &left.template_id).cmp(&(&right.provider_id, &right.template_id)) }); Ok(EnvironmentTemplateListResponse { templates }) } @@ -107,7 +115,7 @@ impl EnvironmentService { let expected = params.template_id; let response = self .list_environment_template_records(EnvironmentTemplateListParams { - binding_id: Some(params.binding_id), + provider_id: Some(params.provider_id), }) .await?; let template = response @@ -119,22 +127,6 @@ impl EnvironmentService { })?; Ok(EnvironmentTemplateReadResponse { template }) } - - pub(crate) async fn read_environment_provider( - &self, - provider_id: &EnvironmentProviderId, - ) -> Result { - ::environments::EnvironmentProviderStore::read_provider(self.store.as_ref(), provider_id) - .await - .map_err(map_environments_error) - } -} - -pub(crate) fn binding_context(record: &EnvironmentProviderBindingRecord) -> ProviderBindingContext { - ProviderBindingContext { - universe_id: record.universe_id.to_string(), - binding_id: record.binding_id.to_string(), - } } pub(crate) fn parse_environment_provider_id( @@ -144,135 +136,113 @@ pub(crate) fn parse_environment_provider_id( .map_err(|error| AgentApiError::invalid_request(format!("invalid provider id: {error}"))) } -pub(crate) fn parse_environment_provider_binding_id( - value: String, -) -> Result { - EnvironmentProviderBindingId::try_new(value).map_err(|error| { - AgentApiError::invalid_request(format!("invalid provider binding id: {error}")) - }) -} - pub(crate) fn environment_provider_binding_view( record: &EnvironmentProviderBindingRecord, ) -> EnvironmentProviderBindingView { EnvironmentProviderBindingView { - binding_id: record.binding_id.to_string(), provider_id: record.provider_id.to_string(), - status: match record.status { - EnvironmentProviderBindingStatus::Enabled => { - EnvironmentProviderBindingStatusView::Enabled - } - EnvironmentProviderBindingStatus::Disabled => { - EnvironmentProviderBindingStatusView::Disabled - } - }, - revision: record.revision, - metadata: record.metadata.clone(), + enabled: record.enabled, created_at_ms: record.created_at_ms, updated_at_ms: record.updated_at_ms, } } pub(crate) fn environment_template_view( - binding: &EnvironmentProviderBindingRecord, + provider_id: &EnvironmentProviderId, template: &EnvironmentTemplate, ) -> EnvironmentTemplateView { EnvironmentTemplateView { - template_id: template.template_id.clone(), - provider_id: binding.provider_id.to_string(), - binding_id: binding.binding_id.to_string(), - display_name: template.display_name.clone(), - description: template.description.clone(), + template_id: template.template_id.to_string(), + provider_id: provider_id.to_string(), + display_name: template.display_name.to_string(), + description: template + .description + .as_ref() + .map(|description| description.to_string()), public_ingress: template.public_ingress, deprecated: template.deprecated, - metadata: template.metadata.clone(), + resources: template + .resources + .as_ref() + .map(|resources| EnvironmentTemplateResourcesView { + vcpus: resources.vcpus, + memory_bytes: resources.memory_bytes, + disk_bytes: resources.disk_bytes, + }), + power_states: template + .power_states + .iter() + .copied() + .map(power_state_view) + .collect(), + idle: template.idle.as_ref().map(idle_policy_view), + metadata: template.metadata.as_map().clone(), } } pub(crate) fn environment_view(record: &EnvironmentRecord) -> EnvironmentView { EnvironmentView { environment_id: record.environment_id.to_string(), - request_id: record.request_id.to_string(), + request_id: record.request_id.as_ref().map(ToString::to_string), source: match &record.source { - EnvironmentSource::Provisioned { - provider_id, - binding_id, - } => EnvironmentSourceView::Provisioned { - provider_id: provider_id.to_string(), - binding_id: binding_id.to_string(), + EnvironmentSource::Provisioned(source) => EnvironmentSourceView::Provisioned { + provider_id: source.provider_id.to_string(), + template_id: source.template_id.to_string(), + target_id: source + .provider_target_id + .as_ref() + .map(|target_id| target_id.as_str().to_owned()), + idle: source.idle.as_ref().map(idle_policy_view), + observed: source.observed.as_ref().map(observed_target_view), + public_endpoint: source.public_endpoint.clone(), }, - EnvironmentSource::External { connection } => EnvironmentSourceView::External { - connection: EnvironmentConnectionView { - endpoint: connection.endpoint.clone(), - transport: match &connection.transport { - environment_protocol::shared::EnvironmentTransport::WebSocket => { - EnvironmentConnectionTransportView::WebSocket - } - environment_protocol::shared::EnvironmentTransport::Http => { - EnvironmentConnectionTransportView::Http - } - environment_protocol::shared::EnvironmentTransport::Stdio => { - EnvironmentConnectionTransportView::Stdio - } - environment_protocol::shared::EnvironmentTransport::Ssh => { - EnvironmentConnectionTransportView::Ssh - } - environment_protocol::shared::EnvironmentTransport::Provider { - provider_type, - } => EnvironmentConnectionTransportView::Provider { - provider_type: provider_type.clone(), - }, - }, - }, + EnvironmentSource::External { endpoint } => EnvironmentSourceView::External { + endpoint: endpoint.as_str().to_owned(), }, - EnvironmentSource::Registered { - registration_key_id, - daemon_id, - identity_mode, - .. - } => EnvironmentSourceView::Registered { - registration_key_id: registration_key_id.to_string(), - daemon_id: daemon_id.to_string(), - identity_mode: identity_mode_view(*identity_mode), + EnvironmentSource::Registered(source) => EnvironmentSourceView::Registered { + registration_key_id: source.registration_key_id.to_string(), + daemon_id: source.daemon_id().to_string(), + identity_mode: identity_mode_view(source.identity_mode), + connected: source.connected, + connection_changed_at_ms: source.connection_changed_at_ms, + last_seen_at_ms: source.last_seen_at_ms, }, }, display_name: record.display_name.clone(), - status: lifecycle_status_view(record.status), - desired_power: power_state_view(record.desired_power), - idle_policy: record.idle_policy.as_ref().map(idle_policy_view), - incarnation: EnvironmentIncarnationView { - incarnation_id: record.incarnation.incarnation_id.to_string(), - provision_request_id: record - .incarnation - .provision_request_id - .as_ref() - .map(ToString::to_string), - provider_target_id: record - .incarnation - .provider_target_id - .as_ref() - .map(ToString::to_string), - template_id: record - .incarnation - .template_id - .as_ref() - .map(ToString::to_string), - power_states: record - .incarnation - .power_states - .iter() - .copied() - .map(power_state_view) - .collect(), - created_at_ms: record.incarnation.created_at_ms, - updated_at_ms: record.incarnation.updated_at_ms, - }, - public_ingress_enabled: record.public_ingress_enabled, - public_endpoint: record.public_endpoint.clone(), + state: state_view(record.state), + status: status_view(record.display_status()), + state_message: record.state_message.clone(), + close_reason: record.close_reason.map(close_reason_view), + state_changed_at_ms: record.state_changed_at_ms, + daemon: record.daemon_build.as_ref().map(daemon_view), metadata: record.metadata.clone(), - last_seen_at_ms: record.last_seen_at_ms, created_at_ms: record.created_at_ms, updated_at_ms: record.updated_at_ms, + closed_at_ms: record.closed_at_ms, + } +} + +fn observed_target_view(observed: &ObservedTarget) -> EnvironmentObservedTargetView { + EnvironmentObservedTargetView { + phase: match observed.phase { + TargetPhase::Provisioning => EnvironmentTargetPhaseView::Provisioning, + TargetPhase::Active => EnvironmentTargetPhaseView::Active, + TargetPhase::Closing => EnvironmentTargetPhaseView::Closing, + TargetPhase::Closed => EnvironmentTargetPhaseView::Closed, + TargetPhase::Failed => EnvironmentTargetPhaseView::Failed, + TargetPhase::Unknown => EnvironmentTargetPhaseView::Unknown, + }, + power: observed.power.map(power_state_view), + message: observed.message.clone(), + observed_at_ms: observed.observed_at_ms, + } +} + +fn daemon_view(build: &DaemonBuild) -> EnvironmentDaemonView { + EnvironmentDaemonView { + version: build.version.clone(), + git_sha: build.git_sha.clone(), + protocol_version: build.protocol_version, } } @@ -302,27 +272,25 @@ pub(crate) fn registry_identity_mode( } } -pub(crate) fn power_state_view(value: ::environments::PowerState) -> EnvironmentPowerStateView { +pub(crate) fn power_state_view(value: PowerState) -> EnvironmentPowerStateView { match value { - ::environments::PowerState::Running => EnvironmentPowerStateView::Running, - ::environments::PowerState::Paused => EnvironmentPowerStateView::Paused, - ::environments::PowerState::Suspended => EnvironmentPowerStateView::Suspended, - ::environments::PowerState::Stopped => EnvironmentPowerStateView::Stopped, + PowerState::Running => EnvironmentPowerStateView::Running, + PowerState::Paused => EnvironmentPowerStateView::Paused, + PowerState::Suspended => EnvironmentPowerStateView::Suspended, + PowerState::Stopped => EnvironmentPowerStateView::Stopped, } } -pub(crate) fn registry_power_state(value: EnvironmentPowerStateView) -> ::environments::PowerState { +pub(crate) fn registry_power_state(value: EnvironmentPowerStateView) -> PowerState { match value { - EnvironmentPowerStateView::Running => ::environments::PowerState::Running, - EnvironmentPowerStateView::Paused => ::environments::PowerState::Paused, - EnvironmentPowerStateView::Suspended => ::environments::PowerState::Suspended, - EnvironmentPowerStateView::Stopped => ::environments::PowerState::Stopped, + EnvironmentPowerStateView::Running => PowerState::Running, + EnvironmentPowerStateView::Paused => PowerState::Paused, + EnvironmentPowerStateView::Suspended => PowerState::Suspended, + EnvironmentPowerStateView::Stopped => PowerState::Stopped, } } -pub(crate) fn idle_policy_view( - value: &::environments::EnvironmentIdlePolicy, -) -> EnvironmentIdlePolicyView { +pub(crate) fn idle_policy_view(value: &IdlePolicy) -> EnvironmentIdlePolicyView { EnvironmentIdlePolicyView { pause_after_ms: value.pause_after_ms, suspend_after_ms: value.suspend_after_ms, @@ -331,46 +299,89 @@ pub(crate) fn idle_policy_view( } } +/// The idle policy a create-time override names, validated: Lightspeed +/// passes it to the provider unchanged and never interprets it. pub(crate) fn registry_idle_policy( value: &EnvironmentIdlePolicyView, -) -> ::environments::EnvironmentIdlePolicy { - ::environments::EnvironmentIdlePolicy { +) -> Result { + let policy = IdlePolicy { pause_after_ms: value.pause_after_ms, suspend_after_ms: value.suspend_after_ms, stop_after_ms: value.stop_after_ms, close_after_ms: value.close_after_ms, + }; + policy + .validate() + .map_err(|error| AgentApiError::invalid_request(format!("invalid idle policy: {error}")))?; + Ok(policy) +} + +pub(crate) fn state_view(value: EnvironmentState) -> EnvironmentStateView { + match value { + EnvironmentState::Provisioning => EnvironmentStateView::Provisioning, + EnvironmentState::Open => EnvironmentStateView::Open, + EnvironmentState::Closing => EnvironmentStateView::Closing, + EnvironmentState::Closed => EnvironmentStateView::Closed, + EnvironmentState::Failed => EnvironmentStateView::Failed, + } +} + +pub(crate) fn registry_state(value: EnvironmentStateView) -> EnvironmentState { + match value { + EnvironmentStateView::Provisioning => EnvironmentState::Provisioning, + EnvironmentStateView::Open => EnvironmentState::Open, + EnvironmentStateView::Closing => EnvironmentState::Closing, + EnvironmentStateView::Closed => EnvironmentState::Closed, + EnvironmentStateView::Failed => EnvironmentState::Failed, } } -fn lifecycle_status_view(value: EnvironmentStatus) -> EnvironmentLifecycleStatusView { +pub(crate) fn status_view(value: EnvironmentDisplayStatus) -> EnvironmentStatusView { match value { - EnvironmentStatus::Provisioning => EnvironmentLifecycleStatusView::Provisioning, - EnvironmentStatus::Booting => EnvironmentLifecycleStatusView::Booting, - EnvironmentStatus::Ready => EnvironmentLifecycleStatusView::Ready, - EnvironmentStatus::Paused => EnvironmentLifecycleStatusView::Paused, - EnvironmentStatus::Suspended => EnvironmentLifecycleStatusView::Suspended, - EnvironmentStatus::Offline => EnvironmentLifecycleStatusView::Offline, - EnvironmentStatus::Closing => EnvironmentLifecycleStatusView::Closing, - EnvironmentStatus::Closed => EnvironmentLifecycleStatusView::Closed, - EnvironmentStatus::Failed => EnvironmentLifecycleStatusView::Failed, - EnvironmentStatus::Unknown => EnvironmentLifecycleStatusView::Unknown, + EnvironmentDisplayStatus::Provisioning => EnvironmentStatusView::Provisioning, + EnvironmentDisplayStatus::Ready => EnvironmentStatusView::Ready, + EnvironmentDisplayStatus::Paused => EnvironmentStatusView::Paused, + EnvironmentDisplayStatus::Suspended => EnvironmentStatusView::Suspended, + EnvironmentDisplayStatus::Stopped => EnvironmentStatusView::Stopped, + EnvironmentDisplayStatus::Unknown => EnvironmentStatusView::Unknown, + EnvironmentDisplayStatus::Away => EnvironmentStatusView::Away, + EnvironmentDisplayStatus::Closing => EnvironmentStatusView::Closing, + EnvironmentDisplayStatus::Closed => EnvironmentStatusView::Closed, + EnvironmentDisplayStatus::Failed => EnvironmentStatusView::Failed, } } -pub(crate) fn registry_lifecycle_status( - value: EnvironmentLifecycleStatusView, -) -> EnvironmentStatus { +fn close_reason_view(value: EnvironmentCloseReason) -> EnvironmentCloseReasonView { match value { - EnvironmentLifecycleStatusView::Provisioning => EnvironmentStatus::Provisioning, - EnvironmentLifecycleStatusView::Booting => EnvironmentStatus::Booting, - EnvironmentLifecycleStatusView::Ready => EnvironmentStatus::Ready, - EnvironmentLifecycleStatusView::Paused => EnvironmentStatus::Paused, - EnvironmentLifecycleStatusView::Suspended => EnvironmentStatus::Suspended, - EnvironmentLifecycleStatusView::Offline => EnvironmentStatus::Offline, - EnvironmentLifecycleStatusView::Closing => EnvironmentStatus::Closing, - EnvironmentLifecycleStatusView::Closed => EnvironmentStatus::Closed, - EnvironmentLifecycleStatusView::Failed => EnvironmentStatus::Failed, - EnvironmentLifecycleStatusView::Unknown => EnvironmentStatus::Unknown, + EnvironmentCloseReason::Requested => EnvironmentCloseReasonView::Requested, + EnvironmentCloseReason::KeyRevoked => EnvironmentCloseReasonView::KeyRevoked, + EnvironmentCloseReason::EphemeralGrace => EnvironmentCloseReasonView::EphemeralGrace, + EnvironmentCloseReason::Departed => EnvironmentCloseReasonView::Departed, + EnvironmentCloseReason::OfflineTimeout => EnvironmentCloseReasonView::OfflineTimeout, + EnvironmentCloseReason::ProviderEnded => EnvironmentCloseReasonView::ProviderEnded, + } +} + +/// The deployment view of a provider record. The effective data route base +/// is absent only when it cannot be derived. +pub(crate) fn environment_provider_view( + record: &EnvironmentProviderRecord, +) -> DeploymentEnvironmentProviderView { + DeploymentEnvironmentProviderView { + provider_id: record.provider_id.to_string(), + display_name: record.display_name.clone(), + controller_endpoint: record.controller_endpoint.as_str().to_owned(), + data_route_base: record + .data_route_base + .as_ref() + .map(|base| base.as_str().to_owned()), + effective_data_route_base: record + .effective_data_route_base() + .ok() + .map(|base| base.as_str().to_owned()), + has_credential: record.has_credential, + created_at_ms: record.created_at_ms, + updated_at_ms: record.updated_at_ms, } } @@ -382,14 +393,6 @@ pub(crate) fn map_environments_error(error: EnvironmentRegistryError) -> AgentAp EnvironmentRegistryError::NotFound { kind, id } => { AgentApiError::not_found(format!("environment registry {kind} not found: {id}")) } - EnvironmentRegistryError::RevisionConflict { - kind, - id, - expected, - actual, - } => AgentApiError::conflict(format!( - "environment registry revision conflict for {kind} {id}: expected {expected:?}, actual {actual:?}" - )), EnvironmentRegistryError::InvalidInput { message } => { AgentApiError::invalid_request(message) } @@ -402,3 +405,145 @@ pub(crate) fn map_environments_error(error: EnvironmentRegistryError) -> AgentAp EnvironmentRegistryError::Store { message } => AgentApiError::internal(message), } } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use ::environments::{ + EndpointScheme, EnvironmentEndpoint, EnvironmentProviderBindingStore, + EnvironmentProviderStore, EnvironmentStore, InMemoryEnvironmentRegistryStore, + PutEnvironmentProvider, PutEnvironmentProviderBinding, + }; + use environment_protocol::routes::DataRouteBase; + + use super::*; + + #[test] + fn the_provider_view_shows_the_effective_data_route_and_no_credential() { + let record = EnvironmentProviderRecord { + provider_id: EnvironmentProviderId::new("provider"), + display_name: None, + controller_endpoint: EnvironmentEndpoint::parse("wss://provider.test:8443/control") + .unwrap(), + data_route_base: None, + has_credential: false, + created_at_ms: 1, + updated_at_ms: 1, + }; + let view = environment_provider_view(&record); + assert_eq!(view.data_route_base, None); + assert_eq!( + view.effective_data_route_base.as_deref(), + Some("wss://provider.test:8443/routes") + ); + assert!(!view.has_credential); + let local = EnvironmentProviderRecord { + controller_endpoint: EnvironmentEndpoint::parse("unix:///run/provider.sock").unwrap(), + data_route_base: Some(DataRouteBase::new("ws://127.0.0.1:9/routes").unwrap()), + has_credential: true, + ..record + }; + assert_eq!(local.controller_endpoint.scheme(), EndpointScheme::Unix); + let view = environment_provider_view(&local); + assert_eq!( + view.effective_data_route_base.as_deref(), + Some("ws://127.0.0.1:9/routes") + ); + assert!(view.has_credential); + } + + #[test] + fn idle_overrides_are_validated_before_they_reach_a_provider() { + let valid = EnvironmentIdlePolicyView { + pause_after_ms: Some(60_000), + stop_after_ms: Some(3_600_000), + ..EnvironmentIdlePolicyView::default() + }; + assert_eq!( + idle_policy_view(®istry_idle_policy(&valid).unwrap()), + valid + ); + let out_of_order = EnvironmentIdlePolicyView { + pause_after_ms: Some(10), + stop_after_ms: Some(5), + ..EnvironmentIdlePolicyView::default() + }; + assert_eq!( + registry_idle_policy(&out_of_order).unwrap_err().kind, + api::AgentApiErrorKind::InvalidRequest + ); + assert_eq!( + registry_idle_policy(&EnvironmentIdlePolicyView::default()) + .unwrap_err() + .kind, + api::AgentApiErrorKind::InvalidRequest + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_provisioned_view_carries_state_status_and_the_create_inputs() { + let store = InMemoryEnvironmentRegistryStore::new(); + let provider_id = EnvironmentProviderId::new("provider"); + store + .put_provider(PutEnvironmentProvider { + provider_id: provider_id.clone(), + display_name: None, + controller_endpoint: EnvironmentEndpoint::parse("ws://provider.test/control") + .unwrap(), + data_route_base: None, + credential: ::environments::ProviderCredentialChange::Keep, + updated_at_ms: 1, + }) + .await + .unwrap(); + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id: store.universe_id(), + provider_id: provider_id.clone(), + enabled: true, + updated_at_ms: 1, + }) + .await + .unwrap(); + let idle = IdlePolicy { + pause_after_ms: Some(60_000), + ..IdlePolicy::default() + }; + let record = store + .create_environment(::environments::CreateEnvironment { + request_id: ::environments::EnvironmentProvisionRequestId::new("request"), + environment_id: ::environments::EnvironmentId::new("environment"), + provider_id: provider_id.clone(), + template_id: ::environments::EnvironmentTemplateId::new("template"), + idle: Some(idle.clone()), + display_name: None, + metadata: BTreeMap::new(), + created_at_ms: 10, + }) + .await + .unwrap(); + let view = environment_view(&record); + assert_eq!(view.state, EnvironmentStateView::Provisioning); + assert_eq!(view.status, EnvironmentStatusView::Provisioning); + assert_eq!(view.request_id.as_deref(), Some("request")); + assert_eq!(view.close_reason, None); + match view.source { + EnvironmentSourceView::Provisioned { + provider_id, + template_id, + target_id, + idle: shown, + observed, + .. + } => { + assert_eq!(provider_id, "provider"); + assert_eq!(template_id, "template"); + assert_eq!(target_id, None); + assert_eq!(shown, Some(idle_policy_view(&idle))); + assert_eq!(observed, None); + } + other => panic!("expected a provisioned source, got {other:?}"), + } + } +} diff --git a/crates/temporal-runtime/src/environments/registration.rs b/crates/temporal-runtime/src/environments/registration.rs index 9317b8276..783120343 100644 --- a/crates/temporal-runtime/src/environments/registration.rs +++ b/crates/temporal-runtime/src/environments/registration.rs @@ -1,58 +1,16 @@ //! Registration keys: the universe-management surface of key-based outbound -//! `envd` registration. Keys are minted here; admission itself happens on -//! the environment gateway's connect route. - -use std::collections::BTreeMap; +//! `envd` registration. Keys are minted, edited and rotated here; admission +//! itself happens on the environment gateway's connect route, and the +//! gateway's registered sweep applies the keys' disconnect policies. use ::environments::{ - BeginCloseEnvironment, CreateEnvironmentRegistrationKey, EnvironmentRecord, - EnvironmentRegistrationKeyRecord, EnvironmentRegistrationKeyStatus, - EnvironmentRegistrationKeyStore, EnvironmentStatus, EnvironmentStore, ListEnvironments, - ObserveRegisteredEnvironment, RegisteredConnectionObservation, RegisteredIdentityMode, - RegistrationKeyPolicy, RevokeEnvironmentRegistrationKey, mint_registration_key, + CreateEnvironmentRegistrationKey, EnvironmentCloseReason, EnvironmentRegistrationKeyRecord, + EnvironmentRegistrationKeyStatus, EnvironmentRegistrationKeyStore, EnvironmentState, + EnvironmentStore, ListEnvironments, RegistrationKeyPolicy, RegistrationKeySettings, + RevokeEnvironmentRegistrationKey, RotateEnvironmentRegistrationKey, + UpdateEnvironmentRegistrationKey, mint_registration_key, mint_registration_secret, }; -/// A registered environment whose heartbeat stamp is older than this is -/// treated as disconnected even if its row still says `Ready`: the gateway -/// that held its control connection stopped without recording the -/// disconnect. Three missed heartbeats leaves room for one slow pong. -pub(crate) const REGISTERED_STALE_AFTER_MS: i64 = - 3 * crate::gateway::registration::HEARTBEAT_INTERVAL.as_millis() as i64; - -/// What the reconciler does with one open registered environment. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum RegisteredSweepAction { - /// `Ready` with a stale stamp: record the disconnect the gateway lost. - MarkOffline, - /// Ephemeral and away past the key's grace: close it. - Close, -} - -/// Pure decision for one open registered environment. Persistent -/// environments only ever get repaired; ephemeral ones close once their -/// daemon has been away longer than the key's disconnect grace. -pub(crate) fn registered_sweep_action( - environment: &EnvironmentRecord, - key: &EnvironmentRegistrationKeyRecord, - now_ms: i64, -) -> Option { - if !environment.registered_daemon_absent(now_ms, REGISTERED_STALE_AFTER_MS) { - return None; - } - let away_ms = environment - .last_seen_at_ms - .map_or(i64::MAX, |seen| now_ms.saturating_sub(seen)); - if key.identity_mode == RegisteredIdentityMode::Ephemeral - && away_ms > i64::try_from(key.ephemeral_disconnect_grace_ms()).unwrap_or(i64::MAX) - { - return Some(RegisteredSweepAction::Close); - } - if environment.status == EnvironmentStatus::Ready { - return Some(RegisteredSweepAction::MarkOffline); - } - None -} - use super::*; use super::{ lifecycle::parse_registration_key_id, @@ -72,6 +30,7 @@ impl EnvironmentService { identity_mode: registry_identity_mode(params.identity_mode), max_active_environments: params.max_active_environments, ephemeral_disconnect_grace_ms: params.ephemeral_disconnect_grace_ms, + offline_close_after_ms: params.offline_close_after_ms, expires_at_ms: params.expires_at_ms, }, now, @@ -80,7 +39,7 @@ impl EnvironmentService { let record = EnvironmentRegistrationKeyStore::create_registration_key( self.store.as_ref(), CreateEnvironmentRegistrationKey { - secret_hash: minted.secret_hash, + secret_hash: minted.secret.secret_hash, record: minted.record, }, ) @@ -95,7 +54,7 @@ impl EnvironmentService { ); Ok(EnvironmentRegistrationKeyCreateResponse { registration_key: self.registration_key_view(&record, now).await?, - secret: EnvironmentRegistrationSecretView(minted.secret.expose().to_owned()), + secret: EnvironmentRegistrationSecretView(minted.secret.secret.expose().to_owned()), }) } @@ -130,6 +89,69 @@ impl EnvironmentService { Ok(EnvironmentRegistrationKeyListResponse { registration_keys }) } + /// `environments/registration-keys/update`: replace the editable policy. + /// The identity mode never changes. + pub(crate) async fn update_environment_registration_key_record( + &self, + params: EnvironmentRegistrationKeyUpdateParams, + ) -> Result { + let registration_key_id = parse_registration_key_id(params.registration_key_id)?; + let now = now_ms()?; + let record = EnvironmentRegistrationKeyStore::update_registration_key( + self.store.as_ref(), + UpdateEnvironmentRegistrationKey { + registration_key_id, + settings: RegistrationKeySettings { + display_name: params.display_name, + max_active_environments: params.max_active_environments, + ephemeral_disconnect_grace_ms: params.ephemeral_disconnect_grace_ms, + offline_close_after_ms: params.offline_close_after_ms, + expires_at_ms: params.expires_at_ms, + }, + updated_at_ms: now, + }, + ) + .await + .map_err(map_environments_error)?; + Ok(EnvironmentRegistrationKeyUpdateResponse { + registration_key: self.registration_key_view(&record, now).await?, + }) + } + + /// `environments/registration-keys/rotate`: retire the live secret at + /// once and mint a new one. The key's group, allowlists and admitted + /// daemons are unaffected. + pub(crate) async fn rotate_environment_registration_key_record( + &self, + params: EnvironmentRegistrationKeyRotateParams, + ) -> Result { + let registration_key_id = parse_registration_key_id(params.registration_key_id)?; + let now = now_ms()?; + let minted = mint_registration_secret(); + let record = EnvironmentRegistrationKeyStore::rotate_registration_key( + self.store.as_ref(), + RotateEnvironmentRegistrationKey { + registration_key_id, + secret_hash: minted.secret_hash, + key_prefix: minted.key_prefix, + rotated_at_ms: now, + }, + ) + .await + .map_err(map_environments_error)?; + tracing::info!( + target: "temporal_runtime", + registration_key_id = %record.registration_key_id, + "environment registration key rotated" + ); + Ok(EnvironmentRegistrationKeyRotateResponse { + registration_key: self.registration_key_view(&record, now).await?, + secret: EnvironmentRegistrationSecretView(minted.secret.expose().to_owned()), + }) + } + + /// Revoke a key; with `closeEnvironments`, close every environment it + /// admitted (`key_revoked`) and end their live routes. pub(crate) async fn revoke_environment_registration_key_record( &self, params: EnvironmentRegistrationKeyRevokeParams, @@ -147,33 +169,37 @@ impl EnvironmentService { .map_err(map_environments_error)?; let mut closed_environment_ids = Vec::new(); if params.close_environments { - let environments = EnvironmentStore::list_environments( - self.store.as_ref(), - ListEnvironments { - metadata: Default::default(), - registration_key_id: Some(registration_key_id.clone()), - ..ListEnvironments::default() - }, - ) - .await - .map_err(map_environments_error)?; - for environment in environments { - if matches!( - environment.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) { - continue; - } - EnvironmentStore::begin_close_environment( + let mut after = None; + loop { + let page = EnvironmentStore::list_environments( self.store.as_ref(), - BeginCloseEnvironment { - environment_id: environment.environment_id.clone(), - updated_at_ms: now, + ListEnvironments { + registration_key_id: Some(registration_key_id.clone()), + limit: ::environments::LIST_ENVIRONMENTS_MAX_LIMIT, + after, + ..ListEnvironments::default() }, ) .await .map_err(map_environments_error)?; - closed_environment_ids.push(environment.environment_id.to_string()); + for environment in &page.environments { + if matches!( + environment.state, + EnvironmentState::Closing | EnvironmentState::Closed + ) { + continue; + } + self.begin_close( + &environment.environment_id, + EnvironmentCloseReason::KeyRevoked, + ) + .await?; + closed_environment_ids.push(environment.environment_id.to_string()); + } + after = page.next_after; + if after.is_none() { + break; + } } } tracing::info!( @@ -188,80 +214,6 @@ impl EnvironmentService { }) } - /// One pass over open registered environments: repair stale `Ready` - /// rows and close ephemeral environments past their disconnect grace. - /// Returns how many rows changed. - pub(crate) async fn reconcile_registered_once(&self) -> Result { - let environments = EnvironmentStore::list_open_registered_environments(self.store.as_ref()) - .await - .map_err(map_environments_error)?; - if environments.is_empty() { - return Ok(0); - } - let now = now_ms()?; - let mut keys: BTreeMap = BTreeMap::new(); - let mut changed = 0; - for environment in environments { - let Some(key_id) = environment.registration_key_id() else { - continue; - }; - let key = match keys.get(key_id.as_str()) { - Some(key) => key.clone(), - None => { - let key = EnvironmentRegistrationKeyStore::read_registration_key( - self.store.as_ref(), - key_id, - ) - .await - .map_err(map_environments_error)?; - keys.insert(key_id.to_string(), key.clone()); - key - } - }; - match registered_sweep_action(&environment, &key, now) { - None => {} - Some(RegisteredSweepAction::MarkOffline) => { - tracing::info!( - target: "temporal_runtime", - environment = %environment.environment_id, - "registered environment heartbeat is stale; marking offline" - ); - EnvironmentStore::observe_registered_environment( - self.store.as_ref(), - ObserveRegisteredEnvironment { - environment_id: environment.environment_id.clone(), - observation: RegisteredConnectionObservation::Disconnected, - observed_at_ms: now, - metadata: BTreeMap::new(), - }, - ) - .await - .map_err(map_environments_error)?; - changed += 1; - } - Some(RegisteredSweepAction::Close) => { - tracing::info!( - target: "temporal_runtime", - environment = %environment.environment_id, - registration_key_id = %key.registration_key_id, - "ephemeral registered environment exceeded its disconnect grace; closing" - ); - EnvironmentStore::begin_close_environment( - self.store.as_ref(), - BeginCloseEnvironment { - environment_id: environment.environment_id.clone(), - updated_at_ms: now, - }, - ) - .await - .map_err(map_environments_error)?; - changed += 1; - } - } - } - Ok(changed) - } - async fn registration_key_view( &self, record: &EnvironmentRegistrationKeyRecord, @@ -273,31 +225,41 @@ impl EnvironmentService { ) .await .map_err(map_environments_error)?; - Ok(EnvironmentRegistrationKeyView { - registration_key_id: record.registration_key_id.to_string(), - display_name: record.display_name.clone(), - key_prefix: record.key_prefix.clone(), - identity_mode: identity_mode_view(record.identity_mode), - max_active_environments: record.max_active_environments, - ephemeral_disconnect_grace_ms: record.ephemeral_disconnect_grace_ms(), - expires_at_ms: record.expires_at_ms, - status: match record.status(now_ms) { - EnvironmentRegistrationKeyStatus::Active => { - EnvironmentRegistrationKeyStatusView::Active - } - EnvironmentRegistrationKeyStatus::Revoked => { - EnvironmentRegistrationKeyStatusView::Revoked - } - EnvironmentRegistrationKeyStatus::Expired => { - EnvironmentRegistrationKeyStatusView::Expired - } - }, - registered_environment_count: usage.registered, - active_environment_count: usage.active, - last_registered_at_ms: usage.last_registered_at_ms, - created_at_ms: record.created_at_ms, - revoked_at_ms: record.revoked_at_ms, - }) + Ok(registration_key_view(record, usage, now_ms)) + } +} + +pub(crate) fn registration_key_view( + record: &EnvironmentRegistrationKeyRecord, + usage: ::environments::RegistrationKeyUsage, + now_ms: i64, +) -> EnvironmentRegistrationKeyView { + EnvironmentRegistrationKeyView { + registration_key_id: record.registration_key_id.to_string(), + display_name: record.display_name.clone(), + key_prefix: record.key_prefix.clone(), + identity_mode: identity_mode_view(record.identity_mode), + max_active_environments: record.max_active_environments, + ephemeral_disconnect_grace_ms: record.ephemeral_disconnect_grace_ms(), + offline_close_after_ms: record.offline_close_after_ms, + expires_at_ms: record.expires_at_ms, + status: match record.status(now_ms) { + EnvironmentRegistrationKeyStatus::Active => { + EnvironmentRegistrationKeyStatusView::Active + } + EnvironmentRegistrationKeyStatus::Revoked => { + EnvironmentRegistrationKeyStatusView::Revoked + } + EnvironmentRegistrationKeyStatus::Expired => { + EnvironmentRegistrationKeyStatusView::Expired + } + }, + registered_environment_count: usage.registered, + active_environment_count: usage.active, + last_registered_at_ms: usage.last_registered_at_ms, + created_at_ms: record.created_at_ms, + updated_at_ms: record.updated_at_ms, + revoked_at_ms: record.revoked_at_ms, } } @@ -310,132 +272,47 @@ fn allocate_registration_key_id() -> ::environments::EnvironmentRegistrationKeyI #[cfg(test)] mod tests { - use std::collections::BTreeMap; - use ::environments::{ - EnvironmentDaemonId, EnvironmentId, EnvironmentIncarnationId, EnvironmentIncarnationRecord, - EnvironmentProvisionRequestId, EnvironmentRegistrationKeyId, EnvironmentSource, PowerState, + EnvironmentRegistrationKeyId, RegisteredIdentityMode, RegistrationKeyUsage, }; use super::*; - fn key( - mode: RegisteredIdentityMode, - grace_ms: Option, - ) -> EnvironmentRegistrationKeyRecord { - EnvironmentRegistrationKeyRecord { - registration_key_id: EnvironmentRegistrationKeyId::new("rk"), - display_name: "pool".to_owned(), - key_prefix: "lsrk_abcdefgh".to_owned(), - identity_mode: mode, - max_active_environments: None, - ephemeral_disconnect_grace_ms: grace_ms, - expires_at_ms: None, - created_at_ms: 0, - revoked_at_ms: None, - } - } - - fn environment(status: EnvironmentStatus, last_seen_at_ms: Option) -> EnvironmentRecord { - let public_key = "ab".repeat(32); - let daemon_id = EnvironmentDaemonId::from_public_key(&[0xab; 32]); - EnvironmentRecord { - environment_id: EnvironmentId::new("env"), - request_id: EnvironmentProvisionRequestId::for_daemon(&daemon_id), - source: EnvironmentSource::Registered { - registration_key_id: EnvironmentRegistrationKeyId::new("rk"), - daemon_id, - daemon_public_key: public_key, - identity_mode: RegisteredIdentityMode::Ephemeral, + #[test] + fn the_key_view_carries_the_editable_policy_and_its_update_time() { + let mut minted = mint_registration_key( + EnvironmentRegistrationKeyId::new("rk"), + RegistrationKeyPolicy { + display_name: "fleet".to_owned(), + identity_mode: RegisteredIdentityMode::Persistent, + max_active_environments: Some(3), + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: Some(86_400_000), + expires_at_ms: None, }, - display_name: None, - status, - desired_power: PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: EnvironmentIncarnationId::new("inc"), - provision_request_id: None, - provider_target_id: None, - template_id: None, - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: 0, - updated_at_ms: 0, + 5, + ) + .unwrap(); + minted.record.updated_at_ms = 9; + let view = registration_key_view( + &minted.record, + RegistrationKeyUsage { + registered: 2, + active: 1, + last_registered_at_ms: Some(7), }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: BTreeMap::new(), - last_seen_at_ms, - created_at_ms: 0, - updated_at_ms: 0, - } - } - - #[test] - fn fresh_heartbeats_need_nothing() { - let ready = environment(EnvironmentStatus::Ready, Some(1_000)); - assert_eq!( - registered_sweep_action( - &ready, - &key(RegisteredIdentityMode::Ephemeral, Some(1)), - 1_500 - ), - None - ); - } - - #[test] - fn stale_ready_rows_are_marked_offline_before_grace_and_closed_after() { - let stale = environment(EnvironmentStatus::Ready, Some(0)); - let now = REGISTERED_STALE_AFTER_MS + 1; - assert_eq!( - registered_sweep_action( - &stale, - &key(RegisteredIdentityMode::Ephemeral, Some(1_000_000)), - now - ), - Some(RegisteredSweepAction::MarkOffline) - ); - assert_eq!( - registered_sweep_action( - &stale, - &key(RegisteredIdentityMode::Ephemeral, Some(1_000)), - now - ), - Some(RegisteredSweepAction::Close) - ); - assert_eq!( - registered_sweep_action( - &stale, - &key(RegisteredIdentityMode::Persistent, Some(1)), - now - ), - Some(RegisteredSweepAction::MarkOffline) - ); - } - - #[test] - fn offline_environments_close_only_when_ephemeral_and_past_grace() { - let offline = environment(EnvironmentStatus::Offline, Some(10_000)); - let persistent = key(RegisteredIdentityMode::Persistent, Some(1)); - assert_eq!( - registered_sweep_action(&offline, &persistent, 1_000_000), - None - ); - let ephemeral = key(RegisteredIdentityMode::Ephemeral, Some(5_000)); - assert_eq!(registered_sweep_action(&offline, &ephemeral, 14_000), None); - assert_eq!( - registered_sweep_action(&offline, &ephemeral, 15_001), - Some(RegisteredSweepAction::Close) + 10, ); - let default_grace = key(RegisteredIdentityMode::Ephemeral, None); + assert_eq!(view.offline_close_after_ms, Some(86_400_000)); + assert_eq!(view.updated_at_ms, 9); + assert_eq!(view.status, EnvironmentRegistrationKeyStatusView::Active); assert_eq!( - registered_sweep_action( - &offline, - &default_grace, - 10_000 + ::environments::DEFAULT_EPHEMERAL_DISCONNECT_GRACE_MS as i64 + 1 + ( + view.registered_environment_count, + view.active_environment_count ), - Some(RegisteredSweepAction::Close) + (2, 1) ); + assert_eq!(view.key_prefix, minted.secret.key_prefix); } } diff --git a/crates/temporal-runtime/src/environments/resolver.rs b/crates/temporal-runtime/src/environments/resolver.rs index 9b9f24a18..98d4f9454 100644 --- a/crates/temporal-runtime/src/environments/resolver.rs +++ b/crates/temporal-runtime/src/environments/resolver.rs @@ -1,12 +1,15 @@ -//! Internal environment lookup and liveness policy. +//! Internal environment lookup and the single use gate. //! //! This is a shared runtime service, not a provider/plugin extension seam. +//! Every consumer asks [`EnvironmentRecord::use_gate`] through it; nothing +//! here writes to a row. use std::sync::Arc; +use environment_client::{ClientError, DataClient}; use environments::{ - EnvironmentId, EnvironmentProviderStore, EnvironmentRecord, EnvironmentRegistryError, - EnvironmentSource, EnvironmentStatus, EnvironmentStore, PowerState, SetEnvironmentPower, + EnvironmentGoneReason, EnvironmentId, EnvironmentRecord, EnvironmentRegistryError, + EnvironmentState, EnvironmentStore, EnvironmentUse, EnvironmentWaitReason, }; use store_pg::PgStore; use thiserror::Error; @@ -14,7 +17,6 @@ use thiserror::Error; #[derive(Clone)] pub(crate) struct EnvironmentResolver { environments: Arc, - providers: Arc, gateway: Option, universe_id: uuid::Uuid, } @@ -23,13 +25,10 @@ impl EnvironmentResolver { pub(crate) fn universe_id(&self) -> uuid::Uuid { self.universe_id } - pub(crate) fn new( - environments: Arc, - providers: Arc, - ) -> Self { + + pub(crate) fn new(environments: Arc) -> Self { Self { environments, - providers, gateway: None, universe_id: uuid::Uuid::nil(), } @@ -37,7 +36,7 @@ impl EnvironmentResolver { pub(crate) fn from_pg_store(store: Arc) -> Self { let universe_id = store.config().universe_id; - let mut resolver = Self::new(store.clone(), store); + let mut resolver = Self::new(store); resolver.universe_id = universe_id; resolver } @@ -59,115 +58,136 @@ impl EnvironmentResolver { Ok(self.environments.read_environment(environment_id).await?) } - /// Validate selection using registry state only. Selecting or reselecting - /// an environment never changes power or proves data-plane reachability. + /// Selection: rejects closing, closed and failed environments; accepts + /// one that is not usable yet and a registered one whose daemon is away. + /// Selecting never changes power or proves reachability. pub(crate) async fn selectable( &self, environment_id: &EnvironmentId, ) -> Result { let environment = self.read(environment_id).await?; - match environment.status { - EnvironmentStatus::Failed => Err(EnvironmentResolveError::Failed { - environment_id: environment.environment_id.to_string(), - message: environment - .metadata - .get(LIFECYCLE_ERROR_METADATA_KEY) - .cloned() - .unwrap_or_else(|| "environment provisioning failed".to_owned()), - }), - EnvironmentStatus::Closing | EnvironmentStatus::Closed => { + match environment.state { + EnvironmentState::Failed => { + Err(failed(&environment, environment.state_message.clone())) + } + EnvironmentState::Closing | EnvironmentState::Closed => { Err(EnvironmentResolveError::Closed { environment_id: environment.environment_id.to_string(), }) } - _ => Ok(environment), + EnvironmentState::Provisioning | EnvironmentState::Open => Ok(environment), } } - /// Check readiness for actual use, requesting wake-up where supported and - /// probing the data route. Selection itself uses only registry validation. - pub(crate) async fn ready_for_use( + /// The gate before a real connection: usable → the record; not yet → + /// `NotReady`; gone → `Closed`, `Failed` or `Away`. Never writes. + pub(crate) async fn resolve_for_connection( &self, environment_id: &EnvironmentId, now_ms: i64, ) -> Result { - let environment = self.resolve_for_connection(environment_id, now_ms).await?; - if let Some(gateway) = &self.gateway { - let connection = gateway.connection_for(self.universe_id, &environment); - if let Ok(mut client) = environment_client::EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("lightspeed-environment-readiness"), - ) - .await - { - let _ = client.close().await; - return Ok(environment); - } + let environment = self.read(environment_id).await?; + match environment.use_gate(now_ms) { + EnvironmentUse::Use => Ok(environment), + gate => Err(gate_error(&environment, gate)), } - Err(EnvironmentResolveError::EnvironmentUnavailable { - environment_id: environment.environment_id.as_str().to_owned(), - status: "environment endpoint is not reachable".to_owned(), - }) } - /// Validate lifecycle and policy immediately before opening a real - /// data-plane connection. Unlike [`Self::ready_for_use`], this does not open - /// a second connection merely to prove reachability. - pub(crate) async fn resolve_for_connection( + /// [`Self::resolve_for_connection`], then one handshake over the worker + /// route. The dial is the wake signal: a powered-down target answers a + /// typed refusal while it wakes, which maps through + /// [`EnvironmentRecord::bound_route_refusal`]. The handshake proves + /// nothing durable and its result is never written. A daemon of another + /// protocol version is a typed error, never a usable environment whose + /// calls all fail. + pub(crate) async fn ready_for_use( &self, environment_id: &EnvironmentId, now_ms: i64, ) -> Result { - let environment = self.selectable(environment_id).await?; - if let Some(provider_id) = environment.provider_id() { - self.providers.read_provider(provider_id).await?; - } - match environment.status { - EnvironmentStatus::Provisioning | EnvironmentStatus::Booting => { - return Err(EnvironmentResolveError::NotReady { - environment_id: environment.environment_id.as_str().to_owned(), - status: environment.status, - }); - } - EnvironmentStatus::Paused - | EnvironmentStatus::Suspended - | EnvironmentStatus::Offline - if wake_on_use_applies(&environment) => - { - if environment.desired_power != PowerState::Running { - self.environments - .set_environment_power(SetEnvironmentPower { - environment_id: environment.environment_id.clone(), - desired_power: PowerState::Running, - updated_at_ms: now_ms.max(0), - }) - .await?; - } - return Err(EnvironmentResolveError::NotReady { - environment_id: environment.environment_id.as_str().to_owned(), - status: environment.status, - }); + let environment = self.resolve_for_connection(environment_id, now_ms).await?; + let Some(gateway) = &self.gateway else { + return Err(EnvironmentResolveError::Unreachable { + environment_id: environment.environment_id.to_string(), + status: "no environment gateway is configured".to_owned(), + }); + }; + let endpoint = gateway.endpoint_for(self.universe_id, &environment); + match DataClient::connect_initialized(&endpoint, gateway.connect_options(READINESS_CLIENT)) + .await + { + Ok((client, _)) => { + let _ = client.close().await; + Ok(environment) } - EnvironmentStatus::Ready if environment.desired_power != PowerState::Running => { - // Use cancels a pending power-down: the idle reaper has asked - // for a lower power state but the reconciler has not converged - // yet. Keeping the intent at `running` stops the reconciler - // from freezing the environment under the call about to - // start; the reaper re-evaluates from the daemon's next idle - // report. Sessions are not consulted — any use counts. - return self - .environments - .set_environment_power(SetEnvironmentPower { - environment_id: environment.environment_id.clone(), - desired_power: PowerState::Running, - updated_at_ms: now_ms.max(0), - }) - .await - .map_err(EnvironmentResolveError::from); + Err(ClientError::VersionMismatch { local, remote, .. }) => { + Err(EnvironmentResolveError::VersionMismatch { + environment_id: environment.environment_id.to_string(), + local, + remote, + }) } - _ => {} + Err(error) => Err( + route_refusal_error(&environment, &error).unwrap_or_else(|| { + EnvironmentResolveError::Unreachable { + environment_id: environment.environment_id.to_string(), + status: error.to_string(), + } + }), + ), } - Ok(environment) + } +} + +/// A worker-route dial error that carries a typed refusal, as a resolve +/// error (through [`EnvironmentRecord::bound_route_refusal`]); `None` when +/// it carries none. +pub(crate) fn route_refusal_error( + record: &EnvironmentRecord, + error: &ClientError, +) -> Option { + let refusal = error.data_route_refusal()?; + match record.bound_route_refusal(refusal, error.retry_after_ms()) { + EnvironmentUse::Use => None, + gate => Some(gate_error(record, gate)), + } +} + +/// The resolve error a gate answer other than `Use` is. +fn gate_error(record: &EnvironmentRecord, gate: EnvironmentUse) -> EnvironmentResolveError { + let environment_id = record.environment_id.to_string(); + match gate { + EnvironmentUse::Use => EnvironmentResolveError::Unreachable { + environment_id, + status: "the environment is usable".to_owned(), + }, + EnvironmentUse::Wait { + reason, + retry_after_ms, + until_ms, + } => EnvironmentResolveError::NotReady { + environment_id, + reason, + retry_after_ms, + until_ms, + }, + EnvironmentUse::Gone { reason, message } => match reason { + EnvironmentGoneReason::Closing | EnvironmentGoneReason::Closed => { + EnvironmentResolveError::Closed { environment_id } + } + EnvironmentGoneReason::Failed => failed(record, message), + EnvironmentGoneReason::Away => EnvironmentResolveError::Away { + environment_id, + message: message.unwrap_or_else(|| "the daemon is away".to_owned()), + }, + }, + } +} + +fn failed(record: &EnvironmentRecord, message: Option) -> EnvironmentResolveError { + EnvironmentResolveError::Failed { + environment_id: record.environment_id.to_string(), + message: message.unwrap_or_else(|| "the environment failed".to_owned()), } } @@ -176,21 +196,17 @@ pub(crate) enum EnvironmentResolveError { #[error(transparent)] Store(#[from] EnvironmentRegistryError), - #[error("environment is unavailable: {environment_id} ({status})")] - EnvironmentUnavailable { - environment_id: String, - status: String, - }, - - /// The environment exists and is being provisioned or booted; it is not - /// yet reachable but selecting it is valid intent. - #[error("environment is not ready yet: {environment_id} ({status:?})")] + /// Not usable yet: re-check after `retry_after_ms`; `until_ms` bounds + /// the wait when the gate knows a bound. + #[error("environment is not ready yet: {environment_id} ({reason})")] NotReady { environment_id: String, - status: EnvironmentStatus, + reason: EnvironmentWaitReason, + retry_after_ms: u64, + until_ms: Option, }, - #[error("environment failed to provision: {environment_id}: {message}")] + #[error("environment failed: {environment_id}: {message}")] Failed { environment_id: String, message: String, @@ -198,39 +214,74 @@ pub(crate) enum EnvironmentResolveError { #[error("environment is closed: {environment_id}")] Closed { environment_id: String }, -} -/// Metadata key under which the lifecycle reconciler records the provider's -/// failure message on a `failed` environment. -pub(crate) const LIFECYCLE_ERROR_METADATA_KEY: &str = "lifecycleError"; + /// Terminal: the registered daemon is away past its bound (gate + /// `Gone{Away}`, a `disconnected` refusal for an ephemeral key, or the + /// gateway's `away` refusal). + #[error("environment {environment_id} is away: {message}")] + Away { + environment_id: String, + message: String, + }, + + /// Retryable: the dial failed without a typed refusal (connect error, + /// timeout, the gateway's bodiless 503 for a provider 5xx/429/timeout, + /// 502, 504, a gateway restart). `status` is a short display string. + #[error("environment is unreachable: {environment_id} ({status})")] + Unreachable { + environment_id: String, + status: String, + }, -/// A powered-down provisioned environment wakes on use when its provider -/// reported that it can be moved back to `running`. External environments and -/// providers without power control keep the reachability probe. -pub(crate) fn wake_on_use_applies(environment: &EnvironmentRecord) -> bool { - matches!(environment.source, EnvironmentSource::Provisioned { .. }) - && environment.status.is_powered_down() - && environment - .incarnation - .power_states - .contains(&PowerState::Running) + /// The environment's envd speaks another data-plane protocol version; + /// it cannot serve this runtime until it is replaced or upgraded. + #[error( + "environment {environment_id} runs envd protocol version {remote}; this runtime requires {local}" + )] + VersionMismatch { + environment_id: String, + local: u32, + remote: u32, + }, } +/// `clientInfo.name` of readiness handshakes. +const READINESS_CLIENT: &str = "lightspeed-environment-readiness"; + #[cfg(test)] mod tests { use std::collections::BTreeMap; - use environment_protocol::shared::{EnvironmentTransport, ProviderTargetId}; + use environment_protocol::{ + control::target::{PowerState, TargetPhase}, + ids::ProviderTargetId, + metadata::DescriptiveText, + routes::DataRouteRefusal, + }; use environments::{ - CreateEnvironment, EnvironmentConnectionSpec, EnvironmentIncarnationId, - EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, - EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProvisionRequestId, - EnvironmentStatus, EnvironmentTemplateId, ObserveProvisionedEnvironment, - PutEnvironmentProvider, PutEnvironmentProviderBinding, + BeginCloseEnvironment, CreateEnvironment, CreateEnvironmentRegistrationKey, + CreateRegisteredEnvironment, DaemonBuild, EnvironmentCloseReason, EnvironmentEndpoint, + EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, + EnvironmentProvisionRequestId, EnvironmentRegistrationKeyId, + EnvironmentRegistrationKeyStore, EnvironmentSource, EnvironmentTemplateId, FailEnvironment, + MarkRegisteredDisconnected, ObservedTarget, OpenEnvironment, ProviderCredentialChange, + PutEnvironmentProvider, PutEnvironmentProviderBinding, REGISTERED_RECONNECT_WAIT_MS, + RecordTargetObservation, RegisteredIdentityMode, RegistrationKeyPolicy, + WAIT_RETRY_AFTER_MS, mint_registration_key, }; use super::*; + fn build() -> DaemonBuild { + DaemonBuild { + version: None, + git_sha: None, + protocol_version: environment_protocol::ENVD_PROTOCOL_VERSION, + } + } + + /// A resolver over an in-memory store holding one provisioning + /// environment. async fn resolver() -> (EnvironmentResolver, EnvironmentId) { let store = Arc::new(environments::InMemoryEnvironmentRegistryStore::new()); let provider_id = EnvironmentProviderId::new("allowed"); @@ -238,11 +289,10 @@ mod tests { .put_provider(PutEnvironmentProvider { provider_id: provider_id.clone(), display_name: None, - controller_connection: EnvironmentConnectionSpec::new( - "http://controller.test", - EnvironmentTransport::Http, - ), - metadata: BTreeMap::new(), + controller_endpoint: EnvironmentEndpoint::parse("ws://controller.test/control") + .unwrap(), + data_route_base: None, + credential: ProviderCredentialChange::Keep, updated_at_ms: 10, }) .await @@ -250,188 +300,282 @@ mod tests { store .put_provider_binding(PutEnvironmentProviderBinding { universe_id: store.universe_id(), - binding_id: EnvironmentProviderBindingId::new("primary"), provider_id: provider_id.clone(), - status: EnvironmentProviderBindingStatus::Enabled, - expected_revision: None, - metadata: BTreeMap::new(), + enabled: true, updated_at_ms: 10, }) .await .expect("binding"); let environment_id = EnvironmentId::new("environment-1"); - let target_id = ProviderTargetId::new("target-1"); store .create_environment(CreateEnvironment { request_id: EnvironmentProvisionRequestId::new("request-1"), environment_id: environment_id.clone(), - incarnation_id: EnvironmentIncarnationId::new("incarnation-1"), - binding_id: EnvironmentProviderBindingId::new("primary"), + provider_id, template_id: EnvironmentTemplateId::new("test-template"), + idle: None, display_name: None, metadata: BTreeMap::new(), - - idle_policy: None, created_at_ms: 10, }) .await .expect("create environment"); + (EnvironmentResolver::new(store), environment_id) + } + + async fn open(store: &Arc, environment_id: &EnvironmentId, at_ms: i64) { store - .observe_provisioned_environment(ObserveProvisionedEnvironment { + .open_environment(OpenEnvironment { environment_id: environment_id.clone(), - provider_target_id: target_id.clone(), - status: EnvironmentStatus::Offline, - power_states: Vec::new(), - observed_at_ms: 10, + daemon_build: build(), + at_ms, }) .await - .expect("environment"); - ( - EnvironmentResolver::new(store.clone(), store), - environment_id, + .expect("open"); + } + + async fn observe( + store: &Arc, + environment_id: &EnvironmentId, + power: PowerState, + at_ms: i64, + ) { + store + .record_target_observation(RecordTargetObservation { + environment_id: environment_id.clone(), + provider_target_id: Some(ProviderTargetId::new("target-1").unwrap()), + observed: ObservedTarget { + phase: TargetPhase::Active, + power: Some(power), + message: None, + observed_at_ms: at_ms, + }, + public_endpoint: None, + observed_at_ms: at_ms, + }) + .await + .expect("observe"); + } + + /// A registered environment of a key in `mode` whose daemon + /// disconnected at `disconnected_at_ms`. + async fn disconnected_registered( + mode: RegisteredIdentityMode, + disconnected_at_ms: i64, + ) -> (EnvironmentResolver, EnvironmentId) { + let store = Arc::new(environments::InMemoryEnvironmentRegistryStore::new()); + let minted = mint_registration_key( + EnvironmentRegistrationKeyId::new("rk"), + RegistrationKeyPolicy { + display_name: "fleet".to_owned(), + identity_mode: mode, + max_active_environments: None, + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, + expires_at_ms: None, + }, + 1, ) + .unwrap(); + let secret_hash = minted.secret.secret_hash; + store + .create_registration_key(CreateEnvironmentRegistrationKey { + secret_hash: secret_hash.clone(), + record: minted.record, + }) + .await + .unwrap(); + let environment_id = EnvironmentId::new("registered-1"); + store + .create_registered_environment(CreateRegisteredEnvironment { + registration_key_id: EnvironmentRegistrationKeyId::new("rk"), + secret_hash, + environment_id: environment_id.clone(), + daemon_public_key: "ab".repeat(32), + display_name: None, + metadata: BTreeMap::new(), + daemon_build: build(), + created_at_ms: 10, + }) + .await + .unwrap(); + store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: environment_id.clone(), + at_ms: disconnected_at_ms, + }) + .await + .unwrap(); + (EnvironmentResolver::new(store), environment_id) + } + + fn refused( + status: u16, + refusal: Option, + retry_after_ms: Option, + ) -> ClientError { + ClientError::UpgradeRejected { + status, + refusal, + retry_after_ms, + } + } + + /// Counters and switches of the fake gateway route below. + #[derive(Clone, Default)] + struct Route { + connections: Arc, + initializations: Arc, + scans: Arc, + unchanged_roots: Arc, + stall_skills: Arc, + stall: Arc, + unsupported: Arc, + } + + impl Route { + fn counts(&self) -> (usize, usize, usize) { + use std::sync::atomic::Ordering::SeqCst; + ( + self.connections.load(SeqCst), + self.initializations.load(SeqCst), + self.scans.load(SeqCst), + ) + } + } + + /// Serve a gateway data route that answers with a real envd's handlers + /// over `root`, counting connections, handshakes and scans. + async fn serve_route( + root: std::path::PathBuf, + state_dir: std::path::PathBuf, + route: Route, + ) -> (String, tokio::task::JoinHandle<()>) { + use environment_protocol::{data::scan::ScanResponse, jsonrpc, paths::EnvironmentPath}; + use futures_util::{SinkExt, StreamExt}; + use std::sync::atomic::Ordering::SeqCst; + let runtime = + environment_daemon::DaemonRuntime::new(environment_daemon::config::DaemonConfig { + listen: None, + cwd: root.clone(), + fs_root: root.clone(), + state_dir, + read_only_fs: true, + registration: None, + scrubbed_env: Vec::new(), + }) + .unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let mut clients = tokio::task::JoinSet::new(); + loop { + let (socket, _) = listener.accept().await.unwrap(); + route.connections.fetch_add(1, SeqCst); + while let Some(result) = clients.try_join_next() { + result.unwrap(); + } + let route = route.clone(); + let runtime = runtime.clone(); + let root = root.clone(); + clients.spawn(async move { + let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap(); + let context = environment_daemon::dispatch::RequestContext::detached(); + while let Some(Ok(message)) = socket.next().await { + let Ok(text) = message.to_text() else { + continue; + }; + let Ok(jsonrpc::Message::Request(request)) = jsonrpc::parse(text) else { + continue; + }; + if route.stall.load(SeqCst) { + std::future::pending::<()>().await; + } + let response = if request.method == "initialize" { + route.initializations.fetch_add(1, SeqCst); + let mut initialized = runtime.initialize_response(); + initialized.home_directory = + Some(EnvironmentPath::new(root.to_str().unwrap()).unwrap()); + if route.unsupported.load(SeqCst) { + initialized.capabilities.filesystem = None; + } + jsonrpc::Response::success(request.id, &initialized) + } else { + if request.method == "fs/scan" { + route.scans.fetch_add(1, SeqCst); + let skills = request.params["filter"]["include"] + .as_array() + .is_some_and(|include| include.iter().any(|p| p == "SKILL.md")); + if skills && route.stall_skills.load(SeqCst) { + std::future::pending::<()>().await; + } + } + let result = environment_daemon::dispatch::handle_request( + &runtime, + &context, + &request.method, + request.params.clone(), + ) + .await + .map(|reply| reply.value); + if let Ok(value) = &result + && let Ok(scan) = serde_json::from_value::(value.clone()) + { + let unchanged = scan + .roots + .iter() + .filter(|root| { + matches!( + root, + environment_protocol::data::scan::ScanRootResult::Unchanged { .. } + ) + }) + .count(); + route.unchanged_roots.fetch_add(unchanged, SeqCst); + } + jsonrpc::Response::from_result(request.id, result) + }; + if socket + .send(tokio_tungstenite::tungstenite::Message::Text( + response.to_text().into(), + )) + .await + .is_err() + { + break; + } + } + }); + } + }); + (base_url, task) } #[tokio::test(flavor = "current_thread")] async fn environment_discovery_shares_connection_preserves_freshness_and_never_wakes() { - use futures_util::{SinkExt, StreamExt}; use harness::{ CoreAgentCommand, storage::{BlobStore, InMemoryBlobStore}, }; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::atomic::Ordering::SeqCst; use tools::skills::environment::{EnvironmentSkillAvailability, EnvironmentSkillCatalog}; let (resolver, environment_id) = resolver().await; let store = resolver.environments.clone(); let temp = tempfile::tempdir().unwrap(); - let root = temp.path().canonicalize().unwrap(); + let root = temp.path().canonicalize().unwrap().join("root"); std::fs::create_dir_all(root.join(".agents/skills/review")).unwrap(); let doc = "---\nname: review\ndescription: Review code.\n---\nbody"; let skill_path = root.join(".agents/skills/review/SKILL.md"); std::fs::write(&skill_path, doc).unwrap(); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let gateway = crate::environments::gateway::EnvironmentGatewayClientConfig::new( - format!("http://{}", listener.local_addr().unwrap()), - "test", - ); - let connections = Arc::new(AtomicUsize::new(0)); - let initializations = Arc::new(AtomicUsize::new(0)); - let metadata_reads = Arc::new(AtomicUsize::new(0)); - let stall_skills = Arc::new(AtomicBool::new(false)); - let scans = Arc::new(AtomicUsize::new(0)); - let unchanged = Arc::new(AtomicUsize::new(0)); - let supported = Arc::new(AtomicBool::new(true)); - let stall = Arc::new(AtomicBool::new(false)); - let task = { - let connections = connections.clone(); - let initializations = initializations.clone(); - let metadata_reads = metadata_reads.clone(); - let stall_skills = stall_skills.clone(); - let scans = scans.clone(); - let unchanged = unchanged.clone(); - let supported = supported.clone(); - let stall = stall.clone(); - let root = root.clone(); - tokio::spawn(async move { - let mut clients = tokio::task::JoinSet::new(); - loop { - let (socket, _) = listener.accept().await.unwrap(); - connections.fetch_add(1, Ordering::SeqCst); - while let Some(result) = clients.try_join_next() { - result.unwrap(); - } - let initializations = initializations.clone(); - let metadata_reads = metadata_reads.clone(); - let stall_skills = stall_skills.clone(); - let scans = scans.clone(); - let unchanged = unchanged.clone(); - let supported = supported.clone(); - let stall = stall.clone(); - let root = root.clone(); - clients.spawn(async move { - let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap(); - while let Some(Ok(message)) = socket.next().await { - let Ok(text) = message.to_text() else { - continue; - }; - let Ok(request) = serde_json::from_str::(text) else { - continue; - }; - let Some(id) = request.get("id") else { - continue; - }; - if stall.load(Ordering::SeqCst) { - std::future::pending::<()>().await; - } - let result = match request["method"].as_str().unwrap() { - "initialize" => { - initializations.fetch_add(1, Ordering::SeqCst); - serde_json::json!({ "protocolVersion": environment_protocol::shared::CURRENT_PROTOCOL_VERSION, "connectionId": "test", "capabilities": {"filesystemRead": true, "filesystemScan": supported.load(Ordering::SeqCst)}, "defaultCwd": root, "homeDirectory": root, "implementation": {"name": "test", "version": "1"} }) - } - "fs/getMetadata" => { - metadata_reads.fetch_add(1, Ordering::SeqCst); - let fs = environment_daemon::filesystem::LocalFileSystem::new( - root.clone(), - root.clone(), - false, - ); - serde_json::to_value( - fs.get_metadata( - serde_json::from_value(request["params"].clone()).unwrap(), - ) - .await - .unwrap(), - ) - .unwrap() - } - "fs/scan" => { - scans.fetch_add(1, Ordering::SeqCst); - if stall_skills.load(Ordering::SeqCst) - && request["params"]["includePatterns"].as_array().unwrap().iter().any(|p| p == "SKILL.md") { - std::future::pending::<()>().await; - } - let fs = environment_daemon::filesystem::LocalFileSystem::new( - root.clone(), - root.clone(), - false, - ); - let result = fs - .scan( - serde_json::from_value(request["params"].clone()).unwrap(), - ) - .await - .unwrap(); - if result.unchanged { - unchanged.fetch_add(1, Ordering::SeqCst); - } - serde_json::to_value(result).unwrap() - } - other => panic!("unexpected discovery RPC: {other}"), - }; - if socket - .send(tokio_tungstenite::tungstenite::Message::Text( - serde_json::json!({"jsonrpc":"2.0", "id":id, "result":result}) - .to_string() - .into(), - )) - .await - .is_err() - { - break; - } - } - }); - } - }) - }; - let counts = || { - ( - connections.load(Ordering::SeqCst), - initializations.load(Ordering::SeqCst), - metadata_reads.load(Ordering::SeqCst), - scans.load(Ordering::SeqCst), - ) - }; + let route = Route::default(); + let (base_url, task) = serve_route( + root.clone(), + temp.path().canonicalize().unwrap().join("state"), + route.clone(), + ) + .await; + let gateway = + crate::environments::gateway::EnvironmentGatewayClientConfig::new(base_url, "test"); let blobs = InMemoryBlobStore::new(); let session_id = harness::SessionId::new(uuid::Uuid::new_v4().to_string()); let feature = harness::EnvironmentsFeature { @@ -461,40 +605,43 @@ mod tests { Some(CoreAgentCommand::UpsertContext { entry, .. }) => entry, other => panic!("expected publication: {other:?}"), }; - // Offline inspection never changes desired power and never opens a data route. + let catalog_of = async |entry: &harness::ContextEntryInput| -> EnvironmentSkillCatalog { + serde_json::from_slice( + &blobs + .read_bytes(entry.provenance_ref.as_ref().unwrap()) + .await + .unwrap(), + ) + .unwrap() + }; + // Neither a provisioning nor a paused environment is dialed: + // discovery never wakes a machine and never writes the row. let before = store.read_environment(&environment_id).await.unwrap(); let unavailable = entry(refresh(None).await.unwrap()); - assert_eq!(scans.load(Ordering::SeqCst), 0); + assert_eq!(route.counts(), (0, 0, 0)); assert_eq!( store.read_environment(&environment_id).await.unwrap(), before ); assert!(refresh(Some(&unavailable)).await.unwrap().is_none()); - store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 20, - }) - .await - .unwrap(); + open(&store, &environment_id, 20).await; + observe(&store, &environment_id, PowerState::Paused, 21).await; + assert!(refresh(Some(&unavailable)).await.unwrap().is_none()); + assert_eq!(route.counts(), (0, 0, 0)); + observe(&store, &environment_id, PowerState::Running, 22).await; let available = entry(refresh(Some(&unavailable)).await.unwrap()); - let catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(available.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + let catalog = catalog_of(&available).await; assert_eq!( catalog.availability, EnvironmentSkillAvailability::Available ); assert_eq!(catalog.skills.len(), 1); + // The working directory is proven by the scan's probe root, not by + // a separate metadata call: one connection, one handshake, one scan. + assert_eq!(route.counts(), (1, 1, 1)); + // An unchanged root answers by fingerprint and reuses the cached skills. assert!(refresh(Some(&available)).await.unwrap().is_none()); - assert_eq!(unchanged.load(Ordering::SeqCst), 1); + assert_eq!(route.unchanged_roots.load(SeqCst), 1); std::fs::write(&skill_path, doc.replace("body", "edit")).unwrap(); assert!( refresh(Some(&available)).await.unwrap().is_none(), @@ -526,13 +673,10 @@ mod tests { }; for text in ["First instructions", "Updated instructions"] { std::fs::write(&prompt_path, text).unwrap(); - let before = counts(); + let before = route.counts(); let result = refresh_sources(&both, Some(&edited)).await; assert!(result.skill_command.is_none()); - assert_eq!( - counts(), - (before.0 + 1, before.1 + 1, before.2 + 1, before.3 + 2) - ); + assert_eq!(route.counts(), (before.0 + 1, before.1 + 1, before.2 + 2)); assert_eq!( blobs .read_bytes(&result.prompt_entries[&prompt_key].content.content_ref) @@ -541,7 +685,7 @@ mod tests { text.as_bytes() ); } - let before = counts(); + let before = route.counts(); let prompts_only = harness::EnvironmentsFeature { prompts: Some(Default::default()), environments: feature.environments.clone(), @@ -550,12 +694,9 @@ mod tests { let result = refresh_sources(&prompts_only, None).await; assert!(result.skill_command.is_none()); assert!(result.prompt_entries.contains_key(&prompt_key)); - assert_eq!( - counts(), - (before.0 + 1, before.1 + 1, before.2 + 1, before.3 + 1) - ); + assert_eq!(route.counts(), (before.0 + 1, before.1 + 1, before.2 + 1)); - let before = counts(); + let before = route.counts(); let disabled_feature = harness::EnvironmentsFeature::default(); let disabled = refresh_sources(&disabled_feature, None).await; assert!(disabled.skill_command.is_none()); @@ -569,30 +710,22 @@ mod tests { .is_none() ); assert_eq!( - counts(), + route.counts(), before, "disabled and controller-owned sources need no connection" ); let result = refresh_sources(&both, Some(&controller_owned)).await; assert!(result.skill_command.is_none()); assert!(result.prompt_entries.contains_key(&prompt_key)); - assert_eq!( - counts(), - (before.0 + 1, before.1 + 1, before.2 + 1, before.3 + 1) - ); + assert_eq!(route.counts(), (before.0 + 1, before.1 + 1, before.2 + 1)); - // A timed-out skill RPC must not poison the prompt scan with an unread response. - stall_skills.store(true, Ordering::SeqCst); - let before = counts(); + // A timed-out skill scan does not delay the prompt scan, which runs + // on a fresh connection. + route.stall_skills.store(true, SeqCst); + let before = route.counts(); let result = refresh_sources(&both, Some(&edited)).await; let failed_skills = entry(result.skill_command); - let failed_catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(failed_skills.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + let failed_catalog = catalog_of(&failed_skills).await; assert_eq!( failed_catalog.availability, EnvironmentSkillAvailability::Stale @@ -606,79 +739,58 @@ mod tests { .unwrap(), b"Updated instructions" ); - assert_eq!( - counts(), - (before.0 + 2, before.1 + 2, before.2 + 2, before.3 + 2) - ); - stall_skills.store(false, Ordering::SeqCst); + assert_eq!(route.counts(), (before.0 + 2, before.1 + 2, before.2 + 2)); + route.stall_skills.store(false, SeqCst); - // An incomplete scan retains the last observation without appending a menu. - std::fs::write(&skill_path, vec![b'x'; 65537]).unwrap(); - let stale = entry(refresh(Some(&edited)).await.unwrap()); - let catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(stale.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), + // One oversized skill is skipped and reported under its root; the + // root's other skills are still listed. + std::fs::create_dir_all(root.join(".agents/skills/other")).unwrap(); + std::fs::write( + root.join(".agents/skills/other/SKILL.md"), + "---\nname: other\ndescription: Other skill.\n---\nbody", ) .unwrap(); - assert_eq!(catalog.availability, EnvironmentSkillAvailability::Stale); - assert_eq!(catalog.skills.len(), 1); - assert_eq!(stale.content, edited.content); - assert!(refresh(Some(&stale)).await.unwrap().is_none()); - let result = refresh_sources(&both, Some(&edited)).await; - assert!(result.skill_command.is_some()); + std::fs::write(&skill_path, vec![b'x'; 65537]).unwrap(); + let degraded = entry(refresh(Some(&edited)).await.unwrap()); + let catalog = catalog_of(°raded).await; assert_eq!( - blobs - .read_bytes(&result.prompt_entries[&prompt_key].content.content_ref) - .await - .unwrap(), - b"Updated instructions" + catalog.availability, + EnvironmentSkillAvailability::Available ); + assert_eq!( + catalog + .skills + .iter() + .map(|skill| skill.name.as_str()) + .collect::>(), + vec!["other"] + ); + let skills_root = root.join(".agents/skills"); + assert!( + catalog + .warnings + .iter() + .any(|warning| warning.starts_with(skills_root.to_str().unwrap())), + "{:?}", + catalog.warnings + ); + std::fs::remove_dir_all(root.join(".agents/skills/other")).unwrap(); std::fs::write(&skill_path, doc).unwrap(); - std::fs::write(&prompt_path, vec![b'x'; 65537]).unwrap(); - let result = refresh_sources(&both, Some(&stale)).await; - let recovered = entry(result.skill_command); - let recovered_catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(recovered.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + + // Without the filesystem family discovery is explicitly + // unavailable, with no scan. + route.unsupported.store(true, SeqCst); + let before = route.counts().2; + let unsupported = entry(refresh(Some(°raded)).await.unwrap()); + assert_eq!(route.counts().2, before); assert_eq!( - recovered_catalog.availability, - EnvironmentSkillAvailability::Available + catalog_of(&unsupported).await.availability, + EnvironmentSkillAvailability::Stale ); - let prompt_report: tools::prompts::environment::EnvironmentPromptReport = - serde_json::from_slice( - &blobs - .read_bytes( - result.prompt_entries[&prompt_key] - .provenance_ref - .as_ref() - .unwrap(), - ) - .await - .unwrap(), - ) - .unwrap(); - assert!(!prompt_report.available); - // Missing fs/scan is explicit unavailable discovery, with no RPC fallback. - supported.store(false, Ordering::SeqCst); - let before = scans.load(Ordering::SeqCst); - assert!(refresh(Some(&stale)).await.unwrap().is_none()); - assert_eq!(scans.load(Ordering::SeqCst), before); - supported.store(true, Ordering::SeqCst); + route.unsupported.store(false, SeqCst); std::fs::remove_file(&skill_path).unwrap(); - let empty = entry(refresh(Some(&stale)).await.unwrap()); - let catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(empty.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + let empty = entry(refresh(Some(&unsupported)).await.unwrap()); + let catalog = catalog_of(&empty).await; assert!(catalog.skills.is_empty()); assert_eq!( catalog.availability, @@ -715,13 +827,7 @@ mod tests { .unwrap() .skill_command, ); - let denied_catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(denied_entry.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); + let denied_catalog = catalog_of(&denied_entry).await; assert_eq!( denied_catalog.availability, EnvironmentSkillAvailability::Unavailable @@ -730,22 +836,15 @@ mod tests { denied_catalog.skills.is_empty(), "a revoked domain grant cannot retain advertised paths" ); - stall.store(true, Ordering::SeqCst); + route.stall.store(true, SeqCst); let start = std::time::Instant::now(); let timed_out = entry(refresh(Some(&available)).await.unwrap()); assert!( start.elapsed() < std::time::Duration::from_secs(6), "unresponsive endpoint must remain bounded" ); - let timed_out_catalog: EnvironmentSkillCatalog = serde_json::from_slice( - &blobs - .read_bytes(timed_out.provenance_ref.as_ref().unwrap()) - .await - .unwrap(), - ) - .unwrap(); assert_eq!( - timed_out_catalog.availability, + catalog_of(&timed_out).await.availability, EnvironmentSkillAvailability::Stale ); assert_eq!(timed_out.content, available.content); @@ -753,291 +852,322 @@ mod tests { } #[tokio::test(flavor = "current_thread")] - async fn offline_environment_without_gateway_is_unavailable_but_readable() { + async fn an_open_environment_without_a_gateway_resolves_but_is_unreachable() { let (resolver, environment_id) = resolver().await; + open(&resolver.environments, &environment_id, 20).await; assert!(resolver.read(&environment_id).await.is_ok()); assert!( resolver .resolve_for_connection(&environment_id, 111) .await .is_ok(), - "execution resolution should defer reachability to the real connection" + "execution resolution defers reachability to the real connection" ); assert!(matches!( resolver.ready_for_use(&environment_id, 111).await, - Err(EnvironmentResolveError::EnvironmentUnavailable { .. }) + Err(EnvironmentResolveError::Unreachable { .. }) )); } #[tokio::test(flavor = "current_thread")] - async fn selection_preserves_power_and_needs_no_gateway_in_any_nonterminal_state() { + async fn readiness_reports_a_daemon_of_another_protocol_version() { + use environment_protocol::{error::ProtocolError, jsonrpc}; + use futures_util::{SinkExt, StreamExt}; let (resolver, environment_id) = resolver().await; - let store = resolver.environments.clone(); - store - .set_environment_power(SetEnvironmentPower { - environment_id: environment_id.clone(), - desired_power: PowerState::Paused, - updated_at_ms: 20, - }) - .await - .unwrap(); - for status in [ - EnvironmentStatus::Provisioning, - EnvironmentStatus::Booting, - EnvironmentStatus::Ready, - EnvironmentStatus::Paused, - EnvironmentStatus::Suspended, - EnvironmentStatus::Offline, - EnvironmentStatus::Unknown, - ] { - store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 30, - }) - .await - .unwrap(); - let before = store.read_environment(&environment_id).await.unwrap(); - for _ in 0..2 { - let selected = resolver - .selectable(&environment_id) - .await - .expect("selection requires only valid registry state"); - assert_eq!(selected, before); - assert_eq!( - store.read_environment(&environment_id).await.unwrap(), - before + open(&resolver.environments, &environment_id, 20).await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let remote = environment_protocol::ENVD_PROTOCOL_VERSION + 1; + let task = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap(); + while let Some(Ok(message)) = socket.next().await { + let Ok(text) = message.to_text() else { + continue; + }; + let Ok(jsonrpc::Message::Request(request)) = jsonrpc::parse(text) else { + continue; + }; + let response = jsonrpc::Response::failure( + Some(request.id), + ProtocolError::version_mismatch( + remote, + environment_protocol::ENVD_PROTOCOL_VERSION, + ), ); + let _ = socket + .send(tokio_tungstenite::tungstenite::Message::Text( + response.to_text().into(), + )) + .await; + } + }); + let resolver = resolver.with_gateway( + crate::environments::gateway::EnvironmentGatewayClientConfig::new(base_url, "test"), + ); + match resolver.ready_for_use(&environment_id, 30).await { + Err(EnvironmentResolveError::VersionMismatch { + local, + remote: reported, + .. + }) => { + assert_eq!(local, environment_protocol::ENVD_PROTOCOL_VERSION); + assert_eq!(reported, remote); } + other => panic!("expected a version mismatch, got {other:?}"), } + task.abort(); + } + + /// Serve one HTTP refusal per connection instead of an upgrade. + async fn serve_refusal( + status: &'static str, + body: &'static str, + retry_after: Option, + ) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + loop { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + match socket.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => request.extend_from_slice(&buffer[..read]), + } + } + let retry = retry_after + .map(|seconds| format!("Retry-After: {seconds}\r\n")) + .unwrap_or_default(); + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\n{retry}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + } + }); + base_url } #[tokio::test(flavor = "current_thread")] - async fn reselection_checks_terminal_status() { + async fn a_typed_route_refusal_maps_to_wait_or_gone_and_a_bodiless_one_is_unreachable() { let (resolver, environment_id) = resolver().await; - let store = resolver.environments.clone(); - resolver.selectable(&environment_id).await.unwrap(); + open(&resolver.environments, &environment_id, 20).await; + let with_route = |base_url: String| { + resolver.clone().with_gateway( + crate::environments::gateway::EnvironmentGatewayClientConfig::new(base_url, "test"), + ) + }; + let waking = with_route( + serve_refusal("503 Service Unavailable", r#"{"state":"waking"}"#, Some(3)).await, + ); assert!(matches!( - resolver.selectable(&EnvironmentId::new("missing")).await, - Err(EnvironmentResolveError::Store( - EnvironmentRegistryError::NotFound { .. } - )) - )); - store - .fail_environment_lifecycle(environments::FailEnvironmentLifecycle { - environment_id: environment_id.clone(), - message: "no capacity".into(), - observed_at_ms: 40, + waking.ready_for_use(&environment_id, 30).await, + Err(EnvironmentResolveError::NotReady { + reason: EnvironmentWaitReason::Waking, + retry_after_ms: 3_000, + until_ms: None, + .. }) - .await - .unwrap(); - assert!(matches!( - resolver.selectable(&environment_id).await, - Err(EnvironmentResolveError::Failed { .. }) )); - store - .begin_close_environment(environments::BeginCloseEnvironment { - environment_id: environment_id.clone(), - updated_at_ms: 50, - }) - .await - .unwrap(); + let gone = with_route(serve_refusal("410 Gone", r#"{"state":"gone"}"#, None).await); assert!(matches!( - resolver.selectable(&environment_id).await, + gone.ready_for_use(&environment_id, 30).await, Err(EnvironmentResolveError::Closed { .. }) )); + let failed = with_route( + serve_refusal( + "409 Conflict", + r#"{"state":"failed","message":"disk"}"#, + None, + ) + .await, + ); + assert!(matches!( + failed.ready_for_use(&environment_id, 30).await, + Err(EnvironmentResolveError::Failed { message, .. }) if message == "disk" + )); + let bodiless = with_route(serve_refusal("503 Service Unavailable", "", None).await); + assert!(matches!( + bodiless.ready_for_use(&environment_id, 30).await, + Err(EnvironmentResolveError::Unreachable { .. }) + )); + // Nothing the readiness dial learns is written. + let record = resolver.read(&environment_id).await.unwrap(); + assert_eq!(record.state, EnvironmentState::Open); + assert_eq!(record.updated_at_ms, 20); } #[tokio::test(flavor = "current_thread")] - async fn use_cancels_a_pending_power_down() { - let (resolver, environment_id) = resolver().await; - let store = resolver.environments.clone(); - store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Ready, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: 20, - }) - .await - .expect("observe"); - // The reaper decided to pause; the reconciler has not acted yet. - store - .set_environment_power(SetEnvironmentPower { - environment_id: environment_id.clone(), - desired_power: PowerState::Paused, - updated_at_ms: 21, + async fn a_disconnected_refusal_follows_the_identity_mode() { + let disconnected = refused(503, Some(DataRouteRefusal::Disconnected {}), Some(2_000)); + let (persistent, persistent_id) = + disconnected_registered(RegisteredIdentityMode::Persistent, 50).await; + // The row still reads connected in the window this refusal covers. + let mut record = persistent.read(&persistent_id).await.unwrap(); + if let EnvironmentSource::Registered(source) = &mut record.source { + source.connected = true; + } + assert!(matches!( + route_refusal_error(&record, &disconnected), + Some(EnvironmentResolveError::NotReady { + reason: EnvironmentWaitReason::Disconnected, + until_ms: None, + .. }) - .await - .expect("pause intent"); - - let resolved = resolver - .resolve_for_connection(&environment_id, 30) - .await - .expect("a ready environment resolves for use"); - assert_eq!(resolved.status, EnvironmentStatus::Ready); - assert_eq!(resolved.desired_power, PowerState::Running); - let stored = store.read_environment(&environment_id).await.expect("read"); - assert_eq!(stored.desired_power, PowerState::Running); - assert!(!stored.power_diverges()); - assert_eq!(stored.updated_at_ms, 30); + )); + let (ephemeral, ephemeral_id) = + disconnected_registered(RegisteredIdentityMode::Ephemeral, 50).await; + let record = ephemeral.read(&ephemeral_id).await.unwrap(); + assert!(matches!( + route_refusal_error(&record, &disconnected), + Some(EnvironmentResolveError::Away { .. }) + )); + assert!(matches!( + route_refusal_error( + &record, + &refused(410, Some(DataRouteRefusal::Away {}), None) + ), + Some(EnvironmentResolveError::Away { .. }) + )); + assert!(route_refusal_error(&record, &refused(502, None, None)).is_none()); + let failed = DataRouteRefusal::Failed { + message: DescriptiveText::new("broken").unwrap(), + }; + assert!(matches!( + route_refusal_error(&record, &refused(409, Some(failed), None)), + Some(EnvironmentResolveError::Failed { .. }) + )); } #[tokio::test(flavor = "current_thread")] - async fn powered_down_environment_with_power_control_wakes_on_use() { - let (resolver, environment_id) = resolver().await; - let store = resolver.environments.clone(); - let observe = |status: EnvironmentStatus, at: i64| { - let store = store.clone(); - let environment_id = environment_id.clone(); - async move { - store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id, - provider_target_id: ProviderTargetId::new("target-1"), - status, - power_states: vec![PowerState::Running, PowerState::Paused], - observed_at_ms: at, - }) - .await - .expect("observe"); - } - }; - observe(EnvironmentStatus::Ready, 20).await; - store - .set_environment_power(SetEnvironmentPower { - environment_id: environment_id.clone(), - desired_power: PowerState::Paused, - updated_at_ms: 21, - }) + async fn a_disconnected_persistent_daemon_is_waited_for_until_its_bound() { + let (resolver, environment_id) = + disconnected_registered(RegisteredIdentityMode::Persistent, 1_000).await; + let until = 1_000 + REGISTERED_RECONNECT_WAIT_MS; + match resolver + .resolve_for_connection(&environment_id, 2_000) .await - .expect("pause intent"); - observe(EnvironmentStatus::Paused, 22).await; - - // Using a paused environment requests a wake and reports it as - // not ready instead of probing an unreachable daemon. - assert!(matches!( - resolver.ready_for_use(&environment_id, 30).await, + { Err(EnvironmentResolveError::NotReady { - status: EnvironmentStatus::Paused, + reason: EnvironmentWaitReason::Disconnected, + retry_after_ms, + until_ms, .. - }) - )); - let woken = store.read_environment(&environment_id).await.expect("read"); - assert_eq!(woken.desired_power, PowerState::Running); - assert!(woken.power_diverges()); - // Activation admits it as intent. - let record = resolver - .selectable(&environment_id) - .await - .expect("activation admits a paused environment"); - assert_eq!(record.status, EnvironmentStatus::Paused); - - // Once the provider observed it running again the ordinary probe - // path applies (no gateway here → unavailable, not NotReady). - observe(EnvironmentStatus::Ready, 40).await; + }) => { + assert_eq!(retry_after_ms, WAIT_RETRY_AFTER_MS); + assert_eq!(until_ms, Some(until)); + } + other => panic!("expected a bounded wait, got {other:?}"), + } assert!(matches!( - resolver.ready_for_use(&environment_id, 50).await, - Err(EnvironmentResolveError::EnvironmentUnavailable { .. }) + resolver + .resolve_for_connection(&environment_id, until) + .await, + Err(EnvironmentResolveError::Away { .. }) )); - - // A stopped environment whose provider offers no power control keeps - // the old behaviour: no wake, plain unavailability. - store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-1"), - status: EnvironmentStatus::Offline, - power_states: Vec::new(), - observed_at_ms: 60, - }) - .await - .expect("observe offline without power control"); + // Selection still accepts it: the daemon may come back. + assert!(resolver.selectable(&environment_id).await.is_ok()); + let (ephemeral, ephemeral_id) = + disconnected_registered(RegisteredIdentityMode::Ephemeral, 1_000).await; assert!(matches!( - resolver.ready_for_use(&environment_id, 70).await, - Err(EnvironmentResolveError::EnvironmentUnavailable { .. }) + ephemeral.resolve_for_connection(&ephemeral_id, 1_001).await, + Err(EnvironmentResolveError::Away { .. }) )); - assert_eq!( - store - .read_environment(&environment_id) - .await - .expect("read") - .desired_power, - PowerState::Running - ); } #[tokio::test(flavor = "current_thread")] - async fn readiness_is_status_aware() { + async fn the_gate_waits_while_provisioning_and_selection_never_writes() { let (resolver, environment_id) = resolver().await; let store = resolver.environments.clone(); - let observe = |status: EnvironmentStatus| { - let store = store.clone(); - let environment_id = environment_id.clone(); - async move { - store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id, - provider_target_id: ProviderTargetId::new("target-1"), - status, - power_states: Vec::new(), - observed_at_ms: 20, - }) - .await - .expect("observe"); - } - }; - - // A provisioning/booting environment is admitted as intent without a - // probe and reported as not ready. - observe(EnvironmentStatus::Provisioning).await; assert!(matches!( resolver.ready_for_use(&environment_id, 30).await, Err(EnvironmentResolveError::NotReady { - status: EnvironmentStatus::Provisioning, + reason: EnvironmentWaitReason::Provisioning, + retry_after_ms: WAIT_RETRY_AFTER_MS, + until_ms: None, .. }) )); - observe(EnvironmentStatus::Booting).await; + for step in 0..3 { + match step { + 1 => open(&store, &environment_id, 20).await, + 2 => observe(&store, &environment_id, PowerState::Suspended, 21).await, + _ => {} + } + let before = store.read_environment(&environment_id).await.unwrap(); + for _ in 0..2 { + let selected = resolver + .selectable(&environment_id) + .await + .expect("selection requires only valid registry state"); + assert_eq!(selected, before); + resolver + .resolve_for_connection(&environment_id, 30) + .await + .ok(); + assert_eq!( + store.read_environment(&environment_id).await.unwrap(), + before + ); + } + } + // A suspended environment is usable: the dial wakes it. + assert!( + resolver + .resolve_for_connection(&environment_id, 30) + .await + .is_ok() + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn failed_and_closed_environments_are_gone() { + let (resolver, environment_id) = resolver().await; + let store = resolver.environments.clone(); assert!(matches!( - resolver.ready_for_use(&environment_id, 30).await, - Err(EnvironmentResolveError::NotReady { - status: EnvironmentStatus::Booting, - .. - }) + resolver.selectable(&EnvironmentId::new("missing")).await, + Err(EnvironmentResolveError::Store( + EnvironmentRegistryError::NotFound { .. } + )) )); - let record = resolver - .selectable(&environment_id) - .await - .expect("activation admits a booting environment"); - assert_eq!(record.status, EnvironmentStatus::Booting); - store - .fail_environment_lifecycle(environments::FailEnvironmentLifecycle { + .fail_environment(FailEnvironment { environment_id: environment_id.clone(), - message: "no capacity".to_owned(), - observed_at_ms: 40, + message: "no capacity".into(), + daemon_build: None, + at_ms: 40, }) .await - .expect("fail"); + .unwrap(); + assert!(matches!( + resolver.selectable(&environment_id).await, + Err(EnvironmentResolveError::Failed { message, .. }) if message == "no capacity" + )); assert!(matches!( resolver.ready_for_use(&environment_id, 50).await, Err(EnvironmentResolveError::Failed { message, .. }) if message == "no capacity" )); - store - .begin_close_environment(environments::BeginCloseEnvironment { + .begin_close_environment(BeginCloseEnvironment { environment_id: environment_id.clone(), - updated_at_ms: 60, + reason: EnvironmentCloseReason::Requested, + at_ms: 50, }) .await - .expect("close"); + .unwrap(); + assert!(matches!( + resolver.selectable(&environment_id).await, + Err(EnvironmentResolveError::Closed { .. }) + )); assert!(matches!( - resolver.ready_for_use(&environment_id, 70).await, + resolver.resolve_for_connection(&environment_id, 60).await, Err(EnvironmentResolveError::Closed { .. }) )); } diff --git a/crates/temporal-runtime/src/environments/runtime.rs b/crates/temporal-runtime/src/environments/runtime.rs index cd453e6de..7db09c04b 100644 --- a/crates/temporal-runtime/src/environments/runtime.rs +++ b/crates/temporal-runtime/src/environments/runtime.rs @@ -2,7 +2,6 @@ use std::collections::BTreeMap; -use environment_client::WebSocketTransport; use environments::EnvironmentRecord; use tools::{ environment::EnvironmentToolContext, environment_protocol::RemoteEnvironmentConnection, @@ -12,7 +11,7 @@ use tools::{ pub struct RuntimeEnvironment { resource: EnvironmentRecord, tool_context: EnvironmentToolContext, - remote_connection: Option>, + remote_connection: Option, } impl RuntimeEnvironment { @@ -34,10 +33,7 @@ impl RuntimeEnvironment { } } - pub fn with_remote_connection( - mut self, - connection: RemoteEnvironmentConnection, - ) -> Self { + pub fn with_remote_connection(mut self, connection: RemoteEnvironmentConnection) -> Self { self.remote_connection = Some(connection); self } @@ -85,11 +81,12 @@ impl RuntimeEnvironment { /// context for this invocation. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ActiveEnvironmentBlocker { - /// The environment exists and is provisioning or booting; the call must + /// The environment exists but cannot serve calls yet (provisioning, + /// waking, booting, or a registered daemon reconnecting); the call must /// wait rather than fail. NotReady { environment_id: String, - status: environments::EnvironmentStatus, + reason: environments::EnvironmentWaitReason, }, /// The environment cannot serve calls (failed, closed, unreachable, or /// not allowed); the call fails with this message. @@ -143,11 +140,10 @@ impl SessionEnvironmentManager { mod tests { use std::{collections::BTreeMap, sync::Arc}; - use environment_protocol::shared::ProviderTargetId; + use environment_protocol::ids::ProviderTargetId; use environments::{ - EnvironmentIncarnationId, EnvironmentIncarnationRecord, EnvironmentProviderBindingId, - EnvironmentProviderId, EnvironmentProvisionRequestId, EnvironmentSource, EnvironmentStatus, - EnvironmentTemplateId, + EnvironmentProviderId, EnvironmentProvisionRequestId, EnvironmentSource, EnvironmentState, + EnvironmentTemplateId, ProvisionedSource, }; use harness::storage::InMemoryBlobStore; use tools::fs::{FsPath, FsToolContext, InMemoryFileSystem}; @@ -155,34 +151,29 @@ mod tests { use super::*; fn resource() -> EnvironmentRecord { - let target_id = ProviderTargetId::new("target-a"); EnvironmentRecord { environment_id: harness::EnvironmentId::new("environment-a"), - request_id: EnvironmentProvisionRequestId::new("request-a"), - source: EnvironmentSource::Provisioned { + request_id: Some(EnvironmentProvisionRequestId::new("request-a")), + source: EnvironmentSource::Provisioned(ProvisionedSource { provider_id: EnvironmentProviderId::new("provider-a"), - binding_id: EnvironmentProviderBindingId::new("binding-a"), - }, + template_id: EnvironmentTemplateId::new("template-a"), + idle: None, + provider_target_id: Some(ProviderTargetId::new("target-a").unwrap()), + observed: None, + public_endpoint: None, + next_attempt_at_ms: None, + attempts: 0, + }), display_name: None, - status: EnvironmentStatus::Offline, - desired_power: environments::PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: EnvironmentIncarnationId::new("incarnation-a"), - provision_request_id: Some(EnvironmentProvisionRequestId::new("request-a")), - provider_target_id: Some(target_id.clone()), - template_id: Some(EnvironmentTemplateId::new("template-a")), - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: 1, - updated_at_ms: 1, - }, - public_ingress_enabled: false, - public_endpoint: None, metadata: BTreeMap::from([("fsRoot".to_owned(), "/sandbox".to_owned())]), - last_seen_at_ms: None, + state: EnvironmentState::Open, + state_message: None, + close_reason: None, + state_changed_at_ms: 1, + daemon_build: None, created_at_ms: 1, updated_at_ms: 1, + closed_at_ms: None, } } diff --git a/crates/temporal-runtime/src/environments/signals.rs b/crates/temporal-runtime/src/environments/signals.rs new file mode 100644 index 000000000..917cefc10 --- /dev/null +++ b/crates/temporal-runtime/src/environments/signals.rs @@ -0,0 +1,122 @@ +//! Process-local environment signals: the reconciler's work nudge and the +//! revocation of live environment routes. +//! +//! One instance per process. A process without the environment-gateway +//! role has no subscribers, so its signals reach nothing; the gateway's own +//! sweeps catch up within one sweep interval. + +use std::{ + collections::HashMap, + sync::{Mutex, PoisonError}, +}; + +use tokio::sync::{Notify, watch}; +use uuid::Uuid; + +/// Process-local environment signals. One instance per process; processes +/// without the environment-gateway role have no subscribers, and the +/// gateway's sweeps catch up within one sweep interval. +#[derive(Default)] +pub struct EnvironmentSignals { + work: Notify, + routes: Mutex>>, +} + +impl EnvironmentSignals { + /// Wake the reconciler now instead of at its next tick. A nudge with no + /// reconciler waiting is kept until one waits. + pub fn nudge_work(&self) { + self.work.notify_one(); + } + + /// Resolves at the next nudge. + pub async fn work_nudged(&self) { + self.work.notified().await; + } + + /// Subscribe a live control connection or data route to the + /// environment's revocation: the receiver changes on every revoke. + pub fn watch(&self, universe_id: Uuid, environment_id: &str) -> watch::Receiver { + let mut routes = self.routes.lock().unwrap_or_else(PoisonError::into_inner); + routes + .entry((universe_id, environment_id.to_owned())) + .or_insert_with(|| watch::channel(0).0) + .subscribe() + } + + /// End every live route of the environment; a control connection + /// re-reads its row and sends `registration/closed` if it is closed. + /// No-op without subscribers. + pub fn revoke(&self, universe_id: Uuid, environment_id: &str) { + let mut routes = self.routes.lock().unwrap_or_else(PoisonError::into_inner); + let key = (universe_id, environment_id.to_owned()); + let Some(sender) = routes.get(&key) else { + return; + }; + if sender.receiver_count() == 0 { + routes.remove(&key); + return; + } + sender.send_modify(|revision| *revision = revision.wrapping_add(1)); + } + + /// Keys with live subscribers; prunes senders without receivers. + pub fn watched(&self) -> Vec<(Uuid, String)> { + let mut routes = self.routes.lock().unwrap_or_else(PoisonError::into_inner); + routes.retain(|_, sender| sender.receiver_count() > 0); + routes.keys().cloned().collect() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[tokio::test(flavor = "current_thread")] + async fn a_nudge_before_the_wait_is_kept() { + let signals = EnvironmentSignals::default(); + signals.nudge_work(); + tokio::time::timeout(Duration::from_secs(1), signals.work_nudged()) + .await + .expect("the stored nudge wakes the waiter"); + assert!( + tokio::time::timeout(Duration::from_millis(20), signals.work_nudged()) + .await + .is_err(), + "one nudge wakes one wait" + ); + } + + #[test] + fn revoke_reaches_every_subscriber_of_the_environment_only() { + let signals = EnvironmentSignals::default(); + let universe = Uuid::new_v4(); + let first = signals.watch(universe, "environment_a"); + let second = signals.watch(universe, "environment_a"); + let other = signals.watch(universe, "environment_b"); + signals.revoke(universe, "environment_a"); + assert!(first.has_changed().unwrap()); + assert!(second.has_changed().unwrap()); + assert!(!other.has_changed().unwrap()); + // Unknown environments and other universes are no-ops. + signals.revoke(universe, "environment_missing"); + signals.revoke(Uuid::new_v4(), "environment_b"); + assert!(!other.has_changed().unwrap()); + } + + #[test] + fn watched_lists_live_subscriptions_and_prunes_dropped_ones() { + let signals = EnvironmentSignals::default(); + let universe = Uuid::new_v4(); + let kept = signals.watch(universe, "environment_a"); + drop(signals.watch(universe, "environment_b")); + assert_eq!( + signals.watched(), + vec![(universe, "environment_a".to_owned())] + ); + drop(kept); + assert!(signals.watched().is_empty()); + } +} diff --git a/crates/temporal-runtime/src/environments/skills.rs b/crates/temporal-runtime/src/environments/skills.rs index a9037fa40..69f0303ee 100644 --- a/crates/temporal-runtime/src/environments/skills.rs +++ b/crates/temporal-runtime/src/environments/skills.rs @@ -11,15 +11,11 @@ use std::{ }; use tools::skills::environment::*; -#[derive(Clone)] -struct CachedObservation { - fingerprint: String, - catalog: EnvironmentSkillCatalog, -} -// An optimization only: losing/evicting this cache forces a complete scan. The -// cached catalog is reused only after the environment confirms its fingerprint. -static OBSERVATIONS: OnceLock>> = OnceLock::new(); -fn observations() -> &'static Mutex> { +// An optimization only: losing/evicting this cache forces a full scan. A +// cached catalog contributes only the roots the environment confirms +// unchanged by their fingerprints. +static OBSERVATIONS: OnceLock>> = OnceLock::new(); +fn observations() -> &'static Mutex> { OBSERVATIONS.get_or_init(Default::default) } @@ -56,10 +52,14 @@ pub(crate) async fn refresh( .expect("serialize discovery scope"); let attempt = async { let connection = discovery.connection().await?; - let mut query = environment_skill_scan_query( + let query = environment_skill_scan_query( config, - Some(&connection.cwd), - connection.initialized.home_directory.as_deref(), + Some(connection.cwd.as_str()), + connection + .initialized + .home_directory + .as_ref() + .map(|home| home.as_str()), )?; let cache_key = serde_json::to_string(&( session_id, @@ -74,42 +74,30 @@ pub(crate) async fn refresh( .expect("observation lock") .get(&cache_key) .cloned(); - query.if_none_match = cached.as_ref().map(|c| c.fingerprint.clone()); - let scan = { + let catalog = { let _timer = PhaseTimer::new("skills_scan"); - connection - .client - .scan(&query) - .await - .map_err(|e| e.to_string())? - }; - if !scan.complete { - return Err(format!( - "incomplete environment discovery: {:?}", - scan.diagnostics - )); - } - if scan.unchanged { - return cached - .filter(|cached| scan.fingerprint.as_ref() == Some(&cached.fingerprint)) - .map(|cached| cached.catalog) - .ok_or("unexpected unchanged scan without matching observation".to_owned()); - } - let catalog = environment_skill_catalog(environment_id.as_str(), &scan)?; - if let Some(fingerprint) = scan.fingerprint { - let mut cache = observations().lock().expect("observation lock"); - if cache.len() >= 128 { - cache.clear(); + let mut cached_query = query.clone(); + if let Some(previous) = &cached { + apply_cached_fingerprints(&mut cached_query, previous); } - cache.insert( - cache_key, - CachedObservation { - fingerprint, - catalog: catalog.clone(), - }, - ); + let scan = connection.scan(cached_query).await?; + match environment_skill_catalog(environment_id.as_str(), &scan, cached.as_ref()) { + Ok(catalog) => catalog, + // A root answered unchanged against an observation the + // catalog cannot reuse: observe every root in full. + Err(_) if cached.is_some() => { + let scan = connection.scan(query).await?; + environment_skill_catalog(environment_id.as_str(), &scan, None)? + } + Err(error) => return Err(error), + } + }; + let mut cache = observations().lock().expect("observation lock"); + if cache.len() >= 128 { + cache.clear(); } - Ok(catalog) + cache.insert(cache_key, catalog.clone()); + Ok::<_, String>(catalog) }; let mut catalog = match tokio::time::timeout(Duration::from_secs(4), attempt).await { Ok(Ok(catalog)) => catalog, @@ -166,6 +154,7 @@ mod tests { catalog.availability = EnvironmentSkillAvailability::Available; catalog.discovery_scope = Some("configured roots".into()); catalog.skills.push(EnvironmentSkill { + root: "/skills".into(), skill_id: tools::skills::SkillId::new("internal-id"), name: "review".into(), description: "Review changes".into(), diff --git a/crates/temporal-runtime/src/environments/sources.rs b/crates/temporal-runtime/src/environments/sources.rs index 8de5fd1e9..1fee51af7 100644 --- a/crates/temporal-runtime/src/environments/sources.rs +++ b/crates/temporal-runtime/src/environments/sources.rs @@ -3,13 +3,13 @@ use crate::{ environments::gateway::EnvironmentGatewayClientConfig, environments::resolver::EnvironmentResolver, }; -use environment_client::{EnvironmentDataClient, JsonRpcTransport, WebSocketTransport}; +use environment_client::DataClient; use environment_protocol::{ data::{ - fs::GetMetadataParams, - handshake::{InitializeParams, InitializeResponse, InitializedParams}, + handshake::InitializeResponse, + scan::{ScanParams, ScanResponse, ScanRootResult}, }, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentPath}, + paths::EnvironmentPath, }; use harness::{ ContextEntryInput, ContextEntryKey, CoreAgentCommand, EnvironmentId, EnvironmentsFeature, @@ -21,30 +21,25 @@ use std::{ time::{Duration, Instant}, }; -pub(crate) async fn working_directory( - client: &mut EnvironmentDataClient, +/// `clientInfo.name` of discovery connections. +const SOURCE_DISCOVERY_CLIENT: &str = "lightspeed-source-discovery"; + +/// The working directory discovery resolves sources against: the +/// attachment's configured directory, else the daemon's `defaultCwd`. It is +/// proven to exist by the probe root of each discovery scan, not by a +/// separate call. +pub(crate) fn working_directory( configured: Option<&str>, - default: Option<&str>, -) -> Result { - let value = configured - .or(default) - .ok_or("environment does not advertise a default working directory")?; + default: &EnvironmentPath, +) -> Result { + let Some(value) = configured else { + return Ok(default.clone()); + }; if !value.starts_with('/') { return Err("environment working directory must be absolute".into()); } - let cwd = tools::environment::sources::absolute(std::path::Path::new("/"), value)? - .to_string_lossy() - .into_owned(); - let metadata = client - .get_metadata(&GetMetadataParams { - path: EnvironmentPath::new(&cwd).map_err(|e| e.to_string())?, - }) - .await - .map_err(|e| format!("working directory {cwd}: {e}"))?; - if !metadata.is_directory { - return Err(format!("working directory is not a directory: {cwd}")); - } - Ok(cwd) + let cwd = tools::environment::sources::absolute(std::path::Path::new("/"), value)?; + EnvironmentPath::new(cwd.to_string_lossy()).map_err(|error| error.to_string()) } pub(crate) struct Discovery<'a> { @@ -56,13 +51,37 @@ pub(crate) struct Discovery<'a> { } pub(crate) struct SourceConnection { - pub client: EnvironmentDataClient, + pub client: DataClient, pub initialized: InitializeResponse, - pub cwd: String, - // Includes universe and incarnation so cached observations cannot cross routes. + pub cwd: EnvironmentPath, + // Includes the universe so cached observations cannot cross routes. pub identity: String, } +impl SourceConnection { + /// Run a discovery scan with the working-directory probe appended, check + /// that probe, and return the response without it. `fs/scan` is an + /// observation, so discovery never resets the idle clock. + pub async fn scan(&self, mut query: ScanParams) -> Result { + tools::environment::sources::with_working_directory_probe(&mut query, &self.cwd); + let mut scan = self + .client + .scan(&query) + .await + .map_err(|error| error.to_string())?; + match scan.roots.pop() { + Some(ScanRootResult::Complete { .. } | ScanRootResult::Unchanged { .. }) => Ok(scan), + Some(ScanRootResult::Missing { .. }) => { + Err(format!("working directory {} does not exist", self.cwd)) + } + Some(ScanRootResult::Failed { error, .. }) => { + Err(format!("working directory {}: {error}", self.cwd)) + } + None => Err("discovery scan answered without its working directory probe".into()), + } + } +} + impl Discovery<'_> { pub async fn connection(&mut self) -> Result<&mut SourceConnection, String> { if self.connection.is_none() { @@ -84,8 +103,13 @@ impl Discovery<'_> { } pub fn discard_connection(&mut self) { - // A cancelled RPC can leave an unread response. Never reuse that stream. - self.connection = None; + // Abandoned scans were cancelled by the multiplexed client; a fresh + // connection keeps a failed attempt from affecting the next source. + if let Some(connection) = self.connection.take() { + tokio::spawn(async move { + let _ = connection.client.close().await; + }); + } } } @@ -123,7 +147,7 @@ pub(crate) async fn refresh( }) } .await; - if let Some(mut connection) = discovery.connection.take() { + if let Some(connection) = discovery.connection.take() { let _close = PhaseTimer::new("close"); let _ = tokio::time::timeout(Duration::from_secs(1), connection.client.close()).await; } @@ -166,54 +190,44 @@ async fn connect( let _timer = PhaseTimer::new("registry"); resolver.read(id).await.map_err(|e| e.to_string())? }; - if environment.status != environments::EnvironmentStatus::Ready - || environment.desired_power != environments::PowerState::Running + // A data route dial wakes a powered-down target, and discovery must + // never do that. A usable row whose last observation shows it powered + // down (or otherwise not ready) is skipped. The observation can be + // stale (providers power targets down on their own clock, and a dial + // wakes them without a write), so the dial also asks the gateway to + // check the provider first. + let now_ms = crate::environments::lifecycle::clock_ms(); + if environment.use_gate(now_ms) != environments::EnvironmentUse::Use + || environment.display_status() != environments::EnvironmentDisplayStatus::Ready { return Err("environment is not accessible; discovery does not wake it".into()); } - let connection = gateway.connection_for(resolver.universe_id(), &environment); + let endpoint = gateway.endpoint_for(resolver.universe_id(), &environment); let identity = - serde_json::to_string(&(resolver.universe_id(), &connection)).map_err(|e| e.to_string())?; - let mut client = { + serde_json::to_string(&(resolver.universe_id(), environment.environment_id.as_str())) + .map_err(|e| e.to_string())?; + let (client, initialized) = { let _timer = PhaseTimer::new("connect"); - EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("lightspeed-source-discovery"), + DataClient::connect_initialized( + &endpoint, + gateway.no_wake_connect_options(SOURCE_DISCOVERY_CLIENT), ) .await .map_err(|e| e.to_string())? }; - let initialized = { - let _timer = PhaseTimer::new("initialize"); - let initialized = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-source-discovery".into(), - scope: connection.scope, - resume_connection_id: None, - }) - .await - .map_err(|e| e.to_string())?; - client - .initialized(&InitializedParams {}) - .await - .map_err(|e| e.to_string())?; - initialized - }; - if initialized.protocol_version != CURRENT_PROTOCOL_VERSION - || !initialized.capabilities.filesystem_read - || !initialized.capabilities.filesystem_scan - { + if initialized.capabilities.filesystem.is_none() { + let _ = client.close().await; return Err("endpoint does not support filesystem source discovery".into()); } - let cwd = { - let _timer = PhaseTimer::new("working_directory"); - working_directory( - &mut client, - attachment.working_directory.as_deref(), - initialized.default_cwd.as_deref(), - ) - .await? + let cwd = match working_directory( + attachment.working_directory.as_deref(), + &initialized.default_cwd, + ) { + Ok(cwd) => cwd, + Err(error) => { + let _ = client.close().await; + return Err(error); + } }; Ok(SourceConnection { client, @@ -222,3 +236,22 @@ async fn connect( identity, }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_working_directory_is_configured_or_the_daemon_default() { + let default = EnvironmentPath::new("/workspace").unwrap(); + assert_eq!(working_directory(None, &default).unwrap(), default); + assert_eq!( + working_directory(Some("/srv/app/./src/.."), &default) + .unwrap() + .as_str(), + "/srv/app" + ); + assert!(working_directory(Some("relative"), &default).is_err()); + assert!(working_directory(Some("/../escape"), &default).is_err()); + } +} diff --git a/crates/temporal-runtime/src/gateway/deployment.rs b/crates/temporal-runtime/src/gateway/deployment.rs index eda0a394c..705706d18 100644 --- a/crates/temporal-runtime/src/gateway/deployment.rs +++ b/crates/temporal-runtime/src/gateway/deployment.rs @@ -18,13 +18,11 @@ use api::{ AgentApiError, AgentApiOutcome, DeploymentApiKeyCreateParams, DeploymentApiKeyCreateResponse, DeploymentApiKeyListParams, DeploymentApiKeyListResponse, DeploymentApiKeyRevokeParams, DeploymentApiKeyRevokeResponse, DeploymentApiKeyRotateParams, DeploymentApiKeyView, - DeploymentApiService, DeploymentEnvironmentAdoptParams, DeploymentEnvironmentAdoptResponse, - DeploymentEnvironmentProviderConnection, DeploymentEnvironmentProviderDeleteParams, + DeploymentApiService, DeploymentEnvironmentProviderDeleteParams, DeploymentEnvironmentProviderDeleteResponse, DeploymentEnvironmentProviderListParams, DeploymentEnvironmentProviderListResponse, DeploymentEnvironmentProviderPutParams, DeploymentEnvironmentProviderPutResponse, DeploymentEnvironmentProviderReadParams, - DeploymentEnvironmentProviderReadResponse, DeploymentEnvironmentProviderTransport, - DeploymentEnvironmentProviderView, DeploymentProviderBindingDeleteParams, + DeploymentEnvironmentProviderReadResponse, DeploymentProviderBindingDeleteParams, DeploymentProviderBindingDeleteResponse, DeploymentProviderBindingPutParams, DeploymentProviderBindingPutResponse, DeploymentUniverseCreateParams, DeploymentUniverseCreateResponse, DeploymentUniverseDeleteParams, @@ -32,13 +30,12 @@ use api::{ DeploymentUniverseReadParams, DeploymentUniverseReadResponse, DeploymentUniverseView, }; use async_trait::async_trait; -use environment_protocol::shared::EnvironmentTransport; +use auth::SecretValue; +use environment_protocol::routes::DataRouteBase; use environments::{ - AdoptEnvironment, EnvironmentConnectionSpec, EnvironmentId, EnvironmentIncarnationId, - EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, - EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderRecord, - EnvironmentProviderStore, EnvironmentProvisionRequestId, EnvironmentStore, - ListEnvironmentProviders, PutEnvironmentProvider, PutEnvironmentProviderBinding, + EnvironmentEndpoint, EnvironmentProviderBindingStore, EnvironmentProviderId, + EnvironmentProviderStore, EnvironmentRegistryError, ProviderCredentialChange, + PutEnvironmentProvider, PutEnvironmentProviderBinding, }; use harness::SessionId; use object_store::ObjectStoreExt as _; @@ -47,7 +44,13 @@ use temporal_workflow::{AgentSessionWorkflow, compose_workflow_id}; use temporalio_client::{Client, WorkflowTerminateOptions, errors::WorkflowInteractionError}; use uuid::Uuid; -use crate::universe::UniverseRuntime; +use crate::{ + environments::{ + provider_controllers::{WebSocketProviderControllerConnector, probe_controller}, + providers::environment_provider_view, + }, + universe::UniverseRuntime, +}; pub struct GatewayDeploymentApi { runtime: Arc, @@ -262,22 +265,80 @@ impl DeploymentApiService for GatewayDeploymentApi { ) -> Result, AgentApiError> { self.admitted(api::METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_PUT, async { let store = self.runtime.stores().store_for(Uuid::nil()); + let provider_id = parse_environment_provider_id(params.provider_id)?; + let controller_endpoint = EnvironmentEndpoint::parse(¶ms.controller_endpoint) + .map_err(|error| { + AgentApiError::invalid_request(format!("invalid controller endpoint: {error}")) + })?; + let data_route_base = params + .data_route_base + .map(DataRouteBase::new) + .transpose() + .map_err(|error| { + AgentApiError::invalid_request(format!("invalid data route base: {error}")) + })?; + let credential = match (params.credential, params.clear_credential) { + (Some(_), true) => { + return Err(AgentApiError::invalid_request( + "credential and clearCredential are mutually exclusive", + )); + } + (Some(credential), false) => { + let credential = SecretValue::new(credential); + environments::validate_provider_credential(&credential) + .map_err(super::service::environment_providers::map_environments_error)?; + ProviderCredentialChange::Set(credential) + } + (None, true) => ProviderCredentialChange::Clear, + (None, false) => ProviderCredentialChange::Keep, + }; + let request = PutEnvironmentProvider { + provider_id: provider_id.clone(), + display_name: params.display_name, + controller_endpoint: controller_endpoint.clone(), + data_route_base, + credential, + updated_at_ms: i64::try_from(current_time_ms()?) + .map_err(|_| AgentApiError::internal("current timestamp exceeds i64"))?, + }; + // Refuse a configuration the store would refuse (a unix:// + // controller without a data route base, a downgrading base) + // before dialing anything. + request + .clone() + .into_record(None) + .map_err(super::service::environment_providers::map_environments_error)?; + // The handshake presents the credential the provider will hold. + let presented = match &request.credential { + ProviderCredentialChange::Set(credential) => Some(credential.clone()), + ProviderCredentialChange::Clear => None, + ProviderCredentialChange::Keep => { + match store.read_provider_credential(&provider_id).await { + Ok(stored) => stored, + Err(EnvironmentRegistryError::NotFound { .. }) => None, + Err(error) => { + return Err( + super::service::environment_providers::map_environments_error( + error, + ), + ); + } + } + } + }; + probe_controller( + &WebSocketProviderControllerConnector, + &controller_endpoint, + presented.as_ref(), + ) + .await?; let provider = store - .put_provider(PutEnvironmentProvider { - provider_id: parse_environment_provider_id(params.provider_id)?, - display_name: params.display_name, - controller_connection: provider_connection_from_api( - params.controller_connection, - ), - metadata: params.metadata, - updated_at_ms: i64::try_from(current_time_ms()?) - .map_err(|_| AgentApiError::internal("current timestamp exceeds i64"))?, - }) + .put_provider(request) .await .map_err(super::service::environment_providers::map_environments_error)?; Ok(AgentApiOutcome::new( DeploymentEnvironmentProviderPutResponse { - provider: environment_provider_view(provider), + provider: environment_provider_view(&provider), }, )) }) @@ -291,15 +352,12 @@ impl DeploymentApiService for GatewayDeploymentApi { self.admitted(api::METHOD_DEPLOYMENT_ENVIRONMENT_PROVIDERS_LIST, async { let store = self.runtime.stores().store_for(Uuid::nil()); let providers = store - .list_providers(ListEnvironmentProviders::default()) + .list_providers() .await .map_err(super::service::environment_providers::map_environments_error)?; Ok(AgentApiOutcome::new( DeploymentEnvironmentProviderListResponse { - providers: providers - .into_iter() - .map(environment_provider_view) - .collect(), + providers: providers.iter().map(environment_provider_view).collect(), }, )) }) @@ -319,7 +377,7 @@ impl DeploymentApiService for GatewayDeploymentApi { .map_err(super::service::environment_providers::map_environments_error)?; Ok(AgentApiOutcome::new( DeploymentEnvironmentProviderReadResponse { - provider: environment_provider_view(provider), + provider: environment_provider_view(&provider), }, )) }) @@ -339,7 +397,7 @@ impl DeploymentApiService for GatewayDeploymentApi { .map_err(super::service::environment_providers::map_environments_error)?; Ok(AgentApiOutcome::new( DeploymentEnvironmentProviderDeleteResponse { - provider: environment_provider_view(provider), + provider: environment_provider_view(&provider), }, )) }) @@ -357,26 +415,8 @@ impl DeploymentApiService for GatewayDeploymentApi { let binding = store .put_provider_binding(PutEnvironmentProviderBinding { universe_id, - binding_id: EnvironmentProviderBindingId::try_new(params.binding_id).map_err( - |error| { - AgentApiError::invalid_request(format!("invalid binding id: {error}")) - }, - )?, - provider_id: EnvironmentProviderId::try_new(params.provider_id).map_err( - |error| { - AgentApiError::invalid_request(format!("invalid provider id: {error}")) - }, - )?, - status: match params.status { - api::EnvironmentProviderBindingStatusView::Enabled => { - EnvironmentProviderBindingStatus::Enabled - } - api::EnvironmentProviderBindingStatusView::Disabled => { - EnvironmentProviderBindingStatus::Disabled - } - }, - metadata: params.metadata, - expected_revision: params.expected_revision, + provider_id: parse_environment_provider_id(params.provider_id)?, + enabled: params.enabled, updated_at_ms: i64::try_from(current_time_ms()?) .map_err(|_| AgentApiError::internal("current timestamp exceeds i64"))?, }) @@ -398,13 +438,10 @@ impl DeploymentApiService for GatewayDeploymentApi { self.admitted(api::METHOD_DEPLOYMENT_PROVIDER_BINDINGS_DELETE, async { let universe_id = parse_universe_id(¶ms.universe_id)?; self.require_universe(universe_id).await?; - let binding_id = - EnvironmentProviderBindingId::try_new(params.binding_id).map_err(|error| { - AgentApiError::invalid_request(format!("invalid binding id: {error}")) - })?; + let provider_id = parse_environment_provider_id(params.provider_id)?; let store = self.runtime.stores().store_for(universe_id); let binding = store - .delete_provider_binding(universe_id, &binding_id) + .delete_provider_binding(universe_id, &provider_id) .await .map_err(super::service::environment_providers::map_environments_error)?; Ok(AgentApiOutcome::new( @@ -419,56 +456,6 @@ impl DeploymentApiService for GatewayDeploymentApi { .await } - async fn adopt_environment( - &self, - params: DeploymentEnvironmentAdoptParams, - ) -> Result, AgentApiError> { - self.admitted(api::METHOD_DEPLOYMENT_ENVIRONMENTS_ADOPT, async { - if !params.take_ownership { - return Err(AgentApiError::invalid_request( - "takeOwnership must be true because adoption transfers lifecycle ownership to Lightspeed", - )); - } - let universe_id = parse_universe_id(¶ms.universe_id)?; - self.require_universe(universe_id).await?; - let request_id = - EnvironmentProvisionRequestId::try_new(params.request_id).map_err(|error| { - AgentApiError::invalid_request(format!("invalid environment request id: {error}")) - })?; - let binding_id = - EnvironmentProviderBindingId::try_new(params.binding_id).map_err(|error| { - AgentApiError::invalid_request(format!("invalid binding id: {error}")) - })?; - validate_adoption_source(¶ms.source_target)?; - let store = self.runtime.stores().store_for(universe_id); - let environment = EnvironmentStore::adopt_environment( - store.as_ref(), - AdoptEnvironment { - request_id, - environment_id: EnvironmentId::new(format!( - "environment_{}", - Uuid::new_v4().simple() - )), - incarnation_id: EnvironmentIncarnationId::new(format!( - "incarnation_{}", - Uuid::new_v4().simple() - )), - binding_id, - source_target: params.source_target, - display_name: params.display_name, - metadata: params.metadata, - created_at_ms: i64::try_from(current_time_ms()?) - .map_err(|_| AgentApiError::internal("current timestamp exceeds i64"))?, - }, - ) - .await - .map_err(super::service::environment_providers::map_environments_error)?; - Ok(AgentApiOutcome::new(DeploymentEnvironmentAdoptResponse { - environment: super::service::environment_providers::environment_view(&environment), - })) - }).await - } - async fn put_universe_slug( &self, params: api::DeploymentUniverseSlugPutParams, @@ -770,63 +757,11 @@ fn parse_universe_id(value: &str) -> Result { .map_err(|error| AgentApiError::invalid_request(format!("invalid universe id: {error}"))) } -fn validate_adoption_source(value: &str) -> Result<(), AgentApiError> { - if value.is_empty() || value.len() > 255 || value.chars().any(char::is_control) { - return Err(AgentApiError::invalid_request( - "sourceTarget must be a non-empty provider-native reference of at most 255 bytes", - )); - } - Ok(()) -} - fn parse_environment_provider_id(value: String) -> Result { EnvironmentProviderId::try_new(value) .map_err(|error| AgentApiError::invalid_request(format!("invalid provider id: {error}"))) } -fn provider_connection_from_api( - connection: DeploymentEnvironmentProviderConnection, -) -> EnvironmentConnectionSpec { - EnvironmentConnectionSpec { - endpoint: connection.endpoint, - transport: match connection.transport { - DeploymentEnvironmentProviderTransport::WebSocket => EnvironmentTransport::WebSocket, - DeploymentEnvironmentProviderTransport::Http => EnvironmentTransport::Http, - DeploymentEnvironmentProviderTransport::Stdio => EnvironmentTransport::Stdio, - DeploymentEnvironmentProviderTransport::Ssh => EnvironmentTransport::Ssh, - DeploymentEnvironmentProviderTransport::Provider { provider_type } => { - EnvironmentTransport::Provider { provider_type } - } - }, - } -} - -fn environment_provider_view( - record: EnvironmentProviderRecord, -) -> DeploymentEnvironmentProviderView { - DeploymentEnvironmentProviderView { - provider_id: record.provider_id.to_string(), - display_name: record.display_name, - controller_connection: DeploymentEnvironmentProviderConnection { - endpoint: record.controller_connection.endpoint, - transport: match record.controller_connection.transport { - EnvironmentTransport::WebSocket => { - DeploymentEnvironmentProviderTransport::WebSocket - } - EnvironmentTransport::Http => DeploymentEnvironmentProviderTransport::Http, - EnvironmentTransport::Stdio => DeploymentEnvironmentProviderTransport::Stdio, - EnvironmentTransport::Ssh => DeploymentEnvironmentProviderTransport::Ssh, - EnvironmentTransport::Provider { provider_type } => { - DeploymentEnvironmentProviderTransport::Provider { provider_type } - } - }, - }, - metadata: record.metadata, - created_at_ms: record.created_at_ms, - updated_at_ms: record.updated_at_ms, - } -} - fn universe_view(stats: store_pg::UniverseStats) -> DeploymentUniverseView { DeploymentUniverseView { universe_id: stats.universe_id.to_string(), diff --git a/crates/temporal-runtime/src/gateway/http.rs b/crates/temporal-runtime/src/gateway/http.rs index 839cf94f0..75902b53f 100644 --- a/crates/temporal-runtime/src/gateway/http.rs +++ b/crates/temporal-runtime/src/gateway/http.rs @@ -18,23 +18,46 @@ use axum::{ routing::{get, post}, }; use environment_protocol::{ - gateway::{PROVIDER_DATA_PATH_PREFIX, ROUTE_PATH_PREFIX}, - registration::{CONNECT_PATH, DATA_PATH}, + control::target::{GetTargetParams, PowerState, TargetOwner, TargetPhase, TargetRef}, + ids::{EnvironmentId, UniverseId}, + limits::{MAX_MESSAGE_BYTES, RETRY_AFTER_MAX_SECS}, + metadata::DescriptiveText, + routes::{ + CONNECT_PATH, CanonicalOrigin, DATA_PATH, DataRouteRefusal, canonical_origin, + parse_retry_after_ms, provider_data_route, + }, +}; +use environments::{ + EndpointScheme, EnvironmentEndpoint, EnvironmentGoneReason, EnvironmentUse, + EnvironmentWaitReason, WAIT_RETRY_AFTER_MS, }; use futures_util::{SinkExt as _, StreamExt as _}; use serde::Deserialize; use store_pg::PgApiKeyStore; -use tokio_tungstenite::{connect_async, tungstenite::Message as ProviderMessage}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::watch, +}; +use tokio_tungstenite::{ + WebSocketStream, client_async_with_config, connect_async_with_config, + tungstenite::{ + Message as UpstreamMessage, client::IntoClientRequest as _, + http::HeaderValue as UpstreamHeaderValue, protocol::WebSocketConfig, + }, +}; use uuid::Uuid; use crate::{ - config::{DeploymentStores, GatewayAuthMode, gateway_auth_mode_from_env}, - environments::gateway::{RouteKey, bearer_matches, close_message}, + config::GatewayAuthMode, + environments::{ + gateway::{NO_WAKE_HEADER, ROUTE_PATH_PREFIX, RouteKey, bearer_matches, close_message}, + signals::EnvironmentSignals, + }, universe::{UniverseError, UniverseRuntime}, }; use super::{ - GatewayAgentApi, GatewayDeploymentApi, OAuthCallbackOutcome, connect_temporal, + GatewayAgentApi, GatewayDeploymentApi, OAuthCallbackOutcome, registration::{self, RegisteredConnections}, request_context, }; @@ -44,18 +67,6 @@ pub const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 64 * 1024 * 1024; pub use super::authentication::{ACTOR_HEADER, UNIVERSE_HEADER}; -#[derive(Clone, Debug)] -pub struct GatewayServerConfig { - pub bind: SocketAddr, - pub task_queue: String, - pub temporal_target: String, - pub namespace: String, - pub max_request_body_bytes: usize, - /// Externally reachable base URL for the OAuth callback - /// (`{base}/auth/callback`). Defaults to `http://{bind}`. - pub public_base_url: Option, -} - /// Per-request universe resolution for the HTTP edge. /// /// The JSON-RPC API itself never carries a universe parameter: session-scoped @@ -80,8 +91,13 @@ pub struct GatewayState { resolution: UniverseResolution, /// Live registered-daemon connections on this replica. registrations: Arc, - /// Public base URL daemons are told to dial for data connections. - public_base_url: String, + /// The origins a daemon's registration signature may name: the + /// canonical origin of the public base URL and any configured + /// registration URLs. Empty when none of them has a WebSocket origin; + /// the connect route then refuses every daemon. + registration_origins: Vec, + /// Revocation of live environment routes and control connections. + signals: Arc, } /// Which route families one HTTP listener serves, derived from the @@ -106,29 +122,35 @@ impl GatewayRoutes { impl GatewayState { /// Route every request to one existing service instance. pub fn for_api(api: Arc) -> Self { - let public_base_url = api.public_base_url().to_owned(); + let registration_origins = registration_origin_of(api.public_base_url()) + .into_iter() + .collect(); + let signals = api.environment_signals().clone(); Self { resolution: UniverseResolution::FixedApi { api }, registrations: Arc::new(RegisteredConnections::new()), - public_base_url, + registration_origins, + signals, } } + /// Replace the origins registration signatures are verified against, + /// as [`registration_origins`] computed them at startup. + pub fn with_registration_origins(mut self, origins: Vec) -> Self { + self.registration_origins = origins; + self + } + pub(super) fn registrations(&self) -> &Arc { &self.registrations } - /// Tell daemons to dial a different public base than the general one. - pub fn with_environment_public_url(mut self, url: Option) -> Self { - if let Some(url) = url { - self.public_base_url = url; - } - self + pub(super) fn registration_origins(&self) -> &[CanonicalOrigin] { + &self.registration_origins } - /// Public data route for reverse-dialed daemon sockets. - pub(super) fn registration_data_url(&self) -> String { - registration::data_url(&self.public_base_url) + pub(super) fn signals(&self) -> &Arc { + &self.signals } /// The universe a register request belongs to: the environment's @@ -148,7 +170,7 @@ impl GatewayState { let universe_id = store.config().universe_id; let known = environments::EnvironmentStore::read_environment_by_daemon_public_key( store.as_ref(), - ¶ms.daemon_public_key, + ¶ms.daemon_public_key.to_hex(), ) .await .map_err(|error| AgentApiError::internal(error.to_string()))?; @@ -168,10 +190,12 @@ impl GatewayState { } UniverseResolution::Multi { runtime, .. } => { let pool = runtime.stores().pool(); - if let Some(universe_id) = - store_pg::find_registered_environment_universe(pool, ¶ms.daemon_public_key) - .await - .map_err(|error| AgentApiError::internal(error.to_string()))? + if let Some(universe_id) = store_pg::find_registered_environment_universe( + pool, + ¶ms.daemon_public_key.to_hex(), + ) + .await + .map_err(|error| AgentApiError::internal(error.to_string()))? { return Ok(Some(universe_id)); } @@ -192,16 +216,21 @@ impl GatewayState { ) -> Self { let api_keys = PgApiKeyStore::new(runtime.stores().pool().clone()); let deployment = Arc::new(GatewayDeploymentApi::new(runtime.clone())); + let registration_origins = registration_origin_of(&public_base_url) + .into_iter() + .collect(); + let signals = runtime.environment_signals().clone(); Self { resolution: UniverseResolution::Multi { mode, runtime, - public_base_url: public_base_url.clone(), + public_base_url, api_keys, deployment, }, registrations: Arc::new(RegisteredConnections::new()), - public_base_url, + registration_origins, + signals, } } @@ -321,20 +350,103 @@ impl GatewayState { } } + fn deployment_store(&self) -> Arc { + match &self.resolution { + UniverseResolution::FixedApi { api } => api.store().clone(), + UniverseResolution::Multi { runtime, .. } => runtime.stores().store_for(Uuid::nil()), + } + } + async fn environment_provider( &self, provider_id: &environments::EnvironmentProviderId, - ) -> Result { - let store = match &self.resolution { - UniverseResolution::FixedApi { api } => api.store().clone(), - UniverseResolution::Multi { runtime, .. } => runtime.stores().store_for(Uuid::nil()), - }; - environments::EnvironmentProviderStore::read_provider(store.as_ref(), provider_id) - .await - .map_err(|error| AgentApiError::rejected(error.to_string())) + ) -> Result + { + environments::EnvironmentProviderStore::read_provider( + self.deployment_store().as_ref(), + provider_id, + ) + .await + } + + async fn environment_provider_credential( + &self, + provider_id: &environments::EnvironmentProviderId, + ) -> Result, environments::EnvironmentRegistryError> { + environments::EnvironmentProviderStore::read_provider_credential( + self.deployment_store().as_ref(), + provider_id, + ) + .await } } +/// The registration signature origin of a public base URL, if it has one. +fn registration_origin_of(public_base_url: &str) -> Option { + match canonical_origin(public_base_url) { + Ok(origin) => Some(origin), + Err(error) => { + tracing::error!( + target: "temporal_runtime", + %error, + public_base_url, + "the public base URL has no WebSocket origin; daemon registration is refused" + ); + None + } + } +} + +/// The origins registered daemons may sign for, computed once at startup by +/// a process that serves the registration route: the canonical origin of +/// `LIGHTSPEED_PUBLIC_BASE_URL`, then the origin of each +/// `LIGHTSPEED_ENVIRONMENT_REGISTRATION_URLS` entry (for example a +/// cluster-internal Service name daemons dial instead). Without a public +/// base URL the default `http://{bind}` is a true origin only on a loopback +/// bind; any other bind needs at least one configured origin, or the +/// process refuses to start rather than reject every remote daemon. A URL +/// without a WebSocket origin is a startup error. +pub fn registration_origins( + public_base_url: Option<&str>, + registration_urls: &[String], + bind: SocketAddr, +) -> anyhow::Result> { + let mut origins = Vec::new(); + match public_base_url { + Some(public_base_url) => { + origins.push(canonical_origin(public_base_url).map_err(|error| { + anyhow::anyhow!("LIGHTSPEED_PUBLIC_BASE_URL has no WebSocket origin: {error}") + })?) + } + None if bind.ip().is_loopback() => { + origins.push(canonical_origin(&format!("http://{bind}"))?); + } + None => {} + } + for url in registration_urls + .iter() + .map(|url| url.trim()) + .filter(|url| !url.is_empty()) + { + let origin = canonical_origin(url).map_err(|error| { + anyhow::anyhow!( + "LIGHTSPEED_ENVIRONMENT_REGISTRATION_URLS entry {url:?} has no WebSocket origin: {error}" + ) + })?; + if !origins.contains(&origin) { + origins.push(origin); + } + } + if origins.is_empty() { + anyhow::bail!( + "LIGHTSPEED_PUBLIC_BASE_URL or LIGHTSPEED_ENVIRONMENT_REGISTRATION_URLS is required to serve \ + environment registration on {bind}: daemons sign the gateway origin they dial, and only \ + those URLs name it" + ); + } + Ok(origins) +} + fn map_universe_error(error: UniverseError) -> AgentApiError { match error { UniverseError::Unknown { universe_id } => { @@ -344,30 +456,6 @@ fn map_universe_error(error: UniverseError) -> AgentApiError { } } -pub async fn serve_gateway(config: GatewayServerConfig) -> anyhow::Result<()> { - let mode = gateway_auth_mode_from_env()?; - let client = connect_temporal(&config.temporal_target, &config.namespace).await?; - let stores = DeploymentStores::from_env() - .await? - .with_blob_cache(crate::config::blob_cache_from_env()?); - let public_base_url = public_base_url_or_default(&config); - let runtime = Arc::new(UniverseRuntime::new( - client, - config.task_queue.clone(), - Some(public_base_url.clone()), - stores, - )?); - prewarm_single_universe(&mode, &runtime).await?; - let reconciler = tokio::spawn(runtime.clone().run_environment_reconciler()); - let state = Arc::new(GatewayState::multi(mode, runtime, public_base_url)); - let app = gateway_router(state, config.max_request_body_bytes, GatewayRoutes::ALL); - let listener = tokio::net::TcpListener::bind(config.bind).await?; - tracing::info!(target: "temporal_runtime", bind = %config.bind, "gateway listening"); - axum::serve(listener, app).await?; - reconciler.abort(); - Ok(()) -} - /// In `single` mode, build the pinned universe's state at startup so /// misconfiguration fails the process instead of the first request. This also /// preserves the legacy behavior of creating the configured universe row. @@ -381,13 +469,6 @@ pub async fn prewarm_single_universe( Ok(()) } -pub fn public_base_url_or_default(config: &GatewayServerConfig) -> String { - config - .public_base_url - .clone() - .unwrap_or_else(|| format!("http://{}", config.bind)) -} - pub fn gateway_router( state: Arc, max_request_body_bytes: usize, @@ -407,7 +488,7 @@ pub fn gateway_router( if routes.environment { router = router .route( - &format!("{ROUTE_PATH_PREFIX}/{{universe}}/{{environment}}/{{incarnation}}"), + &format!("{ROUTE_PATH_PREFIX}/{{universe}}/{{environment}}"), get(environment_route_upgrade), ) .route(CONNECT_PATH, get(registration::connect_upgrade)) @@ -498,9 +579,14 @@ async fn bot_webhook_ingest( (status, Json(payload)).into_response() } +/// `GET {ROUTE_PATH_PREFIX}/{universe}/{environment}`: a worker's data +/// route. The row's use gate decides first; a usable environment is then +/// dialed by its source. Every refusal the gate or a provider makes is a +/// typed [`DataRouteRefusal`]; a bodiless status means the gateway itself +/// could not reach or authorize the upstream. async fn environment_route_upgrade( State(state): State>, - Path((universe, environment, incarnation)): Path<(String, String, String)>, + Path((universe, environment)): Path<(String, String)>, headers: HeaderMap, upgrade: WebSocketUpgrade, ) -> Response { @@ -513,7 +599,6 @@ async fn environment_route_upgrade( let key = RouteKey { universe_id, environment_id: environment, - incarnation_id: incarnation, }; let api = match state.api_for_daemon(universe_id).await { Ok(api) => api, @@ -523,6 +608,9 @@ async fn environment_route_upgrade( Ok(id) => id, Err(_) => return StatusCode::BAD_REQUEST.into_response(), }; + // Subscribe before reading the row: a revocation that lands after the + // read still ends the route. + let revoked = state.signals().watch(universe_id, &key.environment_id); let environment = match environments::EnvironmentStore::read_environment( api.store().as_ref(), &environment_id, @@ -530,114 +618,72 @@ async fn environment_route_upgrade( .await { Ok(environment) => environment, - Err(_) => return StatusCode::NOT_FOUND.into_response(), + Err(environments::EnvironmentRegistryError::NotFound { .. }) => { + return refusal_response(&DataRouteRefusal::Gone {}, None); + } + Err(error) => { + tracing::warn!(target: "temporal_runtime", %error, environment = %key.environment_id, "environment route could not read its environment"); + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } }; - if environment.incarnation.incarnation_id.as_str() != key.incarnation_id { - return StatusCode::CONFLICT.into_response(); + if let Some(refused) = gate_refusal(&environment.use_gate(now_ms())) { + return refused; } + let upgrade = upgrade + .max_message_size(MAX_MESSAGE_BYTES as usize) + .max_frame_size(MAX_MESSAGE_BYTES as usize); match &environment.source { - environments::EnvironmentSource::External { connection } => { - if connection.transport != environment_protocol::shared::EnvironmentTransport::WebSocket - { - return StatusCode::BAD_GATEWAY.into_response(); - } - let endpoint = daemon_data_endpoint(&connection.endpoint); - let daemon_socket = match connect_async(&endpoint).await { - Ok((socket, _)) => socket, - Err(error) => { - tracing::warn!(target: "temporal_runtime", %error, %endpoint, "external environment connection failed"); - return StatusCode::SERVICE_UNAVAILABLE.into_response(); - } - }; + environments::EnvironmentSource::External { endpoint } => { + let upstream = + match dial_upstream(endpoint.as_str(), external_target(endpoint), None).await { + Ok(upstream) => upstream, + Err(failure) => return failure.respond(endpoint.as_str()), + }; upgrade - .max_message_size(64 * 1024 * 1024) - .on_upgrade(move |socket| { - proxy_external_route(state, key, endpoint, socket, daemon_socket) - }) + .on_upgrade(move |socket| proxy_upstream(revoked, socket, upstream)) .into_response() } - environments::EnvironmentSource::Provisioned { - provider_id, - binding_id, - } => { - let Some(target_id) = environment.incarnation.provider_target_id.as_ref() else { - return StatusCode::CONFLICT.into_response(); + environments::EnvironmentSource::Provisioned(source) => { + let (endpoint, credential) = match provider_route(&state, &api, &key, source).await { + Ok(route) => route, + Err(refused) => return *refused, }; - if authorize_provisioned_route( - &state, - &key, - provider_id, - binding_id, - target_id.as_str(), + if headers.contains_key(NO_WAKE_HEADER) + && let Some(refused) = refuse_unless_running(&api, &key, source).await + { + return refused; + } + let upstream = match dial_upstream( + &endpoint, + UpstreamTarget::Url, + credential.as_ref().map(|credential| credential.expose()), ) .await - .is_err() { - return StatusCode::CONFLICT.into_response(); - } - let provider = match state.environment_provider(provider_id).await { - Ok(provider) => provider, - Err(_) => return StatusCode::BAD_GATEWAY.into_response(), + Ok(upstream) => upstream, + Err(failure) => return failure.respond(&endpoint), }; - let endpoint = match provider_data_endpoint( - &provider.controller_connection.endpoint, - &key, - binding_id.as_str(), - target_id.as_str(), - ) { - Ok(endpoint) => endpoint, - Err(_) => return StatusCode::BAD_GATEWAY.into_response(), - }; - let provider_socket = match connect_async(&endpoint).await { - Ok((socket, _)) => socket, - Err(error) => { - tracing::warn!(target: "temporal_runtime", %error, %endpoint, "provider data connection failed"); - return StatusCode::SERVICE_UNAVAILABLE.into_response(); - } - }; - let provider_id = provider_id.clone(); - let binding_id = binding_id.clone(); - let target_id = target_id.to_string(); upgrade - .max_message_size(64 * 1024 * 1024) - .on_upgrade(move |socket| { - proxy_provider_route( - state, - key, - provider_id, - binding_id, - target_id, - socket, - provider_socket, - ) - }) + .on_upgrade(move |socket| proxy_upstream(revoked, socket, upstream)) .into_response() } - environments::EnvironmentSource::Registered { .. } => { - if matches!( - environment.status, - environments::EnvironmentStatus::Closing | environments::EnvironmentStatus::Closed - ) { - return StatusCode::CONFLICT.into_response(); - } - let (daemon_socket, control_connection_id) = match registration::open_registered_route( + environments::EnvironmentSource::Registered(_) => { + let (daemon_socket, control_ended) = match registration::open_registered_route( &state, &key, ) .await { Ok(paired) => paired, - Err(status) => { - tracing::warn!(target: "temporal_runtime", environment = %key.environment_id, %status, "registered environment data route unavailable"); - return status.into_response(); + Err(refused) => { + tracing::warn!(target: "temporal_runtime", environment = %key.environment_id, status = %refused.status(), "registered environment data route unavailable"); + return *refused; } }; upgrade - .max_message_size(64 * 1024 * 1024) .on_upgrade(move |socket| { registration::proxy_registered_route( - state, - key, - control_connection_id, + revoked, + control_ended, socket, daemon_socket, ) @@ -647,238 +693,395 @@ async fn environment_route_upgrade( } } -fn provider_data_endpoint( - controller_endpoint: &str, - key: &RouteKey, - binding_id: &str, - target_id: &str, -) -> anyhow::Result { - let mut endpoint = url::Url::parse(controller_endpoint)?; - let scheme = match endpoint.scheme() { - "http" => "ws", - "https" => "wss", - "ws" => "ws", - "wss" => "wss", - other => anyhow::bail!("unsupported provider controller URL scheme: {other}"), +/// The refusal a worker route answers for a gate that does not admit use, +/// or `None` to dial. Waking and booting never come from the gate itself: +/// only a provider answers them. +fn gate_refusal(gate: &EnvironmentUse) -> Option { + let refusal = match gate { + EnvironmentUse::Use => return None, + EnvironmentUse::Wait { + reason, + retry_after_ms, + .. + } => { + let refusal = match reason { + EnvironmentWaitReason::Provisioning => DataRouteRefusal::Provisioning {}, + EnvironmentWaitReason::Disconnected => DataRouteRefusal::Disconnected {}, + EnvironmentWaitReason::Waking => DataRouteRefusal::Waking {}, + EnvironmentWaitReason::Booting => DataRouteRefusal::Booting {}, + }; + return Some(refusal_response(&refusal, Some(*retry_after_ms))); + } + EnvironmentUse::Gone { reason, message } => match reason { + EnvironmentGoneReason::Closing | EnvironmentGoneReason::Closed => { + DataRouteRefusal::Gone {} + } + EnvironmentGoneReason::Away => DataRouteRefusal::Away {}, + EnvironmentGoneReason::Failed => DataRouteRefusal::Failed { + message: message + .as_deref() + .and_then(DescriptiveText::lossy) + .unwrap_or_else(|| { + DescriptiveText::new("the environment failed") + .expect("a fixed failure message is valid text") + }), + }, + }, }; - endpoint - .set_scheme(scheme) - .map_err(|_| anyhow::anyhow!("invalid provider controller URL scheme"))?; - endpoint.set_query(None); - endpoint.set_fragment(None); - let base_path = endpoint - .path() - .trim_end_matches('/') - .strip_suffix("/control") - .ok_or_else(|| anyhow::anyhow!("provider controller endpoint must end in /control"))?; - endpoint.set_path(&format!("{base_path}{PROVIDER_DATA_PATH_PREFIX}")); - endpoint - .path_segments_mut() - .map_err(|_| anyhow::anyhow!("provider controller URL cannot be a base URL"))? - .pop_if_empty() - .push(&key.universe_id.to_string()) - .push(binding_id) - .push(&key.environment_id) - .push(&key.incarnation_id) - .push(target_id); - Ok(endpoint.into()) + Some(refusal_response(&refusal, None)) } -fn daemon_data_endpoint(endpoint: &str) -> String { - if let Some(rest) = endpoint.strip_prefix("http://") { - format!("ws://{rest}") - } else if let Some(rest) = endpoint.strip_prefix("https://") { - format!("wss://{rest}") - } else { - endpoint.to_owned() +/// A typed data-route refusal: its status, a JSON body tagged by `state`, +/// and for a retryable one `Retry-After` in whole seconds, rounded up and +/// kept within 1 to [`RETRY_AFTER_MAX_SECS`] (default +/// [`WAIT_RETRY_AFTER_MS`]). +pub(super) fn refusal_response( + refusal: &DataRouteRefusal, + retry_after_ms: Option, +) -> Response { + let status = StatusCode::from_u16(refusal.status()).unwrap_or(StatusCode::SERVICE_UNAVAILABLE); + let mut response = (status, Json(refusal)).into_response(); + if refusal.is_retryable() { + response.headers_mut().insert( + header::RETRY_AFTER, + HeaderValue::from(retry_after_secs(retry_after_ms)), + ); } + response +} + +fn retry_after_secs(retry_after_ms: Option) -> u64 { + retry_after_ms + .unwrap_or(WAIT_RETRY_AFTER_MS) + .div_ceil(1_000) + .clamp(1, RETRY_AFTER_MAX_SECS) } -async fn authorize_external_route( +/// How long the gateway waits for a provider route or external envd to +/// accept the upstream WebSocket; longer than a provider may hold a dial +/// while it wakes the target (`MAX_DATA_ROUTE_HOLD_MS`). +const UPSTREAM_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const _: () = assert!( + UPSTREAM_CONNECT_TIMEOUT.as_millis() + > environment_protocol::limits::MAX_DATA_ROUTE_HOLD_MS as u128 +); + +/// The provider data route of a provisioned environment and the credential +/// it is dialed with (none when the provider has none). The universe's +/// binding of the provider must exist and be enabled; a target the +/// provider has not named yet is still provisioning. +async fn provider_route( state: &GatewayState, + api: &GatewayAgentApi, key: &RouteKey, - endpoint: &str, -) -> anyhow::Result<()> { - let api = state - .api_for_daemon(key.universe_id) - .await - .map_err(|error| anyhow::anyhow!(error.message))?; - let environment_id = environments::EnvironmentId::try_new(key.environment_id.clone())?; - let environment = - environments::EnvironmentStore::read_environment(api.store().as_ref(), &environment_id) - .await?; - let environments::EnvironmentSource::External { - connection: assigned_connection, - } = &environment.source - else { - anyhow::bail!("external route no longer belongs to an external environment") + source: &environments::ProvisionedSource, +) -> Result<(String, Option), Box> { + let provider_id = &source.provider_id; + let unavailable = |error: environments::EnvironmentRegistryError| match error { + environments::EnvironmentRegistryError::Store { .. } => { + tracing::warn!(target: "temporal_runtime", %error, provider = %provider_id, "provider record unavailable"); + Box::new(StatusCode::SERVICE_UNAVAILABLE.into_response()) + } + error => { + tracing::error!(target: "temporal_runtime", %error, provider = %provider_id, "provider data route is misconfigured"); + Box::new(StatusCode::BAD_GATEWAY.into_response()) + } }; - if assigned_connection.transport - != environment_protocol::shared::EnvironmentTransport::WebSocket - || daemon_data_endpoint(&assigned_connection.endpoint) != endpoint - || environment.incarnation.incarnation_id.as_str() != key.incarnation_id - || matches!( - environment.status, - environments::EnvironmentStatus::Closing | environments::EnvironmentStatus::Closed - ) + match environments::EnvironmentProviderBindingStore::read_provider_binding( + api.store().as_ref(), + key.universe_id, + provider_id, + ) + .await { - anyhow::bail!("external environment route is stale or no longer authorized") + Ok(binding) if binding.enabled => {} + Ok(_) | Err(environments::EnvironmentRegistryError::NotFound { .. }) => { + return Err(Box::new(StatusCode::FORBIDDEN.into_response())); + } + Err(error) => return Err(unavailable(error)), } - Ok(()) + let Some(target_id) = source.provider_target_id.as_ref() else { + return Err(Box::new(refusal_response( + &DataRouteRefusal::Provisioning {}, + None, + ))); + }; + let provider = state + .environment_provider(provider_id) + .await + .map_err(unavailable)?; + let base = provider.effective_data_route_base().map_err(unavailable)?; + let credential = state + .environment_provider_credential(provider_id) + .await + .map_err(unavailable)?; + let owner = target_owner(key).map_err(|error| { + tracing::error!(target: "temporal_runtime", %error, environment = %key.environment_id, "route identity is not a valid target owner"); + Box::new(StatusCode::BAD_REQUEST.into_response()) + })?; + Ok((provider_data_route(&base, &owner, target_id), credential)) } -async fn authorize_provisioned_route( - state: &GatewayState, +/// Bound of the provider check a no-wake dial makes. +const NO_WAKE_CHECK_TIMEOUT: Duration = Duration::from_secs(5); + +/// For a dial that must not wake its target ([`NO_WAKE_HEADER`]): `None` +/// when the provider reports the target active and running right now (or +/// active without power control), else the refusal. Nothing is dialed or +/// written: a powered-down target is a bodiless 412, and a provider that +/// cannot answer in time a bodiless 503. +async fn refuse_unless_running( + api: &GatewayAgentApi, key: &RouteKey, - provider_id: &environments::EnvironmentProviderId, - binding_id: &environments::EnvironmentProviderBindingId, - target_id: &str, -) -> anyhow::Result<()> { - let api = state - .api_for_daemon(key.universe_id) - .await - .map_err(|error| anyhow::anyhow!(error.message))?; - let environment_id = environments::EnvironmentId::try_new(key.environment_id.clone())?; - let environment = - environments::EnvironmentStore::read_environment(api.store().as_ref(), &environment_id) - .await?; - let environments::EnvironmentSource::Provisioned { - provider_id: assigned_provider, - binding_id: assigned_binding, - } = &environment.source + source: &environments::ProvisionedSource, +) -> Option { + let (Some(target_id), Ok(owner)) = (source.provider_target_id.clone(), target_owner(key)) else { - anyhow::bail!("provider data routes require a provisioned environment") + return Some(StatusCode::SERVICE_UNAVAILABLE.into_response()); }; - if assigned_provider != provider_id - || assigned_binding != binding_id - || environment.incarnation.incarnation_id.as_str() != key.incarnation_id - || environment - .incarnation - .provider_target_id - .as_ref() - .map(|id| id.as_str()) - != Some(target_id) - || matches!( - environment.status, - environments::EnvironmentStatus::Closing | environments::EnvironmentStatus::Closed - ) - { - anyhow::bail!("provider data route is stale or no longer owned") + let service = api.environment_service(); + let observed = tokio::time::timeout(NO_WAKE_CHECK_TIMEOUT, async { + let controller = service + .connect_provider_controller(&source.provider_id) + .await + .map_err(|error| error.to_string())?; + let answer = controller + .client + .get_target(&GetTargetParams { + target: TargetRef { + owner, + target_id: Some(target_id), + }, + }) + .await + .map_err(|error| error.to_string()); + controller.close().await; + answer.map(|response| response.target) + }) + .await; + match observed { + Ok(Ok(target)) + if target.phase == TargetPhase::Active + && target + .power + .is_none_or(|power| power == PowerState::Running) => + { + None + } + Ok(Ok(_)) => Some(StatusCode::PRECONDITION_FAILED.into_response()), + Ok(Err(error)) => { + tracing::debug!(target: "temporal_runtime", %error, environment = %key.environment_id, "a no-wake dial could not check its target's power"); + Some(StatusCode::SERVICE_UNAVAILABLE.into_response()) + } + Err(_) => Some(StatusCode::SERVICE_UNAVAILABLE.into_response()), } - let binding = environments::EnvironmentProviderBindingStore::read_provider_binding( - api.store().as_ref(), - key.universe_id, - binding_id, - ) - .await?; - if binding.status != environments::EnvironmentProviderBindingStatus::Enabled - || binding.provider_id != *provider_id - { - anyhow::bail!("provider binding is disabled or changed") +} + +fn target_owner(key: &RouteKey) -> Result { + Ok(TargetOwner { + universe_id: UniverseId::new(key.universe_id.to_string())?, + environment_id: EnvironmentId::new(key.environment_id.as_str())?, + }) +} + +/// Where an upstream socket is dialed. +#[derive(Clone, Copy)] +enum UpstreamTarget<'a> { + /// `ws://` or `wss://`, dialed as written. + Url, + /// A Unix socket path; the upgrade targets `/` with `Host: localhost`. + Unix(&'a str), +} + +fn external_target(endpoint: &EnvironmentEndpoint) -> UpstreamTarget<'_> { + match (endpoint.scheme(), endpoint.unix_socket_path()) { + (EndpointScheme::Unix, Some(path)) => UpstreamTarget::Unix(path), + _ => UpstreamTarget::Url, } - Ok(()) } -async fn proxy_provider_route( - state: Arc, - key: RouteKey, - provider_id: environments::EnvironmentProviderId, - binding_id: environments::EnvironmentProviderBindingId, - target_id: String, - mut worker: WebSocket, - provider: tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream, - >, -) { - let (mut provider_writer, mut provider_reader) = provider.split(); - let mut reauthorize = tokio::time::interval(Duration::from_secs(5)); - loop { - tokio::select! { - _ = reauthorize.tick() => { - if authorize_provisioned_route( - &state, &key, &provider_id, &binding_id, &target_id, - ).await.is_err() { - let _ = worker.send(close_message("provider route authorization changed")).await; - break; - } +enum Upstream { + Tcp(Box>>), + Unix(Box>), +} + +/// Why an upstream dial failed, and what the worker is told. +#[derive(Debug)] +enum UpstreamFailure { + /// The upstream answered a typed refusal: forwarded unchanged. + Refusal { + refusal: DataRouteRefusal, + retry_after_ms: Option, + }, + /// Unreachable, slow or answering 5xx/429 without a refusal body: the + /// worker may retry (bodiless 503). + Unavailable(String), + /// The upstream refused the upgrade for good (wrong credential, no such + /// route) or the URL is unusable (bodiless 502). + Refused(String), +} + +impl UpstreamFailure { + fn respond(self, endpoint: &str) -> Response { + match self { + Self::Refusal { + refusal, + retry_after_ms, + } => { + tracing::info!(target: "temporal_runtime", %refusal, %endpoint, "environment upstream refused the dial"); + refusal_response(&refusal, retry_after_ms) } - message = worker.recv() => { - let Some(Ok(message)) = message else { break }; - let close = matches!(message, Message::Close(_)); - let message = match message { - Message::Text(value) => ProviderMessage::Text(value.to_string().into()), - Message::Binary(value) => ProviderMessage::Binary(value), - Message::Ping(value) => ProviderMessage::Ping(value), - Message::Pong(value) => ProviderMessage::Pong(value), - Message::Close(_) => ProviderMessage::Close(None), - }; - if provider_writer.send(message).await.is_err() || close { break } + Self::Unavailable(error) => { + tracing::warn!(target: "temporal_runtime", %error, %endpoint, "environment upstream unavailable"); + StatusCode::SERVICE_UNAVAILABLE.into_response() } - message = provider_reader.next() => { - let Some(Ok(message)) = message else { break }; - let close = matches!(message, ProviderMessage::Close(_)); - let message = match message { - ProviderMessage::Text(value) => Some(Message::Text(value.to_string().into())), - ProviderMessage::Binary(value) => Some(Message::Binary(value)), - ProviderMessage::Ping(value) => Some(Message::Ping(value)), - ProviderMessage::Pong(value) => Some(Message::Pong(value)), - ProviderMessage::Close(_) => Some(Message::Close(None)), - ProviderMessage::Frame(_) => None, + Self::Refused(error) => { + tracing::error!(target: "temporal_runtime", %error, %endpoint, "environment upstream refused the connection"); + StatusCode::BAD_GATEWAY.into_response() + } + } + } +} + +fn upstream_config() -> WebSocketConfig { + WebSocketConfig::default() + .max_message_size(Some(MAX_MESSAGE_BYTES as usize)) + .max_frame_size(Some(MAX_MESSAGE_BYTES as usize)) +} + +/// Dial an upstream data socket with the protocol's message limits, an +/// optional bearer credential and a connect bound. +async fn dial_upstream( + endpoint: &str, + target: UpstreamTarget<'_>, + bearer: Option<&str>, +) -> Result { + let url = match target { + UpstreamTarget::Url => endpoint, + UpstreamTarget::Unix(_) => "ws://localhost/", + }; + let mut request = url + .into_client_request() + .map_err(|error| UpstreamFailure::Refused(error.to_string()))?; + if let Some(bearer) = bearer { + let mut value = + UpstreamHeaderValue::from_str(&format!("Bearer {bearer}")).map_err(|_| { + UpstreamFailure::Refused("the credential is not a valid header value".to_owned()) + })?; + value.set_sensitive(true); + request.headers_mut().insert( + tokio_tungstenite::tungstenite::http::header::AUTHORIZATION, + value, + ); + } + let dialed = tokio::time::timeout(UPSTREAM_CONNECT_TIMEOUT, async { + match target { + UpstreamTarget::Url => { + connect_async_with_config(request, Some(upstream_config()), true) + .await + .map(|(socket, _)| Upstream::Tcp(Box::new(socket))) + .map_err(upstream_failure) + } + UpstreamTarget::Unix(path) => { + let stream = tokio::net::UnixStream::connect(path) + .await + .map_err(|error| UpstreamFailure::Unavailable(error.to_string()))?; + client_async_with_config(request, stream, Some(upstream_config())) + .await + .map(|(socket, _)| Upstream::Unix(Box::new(socket))) + .map_err(upstream_failure) + } + } + }) + .await; + dialed.unwrap_or_else(|_| { + Err(UpstreamFailure::Unavailable(format!( + "no upgrade within {} s", + UPSTREAM_CONNECT_TIMEOUT.as_secs() + ))) + }) +} + +fn upstream_failure(error: tokio_tungstenite::tungstenite::Error) -> UpstreamFailure { + use tokio_tungstenite::tungstenite::Error; + match error { + Error::Http(response) => { + let status = response.status(); + let retry_after_ms = response + .headers() + .get(tokio_tungstenite::tungstenite::http::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(parse_retry_after_ms); + let refusal = DataRouteRefusal::from_refused_upgrade( + status.as_u16(), + response.body().as_deref(), + retry_after_ms, + ); + if let Some(refusal) = refusal { + return UpstreamFailure::Refusal { + refusal, + retry_after_ms, }; - if let Some(message) = message - && worker.send(message).await.is_err() - { break } - if close { break } + } + let message = format!("upgrade refused with HTTP {status}"); + if status.is_server_error() || status.as_u16() == 429 { + UpstreamFailure::Unavailable(message) + } else { + UpstreamFailure::Refused(message) } } + Error::Url(error) => UpstreamFailure::Refused(error.to_string()), + error => UpstreamFailure::Unavailable(error.to_string()), } - // Whichever leg ended first, complete the WebSocket closing handshake on - // both sides instead of dropping the opposite TCP stream abruptly. - let _ = provider_writer.send(ProviderMessage::Close(None)).await; - let _ = worker.send(Message::Close(None)).await; } -async fn proxy_external_route( - state: Arc, - key: RouteKey, - endpoint: String, +async fn proxy_upstream(revoked: watch::Receiver, worker: WebSocket, upstream: Upstream) { + match upstream { + Upstream::Tcp(socket) => proxy_frames(revoked, worker, *socket).await, + Upstream::Unix(socket) => proxy_frames(revoked, worker, *socket).await, + } +} + +/// Pipe one worker route and its upstream frame for frame, pings and pongs +/// included, until either side ends or the environment's routes are +/// revoked (close code 1008). +async fn proxy_frames( + mut revoked: watch::Receiver, mut worker: WebSocket, - daemon: tokio_tungstenite::WebSocketStream< - tokio_tungstenite::MaybeTlsStream, - >, -) { - let (mut daemon_writer, mut daemon_reader) = daemon.split(); - let mut reauthorize = tokio::time::interval(Duration::from_secs(5)); + upstream: WebSocketStream, +) where + S: AsyncRead + AsyncWrite + Unpin, +{ + let (mut upstream_writer, mut upstream_reader) = upstream.split(); loop { tokio::select! { - _ = reauthorize.tick() => { - if authorize_external_route(&state, &key, &endpoint).await.is_err() { - let _ = worker.send(close_message("external route authorization changed")).await; - break; - } + // A dropped revocation sender fails closed like a revocation. + _ = revoked.changed() => { + let _ = worker.send(close_message("environment route revoked")).await; + break; } message = worker.recv() => { let Some(Ok(message)) = message else { break }; let close = matches!(message, Message::Close(_)); let message = match message { - Message::Text(value) => ProviderMessage::Text(value.to_string().into()), - Message::Binary(value) => ProviderMessage::Binary(value), - Message::Ping(value) => ProviderMessage::Ping(value), - Message::Pong(value) => ProviderMessage::Pong(value), - Message::Close(_) => ProviderMessage::Close(None), + Message::Text(value) => UpstreamMessage::Text(value.to_string().into()), + Message::Binary(value) => UpstreamMessage::Binary(value), + Message::Ping(value) => UpstreamMessage::Ping(value), + Message::Pong(value) => UpstreamMessage::Pong(value), + Message::Close(_) => UpstreamMessage::Close(None), }; - if daemon_writer.send(message).await.is_err() || close { break } + if upstream_writer.send(message).await.is_err() || close { break } } - message = daemon_reader.next() => { + message = upstream_reader.next() => { let Some(Ok(message)) = message else { break }; - let close = matches!(message, ProviderMessage::Close(_)); + let close = matches!(message, UpstreamMessage::Close(_)); let message = match message { - ProviderMessage::Text(value) => Some(Message::Text(value.to_string().into())), - ProviderMessage::Binary(value) => Some(Message::Binary(value)), - ProviderMessage::Ping(value) => Some(Message::Ping(value)), - ProviderMessage::Pong(value) => Some(Message::Pong(value)), - ProviderMessage::Close(_) => Some(Message::Close(None)), - ProviderMessage::Frame(_) => None, + UpstreamMessage::Text(value) => Some(Message::Text(value.to_string().into())), + UpstreamMessage::Binary(value) => Some(Message::Binary(value)), + UpstreamMessage::Ping(value) => Some(Message::Ping(value)), + UpstreamMessage::Pong(value) => Some(Message::Pong(value)), + UpstreamMessage::Close(_) => Some(Message::Close(None)), + UpstreamMessage::Frame(_) => None, }; if let Some(message) = message && worker.send(message).await.is_err() @@ -887,12 +1090,20 @@ async fn proxy_external_route( } } } - // A worker activity may be cancelled or its client may disappear without - // a close frame. Do not propagate that abrupt teardown to envd. - let _ = daemon_writer.send(ProviderMessage::Close(None)).await; + // Whichever leg ended first, complete the WebSocket closing handshake on + // both sides instead of dropping the opposite stream abruptly; a worker + // that vanished without a close frame never tears envd down abruptly. + let _ = upstream_writer.send(UpstreamMessage::Close(None)).await; let _ = worker.send(Message::Close(None)).await; } +fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + /// Client ID Metadata Document (draft-ietf-oauth-client-id-metadata-document): /// authorization servers fetch this to resolve Lightspeed's CIMD client id. /// Deployment-scoped: it depends only on the public base URL, not a universe. @@ -1105,6 +1316,7 @@ fn html_escape(value: &str) -> String { mod tests { use super::*; use api::AgentApiErrorKind; + use environment_protocol::ids::ProviderTargetId; #[test] fn json_rpc_responses_disable_caching() { @@ -1151,32 +1363,263 @@ mod tests { } #[test] - fn provider_data_url_is_derived_and_path_segments_are_escaped() { + fn provider_data_routes_name_the_universe_environment_and_target() { let key = RouteKey { universe_id: Uuid::parse_str("6f3a1a52-58c1-4f0e-9c2d-1a2b3c4d5e6f").unwrap(), - environment_id: "environment with space".to_owned(), - incarnation_id: "incarnation/one".to_owned(), + environment_id: "environment_1".to_owned(), }; - let endpoint = provider_data_endpoint( - "https://provider.example/control", - &key, - "primary", - "target-one", - ) - .unwrap(); + let owner = target_owner(&key).unwrap(); + let base = + environment_protocol::routes::DataRouteBase::new("wss://provider.example/routes") + .unwrap(); assert_eq!( - endpoint, - "wss://provider.example/routes/6f3a1a52-58c1-4f0e-9c2d-1a2b3c4d5e6f/primary/environment%20with%20space/incarnation%2Fone/target-one" + provider_data_route( + &base, + &owner, + &ProviderTargetId::new("default/pod one").unwrap() + ), + "wss://provider.example/routes/6f3a1a52-58c1-4f0e-9c2d-1a2b3c4d5e6f/environment_1/default%2Fpod%20one" ); - let nested = provider_data_endpoint( - "https://provider.example/services/incus/control", - &key, - "primary", - "target-one", - ) - .unwrap(); - assert!(nested.starts_with("wss://provider.example/services/incus/routes/")); - assert!(provider_data_endpoint("wss://provider.example/custom", &key, "b", "t").is_err()); + } + + fn rejected( + status: u16, + body: Option<&str>, + retry_after: Option<&str>, + ) -> tokio_tungstenite::tungstenite::Error { + let mut response = tokio_tungstenite::tungstenite::http::Response::builder().status(status); + if let Some(retry_after) = retry_after { + response = response.header("retry-after", retry_after); + } + tokio_tungstenite::tungstenite::Error::Http(Box::new( + response + .body(body.map(|body| body.as_bytes().to_vec())) + .unwrap(), + )) + } + + #[test] + fn upstream_refusals_split_into_typed_retryable_and_permanent() { + assert!(matches!( + upstream_failure(rejected(401, None, None)), + UpstreamFailure::Refused(_) + )); + assert!(matches!( + upstream_failure(rejected(404, Some("not json"), None)), + UpstreamFailure::Refused(_) + )); + assert!(matches!( + upstream_failure(rejected(503, None, None)), + UpstreamFailure::Unavailable(_) + )); + assert!(matches!( + upstream_failure(rejected(429, None, None)), + UpstreamFailure::Unavailable(_) + )); + let UpstreamFailure::Refusal { + refusal, + retry_after_ms, + } = upstream_failure(rejected(503, Some(r#"{"state":"waking"}"#), Some("3"))) + else { + panic!("a typed provider refusal is forwarded"); + }; + assert_eq!(refusal, DataRouteRefusal::Waking {}); + assert_eq!(retry_after_ms, Some(3_000)); + let UpstreamFailure::Refusal { + refusal, + retry_after_ms, + } = upstream_failure(rejected( + 409, + Some(r#"{"state":"failed","message":"disk lost"}"#), + None, + )) + else { + panic!("a failed target is forwarded"); + }; + assert_eq!(refusal.status(), 409); + assert_eq!(retry_after_ms, None); + let UpstreamFailure::Refusal { retry_after_ms, .. } = + upstream_failure(rejected(503, Some(r#"{"state":"booting"}"#), Some("3600"))) + else { + panic!("a booting target is forwarded"); + }; + assert_eq!(retry_after_ms, Some(RETRY_AFTER_MAX_SECS * 1_000)); + } + + #[test] + fn a_refusal_whose_body_was_not_read_is_forwarded_by_its_status() { + let forwarded = |status: u16, retry_after: Option<&str>| match upstream_failure(rejected( + status, + None, + retry_after, + )) { + UpstreamFailure::Refusal { + refusal, + retry_after_ms, + } => Some((refusal, retry_after_ms)), + _ => None, + }; + assert_eq!( + forwarded(410, None), + Some((DataRouteRefusal::Gone {}, None)) + ); + assert!(matches!( + forwarded(409, None), + Some((DataRouteRefusal::Failed { .. }, None)) + )); + assert_eq!( + forwarded(503, Some("2")), + Some((DataRouteRefusal::Booting {}, Some(2_000))) + ); + // A 503 without `Retry-After` names no wait: a bodiless 503. + assert_eq!(forwarded(503, None), None); + } + + fn body_of(response: Response) -> DataRouteRefusal { + let bytes = + futures::executor::block_on(axum::body::to_bytes(response.into_body(), usize::MAX)) + .unwrap(); + serde_json::from_slice(&bytes).unwrap() + } + + #[test] + fn refusals_carry_their_status_body_and_retry_hint() { + let waking = refusal_response(&DataRouteRefusal::Waking {}, Some(2_500)); + assert_eq!(waking.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + waking.headers().get(header::RETRY_AFTER), + Some(&HeaderValue::from(3_u64)) + ); + assert_eq!( + waking.headers().get(header::CONTENT_TYPE), + Some(&HeaderValue::from_static("application/json")) + ); + assert_eq!(body_of(waking), DataRouteRefusal::Waking {}); + + let defaulted = refusal_response(&DataRouteRefusal::Disconnected {}, None); + assert_eq!( + defaulted.headers().get(header::RETRY_AFTER), + Some(&HeaderValue::from(WAIT_RETRY_AFTER_MS / 1_000)) + ); + assert_eq!(retry_after_secs(Some(0)), 1); + assert_eq!( + retry_after_secs(Some(10 * 60 * 1_000)), + RETRY_AFTER_MAX_SECS + ); + + let gone = refusal_response(&DataRouteRefusal::Gone {}, Some(5_000)); + assert_eq!(gone.status(), StatusCode::GONE); + assert!( + gone.headers().get(header::RETRY_AFTER).is_none(), + "only retryable refusals carry Retry-After" + ); + } + + #[test] + fn the_use_gate_maps_to_route_refusals() { + assert!(gate_refusal(&EnvironmentUse::Use).is_none()); + let wait = |reason| EnvironmentUse::Wait { + reason, + retry_after_ms: 2_000, + until_ms: None, + }; + let gone = |reason, message: Option<&str>| EnvironmentUse::Gone { + reason, + message: message.map(str::to_owned), + }; + for (gate, status, expected) in [ + ( + wait(EnvironmentWaitReason::Provisioning), + StatusCode::SERVICE_UNAVAILABLE, + DataRouteRefusal::Provisioning {}, + ), + ( + wait(EnvironmentWaitReason::Disconnected), + StatusCode::SERVICE_UNAVAILABLE, + DataRouteRefusal::Disconnected {}, + ), + ( + gone(EnvironmentGoneReason::Closing, None), + StatusCode::GONE, + DataRouteRefusal::Gone {}, + ), + ( + gone(EnvironmentGoneReason::Closed, None), + StatusCode::GONE, + DataRouteRefusal::Gone {}, + ), + ( + gone(EnvironmentGoneReason::Away, Some("daemon away since 1")), + StatusCode::GONE, + DataRouteRefusal::Away {}, + ), + ( + gone(EnvironmentGoneReason::Failed, Some("boot disk missing")), + StatusCode::CONFLICT, + DataRouteRefusal::Failed { + message: DescriptiveText::new("boot disk missing").unwrap(), + }, + ), + ] { + let response = gate_refusal(&gate).expect("the gate refuses"); + assert_eq!(response.status(), status, "{gate:?}"); + assert_eq!(body_of(response), expected, "{gate:?}"); + } + } + + #[test] + fn registration_origins_come_from_the_public_base_url_and_configured_urls() { + let loopback: SocketAddr = "127.0.0.1:18080".parse().unwrap(); + let public: SocketAddr = "0.0.0.0:18080".parse().unwrap(); + let origins = |base: Option<&str>, urls: &[&str], bind| { + registration_origins( + base, + &urls.iter().map(|url| (*url).to_owned()).collect::>(), + bind, + ) + .map(|origins| { + origins + .iter() + .map(|origin| origin.as_str().to_owned()) + .collect::>() + }) + }; + assert_eq!( + origins(None, &[], loopback).unwrap(), + ["ws://127.0.0.1:18080"] + ); + assert!( + origins(None, &[], public).is_err(), + "a non-loopback bind needs a configured origin" + ); + assert_eq!( + origins(Some("https://Lightspeed.example/base/"), &[], public).unwrap(), + ["wss://lightspeed.example"] + ); + assert!(origins(Some("lightspeed.example"), &[], public).is_err()); + assert_eq!( + origins( + Some("https://lightspeed.example"), + &[ + " ws://lightspeed-gateway.lightspeed.svc:18080/environment-gateway/connect ", + "", + "https://lightspeed.example/other", + ], + public, + ) + .unwrap(), + [ + "wss://lightspeed.example", + "ws://lightspeed-gateway.lightspeed.svc:18080" + ], + "configured URLs are reduced to origins, trimmed and deduplicated" + ); + assert_eq!( + origins(None, &["ws://gateway.internal:18080"], public).unwrap(), + ["ws://gateway.internal:18080"], + "a configured URL alone serves a non-loopback bind" + ); + assert!(origins(None, &["gateway.internal"], public).is_err()); } #[test] diff --git a/crates/temporal-runtime/src/gateway/mod.rs b/crates/temporal-runtime/src/gateway/mod.rs index 29d320d88..1927f489f 100644 --- a/crates/temporal-runtime/src/gateway/mod.rs +++ b/crates/temporal-runtime/src/gateway/mod.rs @@ -12,12 +12,11 @@ pub use crate::config::pg_store_from_env; pub use deployment::GatewayDeploymentApi; pub use http::{ ACTOR_HEADER, DEFAULT_GATEWAY_BIND, DEFAULT_MAX_REQUEST_BODY_BYTES, GatewayRoutes, - GatewayServerConfig, GatewayState, UNIVERSE_HEADER, gateway_router, prewarm_single_universe, - public_base_url_or_default, serve_gateway, + GatewayState, UNIVERSE_HEADER, gateway_router, prewarm_single_universe, }; pub use service::{ DEFAULT_PUBLIC_BASE_URL, GatewayAgentApi, GatewayAgentApiBuilder, OAuthCallbackOutcome, - PowerReaperStats, ReconcileFailureLog, + ReconcileFailureLog, }; pub use temporal_workflow::{ AgentAdmission, AgentAdmissionFailure, AgentAdmissionFailureKind, AgentCompletedRunSummary, diff --git a/crates/temporal-runtime/src/gateway/registration.rs b/crates/temporal-runtime/src/gateway/registration.rs index a1c24bdd9..26c992cee 100644 --- a/crates/temporal-runtime/src/gateway/registration.rs +++ b/crates/temporal-runtime/src/gateway/registration.rs @@ -1,20 +1,33 @@ //! Outbound `envd` registration on the environment gateway. //! -//! One public route accepts control connections: the daemon proves its -//! identity against a nonce, is admitted by a registration key the first +//! One public route accepts control connections: the registration channel +//! of `environment_protocol::registration`, on the shared JSON-RPC +//! envelope with the daemon as the client. The gateway sends the frozen +//! `registration/challenge` first; the daemon proves its identity in +//! `registration/register`, is admitted by a registration key the first //! time and by its identity afterwards, and then holds the socket while the -//! gateway pings it and asks it to open data connections. A second public -//! route accepts those reverse-dialed data sockets and pairs each with the -//! worker route that requested it; the pair is then proxied frame for frame -//! exactly like a passive external environment. +//! gateway pings it and asks it, with `registration/openStream`, to dial +//! data streams. A second public route accepts those reverse-dialed stream +//! sockets and pairs each with the worker route that requested it; the +//! pair is then proxied frame for frame exactly like a passive external +//! environment. The gateway never names a URL: the daemon derives the +//! stream URL from its own connect URL. //! -//! The gateway holds no durable lifecycle authority here. Connect, -//! heartbeat, and disconnect are written as observations on the environment -//! row; the lifecycle reconciler derives ephemeral cleanup and stale -//! repair from them. +//! Liveness is written on transitions: admission and reconnect mark the +//! row connected, the end of the current connection marks it disconnected, +//! and while connected a coarse `last_seen_at_ms` stamp is written at most +//! every [`REGISTERED_LIVENESS_STAMP_MS`]. The registered sweep +//! ([`run_registered_sweep`]) flips rows whose stamp went stale, closes +//! ephemeral environments after their key's disconnect grace and +//! persistent ones after its `offlineCloseAfterMs`, and ends the routes of +//! environments closed elsewhere. The one close the session itself makes +//! is a departure: an ephemeral environment whose daemon said +//! `registration/goodbye` closes at once. A close in this process revokes +//! the environment's routes through [`EnvironmentSignals`]; its control +//! connection then re-reads its row and sends `registration/closed`. use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, sync::{Arc, Mutex}, time::{Duration, Instant}, }; @@ -27,66 +40,124 @@ use axum::{ http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, }; -use ed25519_dalek::{Signature, Verifier as _, VerifyingKey}; +use ed25519_dalek::{Signature, VerifyingKey}; use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + bytes::SecretString, + error::{BusyReason, NotFoundReason, PermissionReason, PreconditionReason, ProtocolError}, + ids::{ControlConnectionId, DaemonId, EnvironmentId}, + jsonrpc::{self, Id, Notification, Request}, + limits::{HEARTBEAT_MISSED_INTERVALS, MAX_MESSAGE_BYTES}, + manifest::{ + RegistrationChallenge, RegistrationClosed, RegistrationGoodbye, RegistrationOpenStream, + RegistrationRegister, + }, + metadata::DescriptiveText, registration::{ - DATA_PATH, DaemonControlMessage, GatewayControlMessage, REGISTRATION_NONCE_BYTES, - RegisterParams, RegistrationReceipt, RegistrationRejectionCode, decode_hex, encode_hex, - signed_registration_message, validate_registration_metadata, + Admission, ChallengeNonce, ChallengeParams, ClosedParams, ClosedReason, DaemonPublicKey, + GoodbyeParams, OpenStreamParams, OpenStreamResponse, RegisterParams, RegisterResponse, + RegistrationIdentityMode, RegistrationReceipt, StreamSpec, signed_registration_message, }, - shared::CURRENT_PROTOCOL_VERSION, + routes::{CanonicalOrigin, DataRouteRefusal}, }; use environments::{ - CreateRegisteredEnvironment, EnvironmentRecord, EnvironmentRegistrationKeyStore, - EnvironmentRegistryError, EnvironmentStatus, EnvironmentStore, ObserveRegisteredEnvironment, - RegisteredConnectionObservation, registration_key_hash, validate_daemon_public_key, + BeginCloseEnvironment, CreateRegisteredEnvironment, DaemonBuild, EnvironmentCloseReason, + EnvironmentRecord, EnvironmentRegistrationKeyStore, EnvironmentRegistryError, EnvironmentState, + EnvironmentStore, MarkRegisteredConnected, MarkRegisteredDisconnected, + REGISTERED_LIVENESS_STAMP_MS, RegisteredIdentityMode, RegisteredSweep, StampRegisteredSeen, + WAIT_RETRY_AFTER_MS, registration_key_hash, }; -use tokio::sync::{Semaphore, mpsc, oneshot}; +use tokio::sync::{Semaphore, mpsc, oneshot, watch}; use uuid::Uuid; -use super::http::GatewayState; -use crate::environments::gateway::{RouteKey, close_message}; +use super::http::{GatewayState, refusal_response}; +use crate::{ + environments::{ + gateway::{RouteKey, close_message}, + signals::EnvironmentSignals, + }, + universe::UniverseRuntime, +}; -/// Interval between gateway pings on a control connection; each pong -/// refreshes the environment's heartbeat stamp. +/// Interval between gateway pings on a control connection. Sent to the +/// daemon as `heartbeatIntervalMs`; pongs keep the connection alive, and +/// at most one per [`REGISTERED_LIVENESS_STAMP_MS`] refreshes the row's +/// liveness stamp. pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30); -/// Time a daemon has to answer the challenge. +/// A control connection that sends no frame for this long is dead. +const LIVENESS_TIMEOUT: Duration = + Duration::from_secs(HEARTBEAT_INTERVAL.as_secs() * HEARTBEAT_MISSED_INTERVALS as u64); +/// Time from the challenge to the register request; the gateway closes +/// the socket when it passes and the daemon reconnects with backoff. const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(15); -/// Time a daemon has to dial the data route for one open request. -const DATA_DIAL_TIMEOUT: Duration = Duration::from_secs(15); -/// Interval at which a control connection re-reads durable state. -const REAUTHORIZE_INTERVAL: Duration = Duration::from_secs(5); -/// Unauthenticated handshakes in flight at once. +/// Time a daemon has to dial the stream route and answer one +/// `registration/openStream`. The worker's upgrade waits on this pairing, +/// so worker routes connect with a longer bound +/// (`environments::gateway::WORKER_ROUTE_CONNECT_TIMEOUT`) and receive the +/// gateway's 502/504 instead of their own connect timeout. +pub(crate) const STREAM_OPEN_TIMEOUT: Duration = Duration::from_secs(15); +/// Pause before a control connection re-subscribes to its environment's +/// revocations after the subscription ended. +const RESUBSCRIBE_DELAY: Duration = Duration::from_secs(1); +/// Interval of the registered sweep ([`run_registered_sweep`]). +pub const REGISTERED_SWEEP_INTERVAL: Duration = Duration::from_secs(30); +/// Unauthenticated handshakes in flight at once. A permit is held from the +/// upgrade until the admission is written, never for an admitted session. const MAX_CONCURRENT_HANDSHAKES: usize = 256; -/// Failed authentications per key prefix (or daemon) per window. +/// Failed admissions per verified daemon identity per window. const MAX_FAILURES_PER_WINDOW: usize = 20; const FAILURE_WINDOW: Duration = Duration::from_secs(60); -/// Control frames are small JSON documents; anything larger is hostile. +/// Daemon identities the failure limiter tracks at once; the identity whose +/// last failure is oldest is forgotten first. +const MAX_LIMITED_IDENTITIES: usize = 4096; +/// Registration frames are small JSON documents (a register request with +/// full metadata is about 12 KiB), so the control route accepts far less +/// than the protocol's message limit before anyone is authenticated. const CONTROL_MAX_MESSAGE_BYTES: usize = 64 * 1024; +/// `openStream` requests one control connection may have unanswered. +const CONTROL_COMMAND_QUEUE: usize = 16; /// Live state of registered daemons on this gateway replica. In-memory by /// design: durable truth is the environment row. pub struct RegisteredConnections { controls: Mutex>, - pending: Mutex>, + /// One lock per environment with a connection transition in progress + /// ([`RegisteredConnections::transition`]); pruned when the last holder + /// or waiter is done. + transitions: Mutex>, + pending: Mutex>, handshakes: Arc, - failures: Mutex>>, + failures: Mutex, } struct ControlHandle { - connection_id: String, - incarnation_id: String, + connection_id: ControlConnectionId, commands: mpsc::Sender, + /// Fired when a newer connection of the same identity is installed. + /// A oneshot, so backpressure on `commands` can never drop it. + supersede: oneshot::Sender<()>, + /// Dropped with the handle: every data route paired under this + /// connection subscribes to it and ends when it closes. + ended: watch::Sender<()>, } enum ControlCommand { - OpenData { token: String, data_url: String }, - Supersede, + /// Ask the daemon for one data stream; `answered` receives its + /// `registration/openStream` response. + OpenStream { + token: SecretString, + answered: oneshot::Sender>, + }, } -struct PendingData { +/// A stream token the gateway issued and is waiting to pair. +struct PendingStream { sender: oneshot::Sender, expires_at: Instant, + key: RouteKey, + /// The control connection the token was sent on; pairing is refused + /// once it is no longer current. + control_connection_id: ControlConnectionId, } impl Default for RegisteredConnections { @@ -99,18 +170,42 @@ impl RegisteredConnections { pub fn new() -> Self { Self { controls: Mutex::new(HashMap::new()), + transitions: Mutex::new(HashMap::new()), pending: Mutex::new(HashMap::new()), handshakes: Arc::new(Semaphore::new(MAX_CONCURRENT_HANDSHAKES)), - failures: Mutex::new(HashMap::new()), + failures: Mutex::new(FailureLimiter::default()), + } + } + + /// Serialize this replica's connection writes of one environment: an + /// admission's install and its connected write, a session end's removal + /// and its disconnected write, and a liveness stamp's "still current" + /// check and its writes. A connection's disconnect therefore never lands + /// after a newer connection's connected write, and a handle an admission + /// hands back after a failed write is always removed again, with its + /// disconnect, by its own session end. + async fn transition(self: &Arc, universe_id: Uuid, environment_id: &str) -> Transition { + let key = (universe_id, environment_id.to_owned()); + let lock = self + .transitions + .lock() + .expect("registered transitions poisoned") + .entry(key.clone()) + .or_default() + .clone(); + let guard = lock.lock_owned().await; + Transition { + registrations: self.clone(), + key, + guard: Some(guard), } } - /// The control connection currently serving an environment, if any. - pub fn current_connection_id(&self, key: &RouteKey) -> Option { + fn current(&self, universe_id: Uuid, environment_id: &str) -> Option { self.controls .lock() .expect("registered connections poisoned") - .get(&(key.universe_id, key.environment_id.clone())) + .get(&(universe_id, environment_id.to_owned())) .map(|handle| handle.connection_id.clone()) } @@ -132,236 +227,467 @@ impl RegisteredConnections { &self, universe_id: Uuid, environment_id: &str, - connection_id: &str, + connection_id: &ControlConnectionId, + ) -> bool { + self.uninstall(universe_id, environment_id, connection_id, None) + } + + /// Replace the handle, if it is still the current one, with `previous` + /// (an admission that failed hands the environment back to the + /// connection it displaced) or with nothing; returns true when this + /// connection was current until now. + fn uninstall( + &self, + universe_id: Uuid, + environment_id: &str, + connection_id: &ControlConnectionId, + previous: Option, ) -> bool { let mut controls = self .controls .lock() .expect("registered connections poisoned"); let key = (universe_id, environment_id.to_owned()); - if controls + if !controls .get(&key) - .is_some_and(|handle| handle.connection_id == connection_id) + .is_some_and(|handle| &handle.connection_id == connection_id) { - controls.remove(&key); - true - } else { - false + return false; } + match previous { + Some(previous) => controls.insert(key, previous), + None => controls.remove(&key), + }; + true } - fn record_failure(&self, subject: &str) -> bool { + fn take_pending(&self, token: &str) -> Option { + let mut pending = self.pending.lock().expect("pending streams poisoned"); let now = Instant::now(); - let mut failures = self.failures.lock().expect("failures poisoned"); - let entry = failures.entry(subject.to_owned()).or_default(); - entry.retain(|at| now.duration_since(*at) < FAILURE_WINDOW); - entry.push(now); - entry.len() > MAX_FAILURES_PER_WINDOW + pending.retain(|_, entry| entry.expires_at > now); + pending.remove(token) } - fn over_failure_limit(&self, subject: &str) -> bool { - let now = Instant::now(); - let failures = self.failures.lock().expect("failures poisoned"); - failures.get(subject).is_some_and(|entries| { - entries + fn retry_after(&self, identity: &DaemonPublicKey) -> Option { + self.failures + .lock() + .expect("failure limiter poisoned") + .retry_after(identity, Instant::now()) + } + + fn record_failure(&self, identity: &DaemonPublicKey) { + self.failures + .lock() + .expect("failure limiter poisoned") + .record(*identity, Instant::now()); + } +} + +type TransitionLock = Arc>; + +/// A held connection transition of one environment; the lock entry is +/// pruned on drop once nobody else holds or waits for it. +struct Transition { + registrations: Arc, + key: (Uuid, String), + guard: Option>, +} + +impl Drop for Transition { + fn drop(&mut self) { + drop(self.guard.take()); + let Ok(mut transitions) = self.registrations.transitions.lock() else { + return; + }; + if transitions + .get(&self.key) + .is_some_and(|lock| Arc::strong_count(lock) == 1) + { + transitions.remove(&self.key); + } + } +} + +/// Failed admissions per verified daemon identity. Keyed only on identities +/// whose signature verified, so nobody can spend another daemon's +/// allowance with claimed values; pruned on every insert and capped. +#[derive(Default)] +struct FailureLimiter { + identities: HashMap>, +} + +impl FailureLimiter { + /// How long `identity` must wait before its next attempt, when it is + /// over the limit. + fn retry_after(&self, identity: &DaemonPublicKey, now: Instant) -> Option { + let failures = self.identities.get(identity)?; + let live: Vec<&Instant> = failures + .iter() + .filter(|at| now.duration_since(**at) < FAILURE_WINDOW) + .collect(); + if live.len() < MAX_FAILURES_PER_WINDOW { + return None; + } + let oldest = live.first()?; + Some((FAILURE_WINDOW - now.duration_since(**oldest)).max(Duration::from_millis(1))) + } + + fn record(&mut self, identity: DaemonPublicKey, now: Instant) { + self.identities.retain(|_, failures| { + while failures + .front() + .is_some_and(|at| now.duration_since(*at) >= FAILURE_WINDOW) + { + failures.pop_front(); + } + !failures.is_empty() + }); + if !self.identities.contains_key(&identity) + && self.identities.len() >= MAX_LIMITED_IDENTITIES + && let Some(stalest) = self + .identities .iter() - .filter(|at| now.duration_since(**at) < FAILURE_WINDOW) - .count() - >= MAX_FAILURES_PER_WINDOW - }) + .min_by_key(|(_, failures)| failures.back().copied()) + .map(|(identity, _)| *identity) + { + self.identities.remove(&stalest); + } + let failures = self.identities.entry(identity).or_default(); + failures.push_back(now); + while failures.len() > MAX_FAILURES_PER_WINDOW { + failures.pop_front(); + } } - fn take_pending(&self, token: &str) -> Option { - let mut pending = self.pending.lock().expect("pending data poisoned"); - let now = Instant::now(); - pending.retain(|_, entry| entry.expires_at > now); - pending.remove(token) + #[cfg(test)] + fn len(&self) -> usize { + self.identities.len() } } /// Outcome of admitting one register request against durable state. #[derive(Debug)] -pub struct Admission { +pub struct AdmittedEnvironment { pub environment: EnvironmentRecord, - pub created: bool, + pub admission: Admission, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RegistrationRejection { - pub code: RegistrationRejectionCode, - pub message: String, +/// How a verified register request will be admitted, decided before the +/// gateway installs the connection and writes the row. +#[derive(Debug)] +pub enum AdmissionPlan { + /// A known daemon identity whose environment is open reconnects. + Reconnect { + environment: Box, + daemon_build: DaemonBuild, + }, + /// A new identity that a registration key admits. + Create(CreateRegisteredEnvironment), } -impl RegistrationRejection { - fn new(code: RegistrationRejectionCode, message: impl Into) -> Self { - Self { - code, - message: message.into(), +impl AdmissionPlan { + /// The environment the connection will serve. + pub fn environment_id(&self) -> &environments::EnvironmentId { + match self { + Self::Reconnect { environment, .. } => &environment.environment_id, + Self::Create(request) => &request.environment_id, } } } -/// Check a register request's shape and signature against the challenge -/// nonce. Pure: no store access. +/// Decode `registration/register` params. The protocol version is read +/// first, so a daemon of another version learns it is stale even when the +/// rest of its request has a shape this gateway does not know; then out of +/// bounds descriptive values are `invalidRequest` and every other decode +/// failure `invalidParams`. +pub fn decode_register(params: &serde_json::Value) -> Result { + if let Some(version) = params + .get("protocolVersion") + .and_then(serde_json::Value::as_u64) + && version != u64::from(ENVD_PROTOCOL_VERSION) + { + return Err(ProtocolError::version_mismatch( + ENVD_PROTOCOL_VERSION, + u32::try_from(version).unwrap_or(u32::MAX), + )); + } + let params = RegisterParams::decode(params)?; + if params.protocol_version != ENVD_PROTOCOL_VERSION { + return Err(ProtocolError::version_mismatch( + ENVD_PROTOCOL_VERSION, + params.protocol_version, + )); + } + Ok(params) +} + +/// Check a register request's signature over the challenge nonce bound to +/// one of the gateway's accepted origins. Pure: no store access. pub fn verify_register_request( params: &RegisterParams, - nonce: &[u8], -) -> Result<(), RegistrationRejection> { - if params.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(RegistrationRejection::new( - RegistrationRejectionCode::UnsupportedProtocol, - format!( - "unsupported registration protocol version {}; expected {CURRENT_PROTOCOL_VERSION}", - params.protocol_version - ), - )); + nonce: &ChallengeNonce, + origins: &[CanonicalOrigin], +) -> Result<(), ProtocolError> { + let invalid = |message: &str| { + ProtocolError::permission_denied(Some(PermissionReason::InvalidSignature), message) + }; + let public_key = VerifyingKey::from_bytes(params.daemon_public_key.as_bytes()) + .map_err(|_| invalid("daemon public key is not a valid Ed25519 key"))?; + let signature = Signature::from_bytes(params.signature.as_bytes()); + if origins.iter().any(|origin| { + public_key + .verify_strict(&signed_registration_message(nonce, origin), &signature) + .is_ok() + }) { + return Ok(()); } - validate_daemon_public_key(¶ms.daemon_public_key).map_err(|error| { - RegistrationRejection::new(RegistrationRejectionCode::InvalidRequest, error.to_string()) - })?; - validate_registration_metadata(params.display_name.as_deref(), ¶ms.metadata).map_err( - |message| RegistrationRejection::new(RegistrationRejectionCode::InvalidRequest, message), - )?; - let public_key = decode_hex(¶ms.daemon_public_key) - .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok()) - .and_then(|bytes| VerifyingKey::from_bytes(&bytes).ok()) - .ok_or_else(|| { - RegistrationRejection::new( - RegistrationRejectionCode::InvalidSignature, - "daemon public key is not a valid Ed25519 key", - ) - })?; - let signature = decode_hex(¶ms.signature) - .and_then(|bytes| Signature::from_slice(&bytes).ok()) - .ok_or_else(|| { - RegistrationRejection::new( - RegistrationRejectionCode::InvalidSignature, - "signature is not a valid Ed25519 signature", - ) - })?; - public_key - .verify(&signed_registration_message(nonce), &signature) - .map_err(|_| { - RegistrationRejection::new( - RegistrationRejectionCode::InvalidSignature, - "signature does not verify against the challenge", - ) - }) + Err(invalid( + "signature does not verify against the challenge and this gateway's origins", + )) } -/// Admit a verified register request inside its universe: reconnect a known -/// daemon identity, or create an environment for a new one through its -/// registration key. Every refusal is typed and leaves no rows behind. -pub async fn admit_registration( +/// Decide how a verified register request is admitted inside its universe: +/// a known daemon identity reconnects its open environment, a new one is +/// admitted through its registration key. Reads only; every refusal is +/// typed. +pub async fn plan_admission( store: &S, params: &RegisterParams, now_ms: i64, -) -> Result +) -> Result where S: EnvironmentStore + EnvironmentRegistrationKeyStore + ?Sized, { + let daemon_public_key = params.daemon_public_key.to_hex(); + let daemon_build = + DaemonBuild::from_implementation(¶ms.implementation, params.protocol_version); let known = store - .read_environment_by_daemon_public_key(¶ms.daemon_public_key) + .read_environment_by_daemon_public_key(&daemon_public_key) .await .map_err(unavailable)?; if let Some(environment) = known { - if matches!( - environment.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) { - return Err(RegistrationRejection::new( - RegistrationRejectionCode::EnvironmentClosed, - format!( - "environment {} is closed; this daemon identity is spent", - environment.environment_id - ), - )); + if environment.state == EnvironmentState::Closed { + return Err(environment_closed(&environment)); } - return Ok(Admission { - environment, - created: false, + return Ok(AdmissionPlan::Reconnect { + environment: Box::new(environment), + daemon_build, }); } let Some(secret) = params.registration_key.as_ref() else { - return Err(RegistrationRejection::new( - RegistrationRejectionCode::UnknownDaemon, - "unknown daemon identity and no registration key presented", - )); + return Err(unknown_daemon()); }; + let secret_hash = registration_key_hash(secret.expose()); let key = store - .resolve_registration_key(®istration_key_hash(secret.expose())) + .resolve_registration_key(&secret_hash) .await .map_err(unavailable)? - .ok_or_else(|| { - RegistrationRejection::new( - RegistrationRejectionCode::InvalidRegistrationKey, - "registration key is not recognized", - ) - })?; + .ok_or_else(unknown_registration_key)?; key.check_admits(now_ms).map_err(map_registry_error)?; - let mut metadata = params.metadata.clone(); - metadata.extend( - environments::RegisteredDaemonBuild::from_registration( - ¶ms.implementation, - params.protocol_version, - ) - .metadata(), - ); - let environment = store - .create_registered_environment(CreateRegisteredEnvironment { - registration_key_id: key.registration_key_id.clone(), - environment_id: super::service::allocate_environment_id_public(), - incarnation_id: super::service::allocate_incarnation_id_public(), - daemon_public_key: params.daemon_public_key.clone(), - display_name: params.display_name.clone(), - metadata, - created_at_ms: now_ms, - }) - .await - .map_err(map_registry_error)?; - Ok(Admission { - environment, - created: true, + Ok(AdmissionPlan::Create(CreateRegisteredEnvironment { + registration_key_id: key.registration_key_id.clone(), + secret_hash, + environment_id: super::service::allocate_environment_id_public(), + daemon_public_key, + display_name: params + .display_name + .as_ref() + .map(|name| name.as_str().to_owned()), + metadata: params.metadata.as_map().clone(), + daemon_build, + created_at_ms: now_ms, + })) +} + +/// Write an admission: mark a reconnecting environment connected with the +/// build that serves it now, or create the new environment open and +/// connected (the store re-checks the key and its capacity in the same +/// transaction). Nothing is left behind on a refusal. +pub async fn commit_admission( + store: &S, + plan: AdmissionPlan, + now_ms: i64, +) -> Result +where + S: EnvironmentStore + ?Sized, +{ + match plan { + AdmissionPlan::Reconnect { + environment, + daemon_build, + } => { + let marked = store + .mark_registered_connected(MarkRegisteredConnected { + environment_id: environment.environment_id.clone(), + daemon_build, + at_ms: now_ms, + }) + .await; + match marked { + Ok(environment) => Ok(AdmittedEnvironment { + environment, + admission: Admission::Reconnected, + }), + // Closed since the plan read it: the identity is spent. + Err(EnvironmentRegistryError::InvalidInput { .. }) + if store + .read_environment(&environment.environment_id) + .await + .is_ok_and(|current| current.state == EnvironmentState::Closed) => + { + Err(environment_closed(&environment)) + } + Err(error) => Err(map_registry_error(error)), + } + } + AdmissionPlan::Create(request) => { + let environment = store + .create_registered_environment(request.clone()) + .await + .map_err(map_registry_error)?; + if environment.environment_id != request.environment_id { + // Another connection of this identity was admitted first. + return Err(if environment.state == EnvironmentState::Closed { + environment_closed(&environment) + } else { + ProtocolError::unavailable( + "this daemon identity was admitted by another connection; reconnect", + ) + }); + } + Ok(AdmittedEnvironment { + environment, + admission: Admission::Created, + }) + } + } +} + +/// [`plan_admission`] then [`commit_admission`], for callers that hold no +/// connection to install in between. +pub async fn admit_registration( + store: &S, + params: &RegisterParams, + now_ms: i64, +) -> Result +where + S: EnvironmentStore + EnvironmentRegistrationKeyStore + ?Sized, +{ + let plan = plan_admission(store, params, now_ms).await?; + commit_admission(store, plan, now_ms).await +} + +/// The receipt of an admitted environment, built only from a registered +/// record. +fn registration_receipt( + environment: &EnvironmentRecord, + control_connection_id: &ControlConnectionId, + admission: Admission, +) -> Result { + let Some(source) = environment.registered() else { + return Err(ProtocolError::internal( + "the admitted environment is not a registered environment", + )); + }; + let internal = |error: environment_protocol::ids::IdError| { + ProtocolError::internal(format!("admitted environment id: {error}")) + }; + Ok(RegistrationReceipt { + environment_id: EnvironmentId::new(environment.environment_id.as_str()) + .map_err(internal)?, + daemon_id: DaemonId::new(source.daemon_id().as_str()).map_err(internal)?, + control_connection_id: control_connection_id.clone(), + identity_mode: match source.identity_mode { + RegisteredIdentityMode::Persistent => RegistrationIdentityMode::Persistent, + RegisteredIdentityMode::Ephemeral => RegistrationIdentityMode::Ephemeral, + }, + admission, }) } -fn map_registry_error(error: EnvironmentRegistryError) -> RegistrationRejection { +fn environment_closed(environment: &EnvironmentRecord) -> ProtocolError { + ProtocolError::failed_precondition( + PreconditionReason::EnvironmentClosed, + format!( + "environment {} is closed; this daemon identity is spent", + environment.environment_id + ), + ) +} + +fn unknown_daemon() -> ProtocolError { + ProtocolError::not_found_because( + NotFoundReason::UnknownDaemon, + "unknown daemon identity and no registration key presented", + ) +} + +fn unknown_registration_key() -> ProtocolError { + ProtocolError::permission_denied( + Some(PermissionReason::InvalidRegistrationKey), + "registration key is not recognized", + ) +} + +fn map_registry_error(error: EnvironmentRegistryError) -> ProtocolError { match error { + EnvironmentRegistryError::RegistrationKeyUnavailable { + reason: "expired", .. + } => ProtocolError::expired("registration key is expired"), + // A rotation retired the presented secret while its admission was in + // flight: the same answer a later admission with it gets. + EnvironmentRegistryError::RegistrationKeyUnavailable { + reason: "rotated", .. + } => unknown_registration_key(), EnvironmentRegistryError::RegistrationKeyUnavailable { reason, .. } => { - RegistrationRejection::new( - if reason == "expired" { - RegistrationRejectionCode::RegistrationKeyExpired - } else { - RegistrationRejectionCode::RegistrationKeyRevoked - }, + ProtocolError::permission_denied( + Some(PermissionReason::RegistrationKeyRevoked), format!("registration key is {reason}"), ) } EnvironmentRegistryError::RegistrationCapacityExhausted { limit, .. } => { - RegistrationRejection::new( - RegistrationRejectionCode::CapacityExhausted, + ProtocolError::busy( + Some(BusyReason::CapacityExhausted), + None, format!("registration key has reached its active environment limit of {limit}"), ) } EnvironmentRegistryError::AlreadyExists { kind: "daemon_identity", .. - } => RegistrationRejection::new( - RegistrationRejectionCode::IdentityInUse, + } => ProtocolError::failed_precondition( + PreconditionReason::IdentityInUse, "daemon identity is already bound to an environment elsewhere", ), + // The record the request would create breaks the registry's bounds: + // admission is refused, never granted with the value dropped. EnvironmentRegistryError::InvalidInput { message } => { - RegistrationRejection::new(RegistrationRejectionCode::InvalidRequest, message) + ProtocolError::invalid_request(message) } other => unavailable(other), } } -fn unavailable(error: EnvironmentRegistryError) -> RegistrationRejection { +fn unavailable(error: EnvironmentRegistryError) -> ProtocolError { tracing::warn!(target: "temporal_runtime", %error, "registration store failure"); - RegistrationRejection::new( - RegistrationRejectionCode::Unavailable, - "registration is temporarily unavailable", + registration_unavailable() +} + +fn registration_unavailable() -> ProtocolError { + ProtocolError::unavailable("registration is temporarily unavailable") +} + +/// True for refusals that count against a verified identity's allowance: +/// credentials that did not admit it. +fn counts_as_failure(error: &ProtocolError) -> bool { + use environment_protocol::error::ErrorData; + matches!( + error.data, + ErrorData::PermissionDenied { .. } + | ErrorData::Expired {} + | ErrorData::NotFound { + reason: Some(NotFoundReason::UnknownDaemon) + } ) } @@ -370,22 +696,25 @@ pub(super) async fn connect_upgrade( State(state): State>, upgrade: WebSocketUpgrade, ) -> Response { + if state.registration_origins().is_empty() { + tracing::error!( + target: "temporal_runtime", + "registration refused: the gateway has no valid origin to verify daemon signatures against" + ); + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } let Ok(permit) = state.registrations().handshakes.clone().try_acquire_owned() else { return StatusCode::SERVICE_UNAVAILABLE.into_response(); }; upgrade .max_message_size(CONTROL_MAX_MESSAGE_BYTES) - .on_upgrade(move |socket| async move { - let outcome = control_session(state, socket, permit).await; - if let Err(error) = outcome { - tracing::debug!(target: "temporal_runtime", %error, "registration control session ended with error"); - } - }) + .max_frame_size(CONTROL_MAX_MESSAGE_BYTES) + .on_upgrade(move |socket| control_connection(state, socket, permit)) .into_response() } -/// `GET /environment-gateway/data`: a reverse-dialed data socket presenting -/// its one-time token as a bearer header. +/// `GET /environment-gateway/data`: a reverse-dialed stream socket +/// presenting its one-time token as a bearer header. pub(super) async fn data_upgrade( State(state): State>, headers: HeaderMap, @@ -394,16 +723,24 @@ pub(super) async fn data_upgrade( let token = headers .get(axum::http::header::AUTHORIZATION) .and_then(|value| value.to_str().ok()) - .and_then(|value| value.strip_prefix("Bearer ")) - .map(str::to_owned); + .and_then(|value| value.strip_prefix("Bearer ")); let Some(token) = token else { return StatusCode::UNAUTHORIZED.into_response(); }; - let Some(pending) = state.registrations().take_pending(&token) else { + let registrations = state.registrations(); + let Some(pending) = registrations.take_pending(token) else { return StatusCode::UNAUTHORIZED.into_response(); }; + if registrations.current(pending.key.universe_id, &pending.key.environment_id) + != Some(pending.control_connection_id.clone()) + { + // The control connection that issued the token was superseded or + // ended: whoever dials now is not the daemon it was sent to. + return StatusCode::UNAUTHORIZED.into_response(); + } upgrade - .max_message_size(64 * 1024 * 1024) + .max_message_size(MAX_MESSAGE_BYTES as usize) + .max_frame_size(MAX_MESSAGE_BYTES as usize) .on_upgrade(move |socket| async move { // The waiting route owns the socket from here; if it gave up, // dropping the socket closes the daemon's dial. @@ -412,77 +749,113 @@ pub(super) async fn data_upgrade( .into_response() } -/// Ask the daemon serving `key` to dial a data socket for one worker route -/// and wait for it. Returns the paired socket and the control connection it -/// was issued under, so the proxy can fence on it. +/// Ask the daemon serving `key` to dial a data stream for one worker route +/// and wait for it. Returns the paired socket and a receiver that closes +/// when the control connection it was issued under ends. Without a live +/// control connection the route is refused `disconnected`; a daemon that +/// cannot open the stream is a bodiless 502, one that does not dial within +/// [`STREAM_OPEN_TIMEOUT`] a 504. pub(super) async fn open_registered_route( state: &Arc, key: &RouteKey, -) -> Result<(WebSocket, String), StatusCode> { +) -> Result<(WebSocket, watch::Receiver<()>), Box> { + let disconnected = || { + Box::new(refusal_response( + &DataRouteRefusal::Disconnected {}, + Some(WAIT_RETRY_AFTER_MS), + )) + }; let registrations = state.registrations(); - let (commands, control_connection_id) = { + let (commands, control_connection_id, control_ended) = { let controls = registrations .controls .lock() .expect("registered connections poisoned"); let Some(handle) = controls.get(&(key.universe_id, key.environment_id.clone())) else { - return Err(StatusCode::SERVICE_UNAVAILABLE); + return Err(disconnected()); }; - if handle.incarnation_id != key.incarnation_id { - return Err(StatusCode::CONFLICT); - } - (handle.commands.clone(), handle.connection_id.clone()) + ( + handle.commands.clone(), + handle.connection_id.clone(), + handle.ended.subscribe(), + ) }; let token = new_token(); - let (sender, receiver) = oneshot::channel(); + let (sender, socket) = oneshot::channel(); registrations .pending .lock() - .expect("pending data poisoned") + .expect("pending streams poisoned") .insert( token.clone(), - PendingData { + PendingStream { sender, - expires_at: Instant::now() + DATA_DIAL_TIMEOUT, + expires_at: Instant::now() + STREAM_OPEN_TIMEOUT, + key: key.clone(), + control_connection_id, }, ); - let sent = commands - .send(ControlCommand::OpenData { - token: token.clone(), - data_url: state.registration_data_url(), - }) - .await; - if sent.is_err() { - registrations.take_pending(&token); - return Err(StatusCode::SERVICE_UNAVAILABLE); - } - match tokio::time::timeout(DATA_DIAL_TIMEOUT, receiver).await { - Ok(Ok(socket)) => Ok((socket, control_connection_id)), - _ => { + let opened = tokio::time::timeout(STREAM_OPEN_TIMEOUT, async { + let (answered, answer) = oneshot::channel(); + commands + .send(ControlCommand::OpenStream { + token: SecretString::new(token.clone()), + answered, + }) + .await + .map_err(|_| disconnected())?; + match answer.await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::warn!( + target: "temporal_runtime", + environment = %key.environment_id, + %error, + "registered daemon could not open a stream" + ); + return Err(Box::new(StatusCode::BAD_GATEWAY.into_response())); + } + // The control connection ended before the daemon answered. + Err(_) => return Err(disconnected()), + } + socket + .await + .map_err(|_| Box::new(StatusCode::BAD_GATEWAY.into_response())) + }) + .await; + match opened { + Ok(Ok(socket)) => Ok((socket, control_ended)), + Ok(Err(refused)) => { + registrations.take_pending(&token); + Err(refused) + } + Err(_) => { registrations.take_pending(&token); - Err(StatusCode::GATEWAY_TIMEOUT) + Err(Box::new(StatusCode::GATEWAY_TIMEOUT.into_response())) } } } -/// Pipe one worker route and one reverse-dialed daemon socket, re-checking -/// durable state and the current control connection on an interval. +/// Pipe one worker route and one reverse-dialed daemon socket until either +/// side ends, the environment's routes are revoked, or the control +/// connection the stream was issued under ends. pub(super) async fn proxy_registered_route( - state: Arc, - key: RouteKey, - control_connection_id: String, + mut revoked: watch::Receiver, + mut control_ended: watch::Receiver<()>, mut worker: WebSocket, mut daemon: WebSocket, ) { - let mut reauthorize = tokio::time::interval(REAUTHORIZE_INTERVAL); - reauthorize.tick().await; loop { tokio::select! { - _ = reauthorize.tick() => { - if authorize_registered_route(&state, &key, &control_connection_id).await.is_err() { - let _ = worker.send(close_message("registered route authorization changed")).await; - break; - } + // A dropped revocation sender fails closed like a revocation. + _ = revoked.changed() => { + let _ = worker.send(close_message("environment route revoked")).await; + break; + } + // Nobody sends on it: it resolves when the handle is dropped. + _ = control_ended.changed() => { + let _ = worker.send(close_message("registered control connection ended")).await; + break; } message = worker.recv() => { let Some(Ok(message)) = message else { break }; @@ -500,361 +873,908 @@ pub(super) async fn proxy_registered_route( let _ = worker.send(Message::Close(None)).await; } -async fn authorize_registered_route( - state: &GatewayState, - key: &RouteKey, - control_connection_id: &str, -) -> anyhow::Result<()> { - if state.registrations().current_connection_id(key).as_deref() != Some(control_connection_id) { - anyhow::bail!("control connection superseded"); +/// What the gateway does with one frame before admission. +#[derive(Debug, PartialEq)] +enum PreAdmission { + /// A `registration/register` request. + Register(Request), + /// Answer and keep waiting: a framing error. + Answer(jsonrpc::Response), + /// Answer, then close: a request other than register. + AnswerAndClose(jsonrpc::Response), + /// Close without an answer: a notification before admission. + Close, + /// Nothing to do: a response to an id the gateway never sent, a + /// malformed notification, a ping or pong. + Ignore, +} + +fn classify_pre_admission(message: &Message) -> PreAdmission { + match message { + Message::Text(text) => match jsonrpc::parse(text.as_str()) { + Err(rejection) => match rejection.into_response() { + Some(response) => PreAdmission::Answer(response), + None => PreAdmission::Ignore, + }, + Ok(jsonrpc::Message::Request(request)) + if request.method == RegistrationRegister::NAME => + { + PreAdmission::Register(request) + } + Ok(jsonrpc::Message::Request(request)) => PreAdmission::AnswerAndClose( + jsonrpc::Response::failure(Some(request.id), ProtocolError::not_initialized()), + ), + Ok(jsonrpc::Message::Notification(_)) => PreAdmission::Close, + Ok(jsonrpc::Message::Response(_)) => PreAdmission::Ignore, + }, + Message::Binary(_) => PreAdmission::Answer(binary_frame_refusal()), + Message::Close(_) => PreAdmission::Close, + Message::Ping(_) | Message::Pong(_) => PreAdmission::Ignore, } - let api = state - .api_for_daemon(key.universe_id) - .await - .map_err(|error| anyhow::anyhow!(error.message))?; - let environment_id = environments::EnvironmentId::try_new(key.environment_id.clone())?; - let environment = - EnvironmentStore::read_environment(api.store().as_ref(), &environment_id).await?; - if !environment.is_registered() - || environment.incarnation.incarnation_id.as_str() != key.incarnation_id - || matches!( - environment.status, - EnvironmentStatus::Closing | EnvironmentStatus::Closed - ) - { - anyhow::bail!("registered route no longer valid"); +} + +/// What the gateway does with one text frame after admission. +#[derive(Debug, PartialEq)] +enum Admitted { + /// Send this answer; the connection stays open. + Answer(jsonrpc::Response), + /// The daemon answered the gateway request with this id. + StreamAnswer(Id, Result<(), ProtocolError>), + /// `registration/goodbye`: the daemon is stopping deliberately. + Goodbye, + /// Ignore and log. + Ignore(String), +} + +fn classify_admitted(text: &str) -> Admitted { + match jsonrpc::parse(text) { + Err(rejection) => match rejection.into_response() { + Some(response) => Admitted::Answer(response), + None => Admitted::Ignore("malformed notification".to_owned()), + }, + Ok(jsonrpc::Message::Request(request)) => { + let error = if request.method == RegistrationRegister::NAME { + ProtocolError::invalid_request("the daemon is already admitted") + } else { + ProtocolError::method_not_found(&request.method) + }; + Admitted::Answer(jsonrpc::Response::failure(Some(request.id), error)) + } + Ok(jsonrpc::Message::Notification(notification)) + if notification.method == RegistrationGoodbye::NAME => + { + match notification.params::() { + Ok(_) => Admitted::Goodbye, + Err(error) => Admitted::Ignore(format!("undecodable goodbye: {error}")), + } + } + Ok(jsonrpc::Message::Notification(notification)) => { + Admitted::Ignore(format!("unknown notification {}", notification.method)) + } + Ok(jsonrpc::Message::Response(response)) => match response.id.clone() { + Some(id) => { + let answer = response + .decode::() + .map(|_| ()) + .map_err(|error| match error { + jsonrpc::ResponseError::Failed(error) => error, + jsonrpc::ResponseError::Undecodable(message) => ProtocolError::internal( + format!("the openStream response breaks the contract: {message}"), + ), + }); + Admitted::StreamAnswer(id, answer) + } + None => Admitted::Ignore(format!( + "error response without an id: {}", + response + .result + .err() + .map(|error| error.to_string()) + .unwrap_or_default() + )), + }, } - Ok(()) } -async fn control_session( +fn binary_frame_refusal() -> jsonrpc::Response { + jsonrpc::Response::failure( + None, + ProtocolError::invalid_request("binary frames are not part of the registration channel"), + ) +} + +/// A register request whose signature verified, with its admission decided +/// but not yet written. +struct Verified { + universe_id: Uuid, + store: Arc, + register: RegisterParams, + register_id: Id, + plan: AdmissionPlan, +} + +/// An admission that was written, with the installed connection's +/// channels. +struct AdmittedConnection { + universe_id: Uuid, + store: Arc, + register: RegisterParams, + register_id: Id, + environment: EnvironmentRecord, + receipt: RegistrationReceipt, + connection_id: ControlConnectionId, + commands: mpsc::Receiver, + superseded: oneshot::Receiver<()>, + revoked: watch::Receiver, +} + +/// Why a handshake produced no admitted session. +enum Refused { + /// The socket is gone or the daemon gave up; nothing to answer. + Silent, + /// Answer the register request with this error, then close. + Answer(Id, ProtocolError), +} + +async fn control_connection( state: Arc, mut socket: WebSocket, - _permit: tokio::sync::OwnedSemaphorePermit, -) -> anyhow::Result<()> { - let nonce = { - use rand::TryRng as _; - let mut bytes = [0u8; REGISTRATION_NONCE_BYTES]; - rand::rngs::SysRng - .try_fill_bytes(&mut bytes) - .expect("OS random source unavailable"); - bytes + permit: tokio::sync::OwnedSemaphorePermit, +) { + // The handshake waits on the daemon and is bounded; the admission write + // that follows runs to completion, so a row is never left connected + // without the connection that wrote it. + let verified = + match tokio::time::timeout(HANDSHAKE_TIMEOUT, handshake(&state, &mut socket)).await { + Ok(verified) => verified, + Err(_) => { + // A daemon that is paused or starved retries with backoff. + tracing::info!(target: "temporal_runtime", "registration handshake timed out"); + let _ = socket.send(Message::Close(None)).await; + return; + } + }; + let admitted = match verified { + Ok(verified) => admit(&state, verified).await, + Err(refused) => Err(refused), }; - send_control( - &mut socket, - &GatewayControlMessage::Challenge { - protocol_version: CURRENT_PROTOCOL_VERSION, - nonce: encode_hex(&nonce), + // The permit bounds unauthenticated handshakes only: it is released as + // soon as the admission resolved, whatever its outcome. + drop(permit); + match admitted { + Ok(admitted) => { + if let Err(error) = admitted_session(&state, socket, admitted).await { + tracing::debug!(target: "temporal_runtime", %error, "registration control session ended with error"); + } + } + Err(Refused::Answer(id, error)) => { + tracing::info!(target: "temporal_runtime", kind = error.kind().as_str(), %error, "registration refused"); + let _ = send_text( + &mut socket, + jsonrpc::Response::failure(Some(id), error).to_text(), + ) + .await; + let _ = socket.send(Message::Close(None)).await; + } + Err(Refused::Silent) => { + let _ = socket.send(Message::Close(None)).await; + } + } +} + +/// Challenge, read the register request, verify it and decide its +/// admission. +async fn handshake(state: &Arc, socket: &mut WebSocket) -> Result { + let nonce = new_nonce(); + let challenge = Notification::new( + RegistrationChallenge::NAME, + &ChallengeParams { + protocol_version: ENVD_PROTOCOL_VERSION, + nonce, }, ) - .await?; - let register = - match tokio::time::timeout(HANDSHAKE_TIMEOUT, next_daemon_message(&mut socket)).await { - Ok(Ok(Some(DaemonControlMessage::Register(params)))) => params, - Ok(Ok(None)) => return Ok(()), - Ok(Err(error)) => { - reject( - &mut socket, - RegistrationRejectionCode::InvalidRequest, - error.to_string(), - ) - .await; - return Ok(()); + .map_err(|_| Refused::Silent)?; + send_text(socket, challenge.to_text()) + .await + .map_err(|_| Refused::Silent)?; + let request = loop { + let Some(Ok(message)) = socket.recv().await else { + return Err(Refused::Silent); + }; + match classify_pre_admission(&message) { + PreAdmission::Register(request) => break request, + PreAdmission::Answer(response) => { + send_text(socket, response.to_text()) + .await + .map_err(|_| Refused::Silent)?; } - Err(_) => { - reject( - &mut socket, - RegistrationRejectionCode::InvalidRequest, - "handshake timed out", - ) - .await; - return Ok(()); + PreAdmission::AnswerAndClose(response) => { + let _ = send_text(socket, response.to_text()).await; + return Err(Refused::Silent); } - }; + PreAdmission::Close => return Err(Refused::Silent), + PreAdmission::Ignore => {} + } + }; + let id = request.id.clone(); + let refuse = |error: ProtocolError| Refused::Answer(id.clone(), error); + let register = decode_register(&request.params).map_err(refuse)?; + verify_register_request(®ister, &nonce, state.registration_origins()).map_err(refuse)?; + + // From here the identity is verified: rate limits key on it alone. let registrations = state.registrations(); - let subject = register - .registration_key - .as_ref() - .map(|key| format!("key:{}", key.expose().chars().take(12).collect::())) - .unwrap_or_else(|| { - format!( - "daemon:{}", - ®ister.daemon_public_key[..16.min(register.daemon_public_key.len())] - ) - }); - if registrations.over_failure_limit(&subject) { - reject( - &mut socket, - RegistrationRejectionCode::RateLimited, - "too many failed registrations; retry later", - ) - .await; - return Ok(()); - } - if let Err(rejection) = verify_register_request(®ister, &nonce) { - registrations.record_failure(&subject); - reject(&mut socket, rejection.code, rejection.message).await; - return Ok(()); + let identity = register.daemon_public_key; + if let Some(retry_after) = registrations.retry_after(&identity) { + return Err(refuse(ProtocolError::busy( + Some(BusyReason::RateLimited), + Some(u64::try_from(retry_after.as_millis()).unwrap_or(u64::MAX)), + "too many failed registrations for this daemon identity; retry later", + ))); } + let refuse_counted = |error: ProtocolError| { + if counts_as_failure(&error) { + registrations.record_failure(&identity); + } + refuse(error) + }; let universe_id = match state.registration_universe(®ister).await { Ok(Some(universe_id)) => universe_id, Ok(None) => { - registrations.record_failure(&subject); - let (code, message) = if register.registration_key.is_some() { - ( - RegistrationRejectionCode::InvalidRegistrationKey, - "registration key is not recognized", - ) + return Err(refuse_counted(if register.registration_key.is_some() { + unknown_registration_key() } else { - ( - RegistrationRejectionCode::UnknownDaemon, - "unknown daemon identity and no registration key presented", - ) - }; - reject(&mut socket, code, message).await; - return Ok(()); + unknown_daemon() + })); } Err(error) => { tracing::warn!(target: "temporal_runtime", error = %error.message, "registration universe lookup failed"); - reject( - &mut socket, - RegistrationRejectionCode::Unavailable, - "registration is temporarily unavailable", - ) - .await; - return Ok(()); - } - }; - let api = match state.api_for_daemon(universe_id).await { - Ok(api) => api, - Err(error) => { - tracing::warn!(target: "temporal_runtime", error = %error.message, "registration universe unavailable"); - reject( - &mut socket, - RegistrationRejectionCode::Unavailable, - "registration is temporarily unavailable", - ) - .await; - return Ok(()); + return Err(refuse(registration_unavailable())); } }; + let api = state.api_for_daemon(universe_id).await.map_err(|error| { + tracing::warn!(target: "temporal_runtime", error = %error.message, "registration universe unavailable"); + refuse(registration_unavailable()) + })?; let store = api.store().clone(); - let admission = match admit_registration(store.as_ref(), ®ister, now_ms()).await { - Ok(admission) => admission, - Err(rejection) => { - if matches!( - rejection.code, - RegistrationRejectionCode::InvalidRegistrationKey - | RegistrationRejectionCode::RegistrationKeyRevoked - | RegistrationRejectionCode::RegistrationKeyExpired - ) { - registrations.record_failure(&subject); - } - reject(&mut socket, rejection.code, rejection.message).await; - return Ok(()); - } - }; - let environment = admission.environment; - let environment_id = environment.environment_id.to_string(); - let connection_id = format!("connection_{}", Uuid::new_v4().simple()); - let (commands, mut command_rx) = mpsc::channel::(16); - if let Some(previous) = registrations.install( + let plan = plan_admission(store.as_ref(), ®ister, now_ms()) + .await + .map_err(refuse_counted)?; + Ok(Verified { universe_id, - environment_id.clone(), - ControlHandle { - connection_id: connection_id.clone(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - commands, - }, - ) { - let _ = previous.commands.try_send(ControlCommand::Supersede); - } - // Every admission re-records which build serves the environment, so a - // replaced daemon shows its new version, commit, and protocol on the row - // instead of the one it first registered with. - let daemon_build = environments::RegisteredDaemonBuild::from_registration( - ®ister.implementation, - register.protocol_version, + store, + register, + register_id: id, + plan, + }) +} + +/// Install the connection, then write the admission, both inside the +/// environment's connection transition. The handle goes in first, so a row +/// that reads connected without a handle exists only while a connection +/// ends. The connection it displaces is superseded only once the write +/// succeeded; a refused admission hands the environment back to it, and +/// since that connection's own session end takes the same transition, a +/// handle handed back after its session ended is removed again with its +/// disconnect. +async fn admit( + state: &Arc, + verified: Verified, +) -> Result { + let Verified { + universe_id, + store, + register, + register_id, + plan, + } = verified; + let registrations = state.registrations(); + let environment_id = plan.environment_id().to_string(); + let connection_id = ControlConnectionId::new(format!("connection_{}", Uuid::new_v4().simple())) + .expect("generated control connection ids are valid"); + let (commands, command_rx) = mpsc::channel(CONTROL_COMMAND_QUEUE); + let (supersede, superseded) = oneshot::channel(); + let (ended, _) = watch::channel(()); + // Subscribed before the write: a close that lands after it still + // reaches this connection. + let revoked = state.signals().watch(universe_id, &environment_id); + let _transition = registrations.transition(universe_id, &environment_id).await; + let previous = registrations.install( + universe_id, + environment_id.clone(), + ControlHandle { + connection_id: connection_id.clone(), + commands, + supersede, + ended, + }, ); - let observe = |observation: RegisteredConnectionObservation| { - let store = store.clone(); - let environment_id = environment.environment_id.clone(); - let metadata = match observation { - RegisteredConnectionObservation::Connected => daemon_build.metadata(), - _ => std::collections::BTreeMap::new(), - }; - async move { - let result = EnvironmentStore::observe_registered_environment( - store.as_ref(), - ObserveRegisteredEnvironment { - environment_id, - observation, - observed_at_ms: now_ms(), - metadata, - }, - ) - .await; - if let Err(error) = result { - tracing::warn!(target: "temporal_runtime", %error, "registered observation failed"); - } + let refuse = |error: ProtocolError| { + if counts_as_failure(&error) { + registrations.record_failure(®ister.daemon_public_key); } + Refused::Answer(register_id.clone(), error) }; - observe(RegisteredConnectionObservation::Connected).await; - let identity_mode = environment - .source - .identity_mode() - .map(|mode| mode.as_str().to_owned()) - .unwrap_or_default(); - let daemon_id = match &environment.source { - environments::EnvironmentSource::Registered { daemon_id, .. } => daemon_id.to_string(), - _ => String::new(), + let admitted = match commit_admission(store.as_ref(), plan, now_ms()).await { + Ok(admitted) => admitted, + Err(error) => { + registrations.uninstall(universe_id, &environment_id, &connection_id, previous); + return Err(refuse(error)); + } }; + let receipt = + match registration_receipt(&admitted.environment, &connection_id, admitted.admission) { + Ok(receipt) => receipt, + Err(error) => { + // A handed-back connection is live and keeps the row + // connected; without one nobody holds it. + let handed_back = previous.is_some(); + if registrations.uninstall(universe_id, &environment_id, &connection_id, previous) + && !handed_back + { + mark_disconnected(&store, &admitted.environment.environment_id).await; + } + return Err(refuse(error)); + } + }; + if let Some(previous) = previous { + // Dropping the displaced handle also ends its data routes. + let _ = previous.supersede.send(()); + } + Ok(AdmittedConnection { + universe_id, + store, + register, + register_id, + environment: admitted.environment, + receipt, + connection_id, + commands: command_rx, + superseded, + revoked, + }) +} + +/// The admitted control connection: answer the register request, then +/// ping, pair streams and watch for revocation until the session ends. +/// Cleanup runs on every exit path. +async fn admitted_session( + state: &Arc, + mut socket: WebSocket, + admitted: AdmittedConnection, +) -> anyhow::Result<()> { + let AdmittedConnection { + universe_id, + store, + register, + register_id, + environment, + receipt, + connection_id, + commands, + superseded, + revoked, + } = admitted; + let environment_id = environment.environment_id.to_string(); + let registrations = state.registrations(); tracing::info!( target: "temporal_runtime", %universe_id, environment = %environment_id, - daemon = %daemon_id, + daemon = %receipt.daemon_id, + daemon_public_key = %register.daemon_public_key, connection = %connection_id, - created = admission.created, + admission = ?receipt.admission, "registered environment control connection admitted" ); - send_control( - &mut socket, - &GatewayControlMessage::Accepted { - receipt: RegistrationReceipt { - environment_id: environment_id.clone(), - incarnation_id: environment.incarnation.incarnation_id.to_string(), - daemon_id, - connection_id: connection_id.clone(), - identity_mode, - }, - heartbeat_interval_ms: HEARTBEAT_INTERVAL.as_millis() as u64, + let session = ControlSession { + state, + store: &store, + universe_id, + environment: &environment, + connection_id: &connection_id, + daemon_build: DaemonBuild::from_implementation( + ®ister.implementation, + register.protocol_version, + ), + }; + let end = session + .serve( + &mut socket, + register_id, + receipt, + commands, + superseded, + revoked, + ) + .await; + // Inside the transition: a liveness write that already passed its + // "still current" check lands first, a newer admission's connected + // write lands after this disconnect, and a later liveness write sees + // the removal. + let transition = registrations.transition(universe_id, &environment_id).await; + let was_current = registrations.remove_if_current(universe_id, &environment_id, &connection_id); + tracing::info!( + target: "temporal_runtime", + %universe_id, + environment = %environment_id, + connection = %connection_id, + reason = end.reason(), + current = was_current, + "registered environment control connection ended" + ); + if was_current { + mark_disconnected(&store, &environment.environment_id).await; + if end == SessionEnd::Goodbye && departure_closes(&environment) { + close_departed(&store, &environment).await; + state.signals().revoke(universe_id, &environment_id); + } + } + drop(transition); + let _ = socket.send(Message::Close(None)).await; + Ok(()) +} + +/// A deliberate goodbye ends an ephemeral daemon's sandbox for good, so its +/// environment closes at once instead of after the key's disconnect grace. +/// A persistent daemon's environment waits for it to return. +fn departure_closes(environment: &EnvironmentRecord) -> bool { + environment + .registered() + .is_some_and(|source| source.identity_mode == RegisteredIdentityMode::Ephemeral) +} + +async fn mark_disconnected( + store: &Arc, + environment_id: &environments::EnvironmentId, +) { + let result = EnvironmentStore::mark_registered_disconnected( + store.as_ref(), + MarkRegisteredDisconnected { + environment_id: environment_id.clone(), + at_ms: now_ms(), }, ) - .await?; + .await; + // The registered sweep marks the row disconnected once its liveness + // stamp goes stale. + if let Err(error) = result { + tracing::warn!(target: "temporal_runtime", %error, environment = %environment_id, "recording a registered disconnect failed"); + } +} - let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); - heartbeat.tick().await; - let mut reauthorize = tokio::time::interval(REAUTHORIZE_INTERVAL); - reauthorize.tick().await; - let end_reason: &str = loop { - tokio::select! { - command = command_rx.recv() => match command { - Some(ControlCommand::OpenData { token, data_url }) => { - let message = GatewayControlMessage::OpenData { - token: token.into(), - data_url, +async fn close_departed(store: &Arc, environment: &EnvironmentRecord) { + let result = EnvironmentStore::begin_close_environment( + store.as_ref(), + BeginCloseEnvironment { + environment_id: environment.environment_id.clone(), + reason: EnvironmentCloseReason::Departed, + at_ms: now_ms(), + }, + ) + .await; + match result { + Ok(_) => tracing::info!( + target: "temporal_runtime", + environment = %environment.environment_id, + "ephemeral registered daemon departed; closed its environment" + ), + // The registered sweep closes it after the key's grace. + Err(error) => tracing::warn!( + target: "temporal_runtime", + environment = %environment.environment_id, + %error, + "closing a departed ephemeral environment failed" + ), + } +} + +/// Why an admitted session ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SessionEnd { + Superseded, + EnvironmentClosed, + Goodbye, + HeartbeatTimeout, + DaemonClosed, + SendFailed, +} + +impl SessionEnd { + fn reason(self) -> &'static str { + match self { + Self::Superseded => "superseded", + Self::EnvironmentClosed => "environment closed", + Self::Goodbye => "goodbye", + Self::HeartbeatTimeout => "heartbeat timeout", + Self::DaemonClosed => "daemon closed", + Self::SendFailed => "send failed", + } + } +} + +struct ControlSession<'a> { + state: &'a Arc, + store: &'a Arc, + universe_id: Uuid, + environment: &'a EnvironmentRecord, + connection_id: &'a ControlConnectionId, + daemon_build: DaemonBuild, +} + +impl ControlSession<'_> { + /// Serve the admitted session until it ends. + async fn serve( + &self, + socket: &mut WebSocket, + register_id: Id, + receipt: RegistrationReceipt, + commands: mpsc::Receiver, + superseded: oneshot::Receiver<()>, + revoked: watch::Receiver, + ) -> SessionEnd { + let response = jsonrpc::Response::success( + register_id, + &RegisterResponse { + receipt, + heartbeat_interval_ms: u64::try_from(HEARTBEAT_INTERVAL.as_millis()) + .unwrap_or(u64::MAX), + }, + ); + if send_text(socket, response.to_text()).await.is_err() { + return SessionEnd::SendFailed; + } + self.run(socket, commands, superseded, revoked).await + } + + async fn run( + &self, + socket: &mut WebSocket, + mut commands: mpsc::Receiver, + mut superseded: oneshot::Receiver<()>, + revoked: watch::Receiver, + ) -> SessionEnd { + let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + heartbeat.tick().await; + let (mut environment_closed, _watcher) = self.watch_environment_closed(revoked); + let mut watching = true; + let mut deadline = tokio::time::Instant::now() + LIVENESS_TIMEOUT; + // The admission wrote the first stamp. + let mut last_stamp_ms = self + .environment + .registered() + .map_or_else(now_ms, |source| source.last_seen_at_ms); + let mut next_request = 0_u64; + let mut streams: HashMap>> = HashMap::new(); + let mut commands_closed = false; + // One liveness write at a time. One still running when the session + // ends finishes on its own: it holds the environment's transition, + // so the disconnect waits for it. + let mut seen_write: Option> = None; + loop { + // Biased: a frame that already arrived is read before the + // liveness deadline is checked, so a loop that fell behind + // never declares an answering daemon dead. + tokio::select! { + biased; + _ = &mut superseded => { + self.closed(socket, ClosedReason::Superseded, "another process with this daemon identity connected").await; + return SessionEnd::Superseded; + } + closed = &mut environment_closed, if watching => { + if closed.is_ok() { + self.closed(socket, ClosedReason::EnvironmentClosed, "the environment was closed").await; + return SessionEnd::EnvironmentClosed; + } + watching = false; + } + command = commands.recv(), if !commands_closed => { + // The channel closes only when a newer connection + // replaced this session's handle; supersede follows. + let Some(ControlCommand::OpenStream { token, answered }) = command else { + commands_closed = true; + continue; }; - if send_control(&mut socket, &message).await.is_err() { - break "send failed"; + next_request += 1; + let id = Id::String(format!("g{next_request}")); + let request = Request::new( + id.clone(), + RegistrationOpenStream::NAME, + &OpenStreamParams { token, stream: StreamSpec::DataProtocol {} }, + ); + let Ok(request) = request else { + let _ = answered.send(Err(ProtocolError::internal("encode openStream"))); + continue; + }; + if send_text(socket, request.to_text()).await.is_err() { + return SessionEnd::SendFailed; } + streams.retain(|_, waiting| !waiting.is_closed()); + streams.insert(id, answered); } - Some(ControlCommand::Supersede) => { - let _ = socket.send(close_message("superseded by a newer connection of the same daemon")).await; - break "superseded"; + _ = heartbeat.tick() => { + if socket.send(Message::Ping(Default::default())).await.is_err() { + return SessionEnd::SendFailed; + } } - None => break "command channel closed", - }, - _ = heartbeat.tick() => { - if socket.send(Message::Ping(Default::default())).await.is_err() { - break "ping failed"; + message = socket.recv() => { + let Some(Ok(message)) = message else { + return SessionEnd::DaemonClosed; + }; + deadline = tokio::time::Instant::now() + LIVENESS_TIMEOUT; + let answer = match message { + Message::Pong(_) => { + let at_ms = now_ms(); + if at_ms.saturating_sub(last_stamp_ms) >= REGISTERED_LIVENESS_STAMP_MS + && seen_write.as_ref().is_none_or(|write| write.is_finished()) + { + last_stamp_ms = at_ms; + seen_write = Some(self.record_seen(at_ms)); + } + None + } + Message::Ping(_) => None, + Message::Close(_) => return SessionEnd::DaemonClosed, + Message::Binary(_) => Some(binary_frame_refusal()), + Message::Text(text) => match classify_admitted(text.as_str()) { + Admitted::Answer(response) => Some(response), + Admitted::StreamAnswer(id, answer) => { + match streams.remove(&id) { + Some(waiting) => { let _ = waiting.send(answer); } + None => tracing::debug!(target: "temporal_runtime", %id, "ignoring a response to an unknown request id"), + } + None + } + Admitted::Goodbye => { + tracing::info!(target: "temporal_runtime", environment = %self.environment.environment_id, "registered daemon said goodbye"); + return SessionEnd::Goodbye; + } + Admitted::Ignore(reason) => { + tracing::debug!(target: "temporal_runtime", %reason, "ignoring a registration frame"); + None + } + }, + }; + if let Some(answer) = answer + && send_text(socket, answer.to_text()).await.is_err() + { + return SessionEnd::SendFailed; + } + } + _ = tokio::time::sleep_until(deadline) => { + return SessionEnd::HeartbeatTimeout; } } - _ = reauthorize.tick() => { - match EnvironmentStore::read_environment(store.as_ref(), &environment.environment_id).await { - Ok(current) if matches!(current.status, EnvironmentStatus::Closing | EnvironmentStatus::Closed) => { - let _ = socket.send(close_message("environment closed")).await; - break "environment closed"; + } + } + + /// Wait off the session loop for the environment's routes to be + /// revoked, then re-read the row once, so a slow store never delays + /// pings, pongs or stream requests. The receiver fires once the + /// environment is closed; the watcher stops when the returned guard + /// drops. + fn watch_environment_closed( + &self, + mut revoked: watch::Receiver, + ) -> (oneshot::Receiver<()>, AbortOnDrop) { + let signals: Arc = self.state.signals().clone(); + let store = self.store.clone(); + let universe_id = self.universe_id; + let environment_id = self.environment.environment_id.clone(); + let (closed, receiver) = oneshot::channel(); + let task = tokio::spawn(async move { + loop { + if revoked.changed().await.is_err() { + // The subscription ended without a revocation: take a + // new one, then read the row in case a close was missed. + tokio::time::sleep(RESUBSCRIBE_DELAY).await; + revoked = signals.watch(universe_id, environment_id.as_str()); + } + match EnvironmentStore::read_environment(store.as_ref(), &environment_id).await { + Ok(current) if current.state == EnvironmentState::Closed => { + let _ = closed.send(()); + return; } Ok(_) => {} Err(error) => { - tracing::warn!(target: "temporal_runtime", %error, "registered control reauthorization failed"); + tracing::warn!(target: "temporal_runtime", %error, environment = %environment_id, "registered control connection could not re-read its environment"); } } } - message = socket.recv() => match message { - Some(Ok(Message::Pong(_))) => observe(RegisteredConnectionObservation::Heartbeat).await, - Some(Ok(Message::Ping(bytes))) => { - if socket.send(Message::Pong(bytes)).await.is_err() { - break "pong failed"; - } - } - Some(Ok(Message::Close(_))) | None => break "daemon closed", - Some(Ok(_)) => {} - Some(Err(_)) => break "socket error", + }); + (receiver, AbortOnDrop(task)) + } + + /// Tell the daemon its session ends for good, then close. + async fn closed(&self, socket: &mut WebSocket, reason: ClosedReason, message: &str) { + if let Ok(notification) = Notification::new( + RegistrationClosed::NAME, + &ClosedParams { + reason, + message: DescriptiveText::lossy(message), }, + ) { + let _ = send_text(socket, notification.to_text()).await; } - }; - let was_current = registrations.remove_if_current(universe_id, &environment_id, &connection_id); - tracing::info!( - target: "temporal_runtime", - %universe_id, - environment = %environment_id, - connection = %connection_id, - reason = end_reason, - current = was_current, - "registered environment control connection ended" - ); - if was_current { - observe(RegisteredConnectionObservation::Disconnected).await; + let _ = socket + .send(close_message(match reason { + ClosedReason::Superseded => "superseded by a newer connection of the same daemon", + ClosedReason::EnvironmentClosed => "environment closed", + })) + .await; + } + + /// The coarse liveness stamp of the current control connection, written + /// off the session loop so a slow store never delays pings or stream + /// requests, inside the environment's connection transition. When the + /// row no longer reads connected (a stale sweep or a lost write marked + /// it disconnected while this connection was up), the connection is + /// recorded again instead. + fn record_seen(&self, at_ms: i64) -> tokio::task::JoinHandle<()> { + let registrations = self.state.registrations().clone(); + let store = self.store.clone(); + let universe_id = self.universe_id; + let environment_id = self.environment.environment_id.clone(); + let connection_id = self.connection_id.clone(); + let daemon_build = self.daemon_build.clone(); + tokio::spawn(async move { + let _transition = registrations + .transition(universe_id, environment_id.as_str()) + .await; + if registrations.current(universe_id, environment_id.as_str()) != Some(connection_id) { + return; + } + let stamped = EnvironmentStore::stamp_registered_seen( + store.as_ref(), + StampRegisteredSeen { + environment_id: environment_id.clone(), + at_ms, + }, + ) + .await; + let restore = match stamped { + Ok(true) => false, + Ok(false) => EnvironmentStore::read_environment(store.as_ref(), &environment_id) + .await + .is_ok_and(|current| { + current.state == EnvironmentState::Open + && current.registered().is_some_and(|source| !source.connected) + }), + Err(error) => { + tracing::warn!(target: "temporal_runtime", %error, environment = %environment_id, "registered liveness stamp failed"); + false + } + }; + if restore + && let Err(error) = EnvironmentStore::mark_registered_connected( + store.as_ref(), + MarkRegisteredConnected { + environment_id: environment_id.clone(), + daemon_build, + at_ms, + }, + ) + .await + { + tracing::warn!(target: "temporal_runtime", %error, environment = %environment_id, "restoring a registered connection failed"); + } + }) } - Ok(()) } -async fn send_control( - socket: &mut WebSocket, - message: &GatewayControlMessage, -) -> anyhow::Result<()> { - socket - .send(Message::Text(serde_json::to_string(message)?.into())) - .await - .map_err(|error| anyhow::anyhow!("send control message: {error}")) +/// Aborts a background task when the session that owns it ends. +struct AbortOnDrop(tokio::task::JoinHandle<()>); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } } -async fn reject( - socket: &mut WebSocket, - code: RegistrationRejectionCode, - message: impl Into, -) { - let message = message.into(); - tracing::info!(target: "temporal_runtime", ?code, %message, "registration rejected"); - let _ = send_control(socket, &GatewayControlMessage::Rejected { code, message }).await; - let _ = socket.send(Message::Close(None)).await; +/// Startup repair, before the environment-gateway listener accepts +/// connections: every registered row that reads connected loses its +/// connection now, since this process held all of them (one +/// environment-gateway replica per deployment). Persistent rows then get +/// their full reconnect wait while their daemons reconnect. +pub async fn repair_registered_connections( + runtime: &UniverseRuntime, +) -> Result { + store_pg::repair_registered_connections(runtime.stores().pool(), now_ms()).await } -async fn next_daemon_message( - socket: &mut WebSocket, -) -> anyhow::Result> { +/// The registered sweep of the environment-gateway role, every +/// [`REGISTERED_SWEEP_INTERVAL`]: [`sweep_registered_universe`] for every +/// universe with registered environments, then [`revoke_ended_routes`]. +pub async fn run_registered_sweep(runtime: Arc) { + let mut interval = tokio::time::interval(REGISTERED_SWEEP_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { - match socket.recv().await { - Some(Ok(Message::Text(text))) => { - return serde_json::from_str(&text) - .map(Some) - .map_err(|error| anyhow::anyhow!("decode register message: {error}")); - } - Some(Ok(Message::Binary(bytes))) => { - return serde_json::from_slice(&bytes) - .map(Some) - .map_err(|error| anyhow::anyhow!("decode register message: {error}")); + interval.tick().await; + let universes = match store_pg::list_universes_with_registered_environments( + runtime.stores().pool(), + ) + .await + { + Ok(universes) => universes, + Err(error) => { + tracing::warn!(target: "temporal_runtime", %error, "registered sweep could not list universes"); + Vec::new() } - Some(Ok(Message::Close(_))) | None => return Ok(None), - Some(Ok(Message::Ping(bytes))) => { - socket.send(Message::Pong(bytes)).await?; + }; + let now = now_ms(); + for universe_id in universes { + if let Err(error) = sweep_registered_universe(&runtime, universe_id, now).await { + tracing::warn!(target: "temporal_runtime", %error, %universe_id, "registered sweep failed"); } - Some(Ok(_)) => {} - Some(Err(error)) => return Err(error.into()), } + if let Err(error) = revoke_ended_routes(&runtime).await { + tracing::warn!(target: "temporal_runtime", %error, "route revocation sweep failed"); + } + } +} + +/// One universe's registered sweep at `now_ms`: rows whose liveness stamp +/// went stale become disconnected, ephemeral rows away past their key's +/// grace close (`ephemeral_grace`), persistent rows away past the key's +/// `offlineCloseAfterMs` close (`offline_timeout`). Every closed +/// environment's routes are revoked. +pub async fn sweep_registered_universe( + runtime: &UniverseRuntime, + universe_id: Uuid, + now_ms: i64, +) -> Result { + let store = runtime.stores().store_for(universe_id); + let sweep = EnvironmentStore::sweep_registered_environments(store.as_ref(), now_ms).await?; + for environment_id in &sweep.disconnected { + tracing::info!(target: "temporal_runtime", %universe_id, environment = %environment_id, "registered environment's liveness stamp went stale; marked disconnected"); + } + let signals = runtime.environment_signals(); + for (environment_id, reason) in &sweep.closed { + tracing::info!(target: "temporal_runtime", %universe_id, environment = %environment_id, %reason, "registered sweep closed an environment"); + signals.revoke(universe_id, environment_id.as_str()); + } + Ok(sweep) +} + +/// End the live routes and control connections of environments that are no +/// longer open, or whose provider binding is gone or disabled, when that +/// happened outside this process. Returns how many environments it revoked. +pub async fn revoke_ended_routes( + runtime: &UniverseRuntime, +) -> Result { + let signals = runtime.environment_signals(); + let watched = signals.watched(); + if watched.is_empty() { + return Ok(0); + } + let ended = + store_pg::list_revoked_environment_routes(runtime.stores().pool(), &watched).await?; + for (universe_id, environment_id) in &ended { + signals.revoke(*universe_id, environment_id); } + Ok(ended.len()) +} + +async fn send_text(socket: &mut WebSocket, text: String) -> Result<(), axum::Error> { + socket.send(Message::Text(text.into())).await +} + +fn new_nonce() -> ChallengeNonce { + ChallengeNonce::from_bytes(random_bytes()) } fn new_token() -> String { use base64::Engine as _; + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(random_bytes::<32>()) +} + +fn random_bytes() -> [u8; N] { use rand::TryRng as _; - let mut bytes = [0u8; 32]; + let mut bytes = [0u8; N]; rand::rngs::SysRng .try_fill_bytes(&mut bytes) .expect("OS random source unavailable"); - base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes) + bytes } fn now_ms() -> i64 { @@ -864,25 +1784,22 @@ fn now_ms() -> i64 { .unwrap_or(0) } -/// The public data route daemons dial for reverse data connections. -pub fn data_url(public_base_url: &str) -> String { - format!( - "{}{DATA_PATH}", - crate::environments::gateway::websocket_base(public_base_url.trim_end_matches('/')) - ) -} - #[cfg(test)] mod tests { - use std::collections::BTreeMap; - use ed25519_dalek::{Signer as _, SigningKey}; - use environment_protocol::shared::{ImplementationInfo, SecretString}; + use environment_protocol::{ + error::{ErrorData, ProtocolErrorKind}, + handshake::ImplementationInfo, + metadata::{DisplayName, Metadata}, + registration::Ed25519Signature, + routes::canonical_origin, + }; use environments::{ - BeginCloseEnvironment, CreateEnvironmentRegistrationKey, EnvironmentRegistrationKeyId, - FinishCloseEnvironment, InMemoryEnvironmentRegistryStore, RegisteredIdentityMode, + BeginCloseEnvironment, CreateEnvironmentRegistrationKey, EnvironmentCloseReason, + EnvironmentRegistrationKeyId, InMemoryEnvironmentRegistryStore, MarkRegisteredDisconnected, RegistrationKeyPolicy, RevokeEnvironmentRegistrationKey, mint_registration_key, }; + use serde_json::json; use super::*; @@ -890,23 +1807,38 @@ mod tests { SigningKey::from_bytes(&[seed; 32]) } - fn register(key: &SigningKey, nonce: &[u8], secret: Option<&str>) -> RegisterParams { + fn origin() -> CanonicalOrigin { + canonical_origin("wss://gateway.example/environment-gateway/connect").unwrap() + } + + fn register_signed( + key: &SigningKey, + nonce: &ChallengeNonce, + origin: &CanonicalOrigin, + secret: Option<&str>, + ) -> RegisterParams { RegisterParams { - protocol_version: CURRENT_PROTOCOL_VERSION, + protocol_version: ENVD_PROTOCOL_VERSION, registration_key: secret.map(SecretString::new), - daemon_public_key: encode_hex(key.verifying_key().as_bytes()), - signature: encode_hex(&key.sign(&signed_registration_message(nonce)).to_bytes()), - display_name: Some("worker".to_owned()), - metadata: BTreeMap::new(), - implementation: ImplementationInfo { - name: "lightspeed-envd".to_owned(), - version: Some("0.1.0".to_owned()), - git_sha: Some("1111111111111111111111111111111111111111".to_owned()), - target: Some("x86_64-unknown-linux-musl".to_owned()), - }, + daemon_public_key: DaemonPublicKey::from_bytes(key.verifying_key().to_bytes()), + signature: Ed25519Signature::from_bytes( + key.sign(&signed_registration_message(nonce, origin)) + .to_bytes(), + ), + display_name: Some(DisplayName::new("worker").unwrap()), + metadata: Metadata::default(), + implementation: ImplementationInfo::new("lightspeed-envd") + .and_then(|info| info.with_version("0.1.0")) + .and_then(|info| info.with_git_sha("1111111111111111111111111111111111111111")) + .and_then(|info| info.with_target("x86_64-unknown-linux-musl")) + .unwrap(), } } + fn register(key: &SigningKey, secret: Option<&str>) -> RegisterParams { + register_signed(key, &ChallengeNonce::from_bytes([1; 32]), &origin(), secret) + } + async fn store_with_key( id: &str, policy: RegistrationKeyPolicy, @@ -916,12 +1848,12 @@ mod tests { .expect("mint"); store .create_registration_key(CreateEnvironmentRegistrationKey { - secret_hash: minted.secret_hash.clone(), + secret_hash: minted.secret.secret_hash.clone(), record: minted.record.clone(), }) .await .expect("create key"); - (store, minted.secret.expose().to_owned()) + (store, minted.secret.secret.expose().to_owned()) } fn policy(mode: RegisteredIdentityMode) -> RegistrationKeyPolicy { @@ -930,122 +1862,382 @@ mod tests { identity_mode: mode, max_active_environments: None, ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, expires_at_ms: None, } } #[test] - fn register_requests_are_verified_against_the_nonce_and_domain() { + fn signatures_bind_the_nonce_and_the_gateway_origin() { let key = daemon(1); - let nonce = [5u8; 32]; - assert!(verify_register_request(®ister(&key, &nonce, None), &nonce).is_ok()); - let replayed = verify_register_request(®ister(&key, &nonce, None), &[6u8; 32]); + let nonce = ChallengeNonce::from_bytes([5; 32]); + let params = register_signed(&key, &nonce, &origin(), None); + assert!(verify_register_request(¶ms, &nonce, &[origin()]).is_ok()); + + let replayed = + verify_register_request(¶ms, &ChallengeNonce::from_bytes([6; 32]), &[origin()]) + .unwrap_err(); + assert!(matches!( + replayed.data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::InvalidSignature) + } + )); + + // A nonce relayed from another gateway was signed for that origin. + let elsewhere = canonical_origin("wss://other.example").unwrap(); + let relayed = register_signed(&key, &nonce, &elsewhere, None); assert_eq!( - replayed.unwrap_err().code, - RegistrationRejectionCode::InvalidSignature + verify_register_request(&relayed, &nonce, &[origin()]) + .unwrap_err() + .kind(), + ProtocolErrorKind::PermissionDenied ); - let mut wrong_key = register(&key, &nonce, None); - wrong_key.daemon_public_key = encode_hex(daemon(2).verifying_key().as_bytes()); + assert!(verify_register_request(&relayed, &nonce, &[origin(), elsewhere]).is_ok()); + + let mut wrong_key = params.clone(); + wrong_key.daemon_public_key = + DaemonPublicKey::from_bytes(daemon(2).verifying_key().to_bytes()); assert_eq!( - verify_register_request(&wrong_key, &nonce) + verify_register_request(&wrong_key, &nonce, &[origin()]) .unwrap_err() - .code, - RegistrationRejectionCode::InvalidSignature + .kind(), + ProtocolErrorKind::PermissionDenied ); - let mut old = register(&key, &nonce, None); - old.protocol_version = 0; + } + + fn register_value() -> serde_json::Value { + serde_json::to_value(register(&daemon(1), None)).unwrap() + } + + #[test] + fn register_decoding_checks_the_version_first_and_refuses_out_of_bounds_facts() { + assert!(decode_register(®ister_value()).is_ok()); + + // A future daemon with members this gateway does not know still + // learns the version it must speak. + let mut future = register_value(); + future["protocolVersion"] = json!(ENVD_PROTOCOL_VERSION + 1); + future["newMember"] = json!(true); + let mismatch = decode_register(&future).unwrap_err(); assert_eq!( - verify_register_request(&old, &nonce).unwrap_err().code, - RegistrationRejectionCode::UnsupportedProtocol + mismatch.data, + ErrorData::VersionMismatch { + expected: ENVD_PROTOCOL_VERSION, + received: ENVD_PROTOCOL_VERSION + 1, + } ); - let mut reserved = register(&key, &nonce, None); - reserved - .metadata - .insert("lightspeed.x".to_owned(), "y".to_owned()); + + let mut poisoned = register_value(); + poisoned["implementation"]["version"] = json!("x".repeat(300)); assert_eq!( - verify_register_request(&reserved, &nonce).unwrap_err().code, - RegistrationRejectionCode::InvalidRequest + decode_register(&poisoned).unwrap_err().kind(), + ProtocolErrorKind::InvalidRequest ); + let mut reserved = register_value(); + reserved["metadata"] = json!({"lightspeed.x": "y"}); + assert_eq!( + decode_register(&reserved).unwrap_err().kind(), + ProtocolErrorKind::InvalidRequest + ); + + // Hex that is not canonical, or not ASCII, never panics: it is an + // ordinary invalidParams before anything else runs. + for key in [ + "AB".repeat(32), + format!("{}\u{e9}", "a".repeat(62)), + "+1".repeat(32), + ] { + let mut value = register_value(); + value["daemonPublicKey"] = json!(key); + assert_eq!( + decode_register(&value).unwrap_err().kind(), + ProtocolErrorKind::InvalidParams + ); + } + let mut data_url = register_value(); + data_url["dataUrl"] = json!("wss://elsewhere.example/data"); + assert_eq!( + decode_register(&data_url).unwrap_err().kind(), + ProtocolErrorKind::InvalidParams + ); + } + + #[test] + fn the_failure_limiter_keys_on_identities_and_hints_a_retry() { + let start = Instant::now(); + let mut limiter = FailureLimiter::default(); + let first = DaemonPublicKey::from_bytes([1; 32]); + let second = DaemonPublicKey::from_bytes([2; 32]); + for _ in 0..MAX_FAILURES_PER_WINDOW { + assert_eq!(limiter.retry_after(&first, start), None); + limiter.record(first, start); + } + let hint = limiter.retry_after(&first, start).expect("over the limit"); + assert!(hint <= FAILURE_WINDOW && hint > Duration::ZERO); + assert_eq!(limiter.retry_after(&second, start), None); + assert_eq!( + limiter.retry_after(&first, start + FAILURE_WINDOW), + None, + "the window slides" + ); + + // Expired entries are pruned on the next insert. + limiter.record(second, start + FAILURE_WINDOW); + assert_eq!(limiter.len(), 1); + } + + #[test] + fn the_failure_limiter_is_capped() { + let now = Instant::now(); + let mut limiter = FailureLimiter::default(); + for index in 0..=MAX_LIMITED_IDENTITIES { + let mut bytes = [0u8; 32]; + bytes[..8].copy_from_slice(&(index as u64).to_le_bytes()); + limiter.record( + DaemonPublicKey::from_bytes(bytes), + now + Duration::from_millis(index as u64), + ); + } + assert_eq!(limiter.len(), MAX_LIMITED_IDENTITIES); + assert!( + !limiter + .identities + .contains_key(&DaemonPublicKey::from_bytes([0; 32])), + "the identity whose last failure is oldest is forgotten first" + ); + } + + #[test] + fn only_credential_refusals_count_against_an_identity() { + assert!(counts_as_failure(&unknown_registration_key())); + assert!(counts_as_failure(&unknown_daemon())); + assert!(counts_as_failure(&ProtocolError::expired("x"))); + assert!(!counts_as_failure(®istration_unavailable())); + assert!(!counts_as_failure(&ProtocolError::busy( + Some(BusyReason::CapacityExhausted), + None, + "x" + ))); } #[tokio::test(flavor = "current_thread")] async fn admission_creates_once_reconnects_by_identity_and_spends_closed_identities() { let (store, secret) = store_with_key("rk", policy(RegisteredIdentityMode::Ephemeral)).await; let key = daemon(3); - let nonce = [1u8; 32]; - let unknown = admit_registration(&store, ®ister(&key, &nonce, None), 2_000) + let unknown = admit_registration(&store, ®ister(&key, None), 2_000) .await .unwrap_err(); - assert_eq!(unknown.code, RegistrationRejectionCode::UnknownDaemon); + assert!(matches!( + unknown.data, + ErrorData::NotFound { + reason: Some(NotFoundReason::UnknownDaemon) + } + )); - let created = admit_registration(&store, ®ister(&key, &nonce, Some(&secret)), 2_000) + let created = admit_registration(&store, ®ister(&key, Some(&secret)), 2_000) .await .expect("created"); - assert!(created.created); - assert_eq!(created.environment.status, EnvironmentStatus::Ready); - assert_eq!( - created - .environment - .metadata - .get("lightspeed.envd.version") - .map(String::as_str), - Some("0.1.0") - ); - assert_eq!( - created - .environment - .metadata - .get("lightspeed.envd.gitSha") - .map(String::as_str), - Some("1111111111111111111111111111111111111111") + assert_eq!(created.admission, Admission::Created); + assert_eq!(created.environment.state, EnvironmentState::Open); + assert_eq!(created.environment.request_id, None); + let source = created.environment.registered().expect("registered"); + assert!(source.connected); + assert_eq!(source.connection_changed_at_ms, 2_000); + assert_eq!(source.last_seen_at_ms, 2_000); + let build = created.environment.daemon_build.as_ref().expect("build"); + assert_eq!(build.version.as_deref(), Some("0.1.0")); + assert_eq!(build.protocol_version, ENVD_PROTOCOL_VERSION); + assert!( + created.environment.metadata.is_empty(), + "build facts are columns, not metadata" ); + + let connection = ControlConnectionId::new("connection_1").unwrap(); + let receipt = + registration_receipt(&created.environment, &connection, created.admission).unwrap(); + assert_eq!(receipt.identity_mode, RegistrationIdentityMode::Ephemeral); + assert_eq!(receipt.admission, Admission::Created); + assert_eq!(receipt.control_connection_id, connection); assert_eq!( - created - .environment - .metadata - .get("lightspeed.envd.protocolVersion") - .map(String::as_str), - Some(CURRENT_PROTOCOL_VERSION.to_string().as_str()) + receipt.environment_id.as_str(), + created.environment.environment_id.as_str() ); + assert_eq!(receipt.daemon_id.as_str(), source.daemon_id().as_str()); + assert!(receipt.daemon_id.as_str().starts_with("daemon_")); - // Reconnects ignore the key entirely, even a wrong one. - let reconnected = - admit_registration(&store, ®ister(&key, &nonce, Some("lsrk_wrong")), 3_000) - .await - .expect("reconnect"); - assert!(!reconnected.created); + // A dropped connection, then a reconnect that ignores the key + // entirely, even a wrong one, and records the connection again. + store + .mark_registered_disconnected(MarkRegisteredDisconnected { + environment_id: created.environment.environment_id.clone(), + at_ms: 2_500, + }) + .await + .expect("disconnect"); + let plan = plan_admission(&store, ®ister(&key, Some("lsrk_wrong")), 3_000) + .await + .expect("plan"); + assert!(matches!(plan, AdmissionPlan::Reconnect { .. })); + assert_eq!(plan.environment_id(), &created.environment.environment_id); + let reconnected = commit_admission(&store, plan, 3_000) + .await + .expect("reconnect"); + assert_eq!(reconnected.admission, Admission::Reconnected); assert_eq!( reconnected.environment.environment_id, created.environment.environment_id ); + let source = reconnected.environment.registered().expect("registered"); + assert!(source.connected); + assert_eq!(source.connection_changed_at_ms, 3_000); + // Closing a registered environment is immediate and spends the + // identity for good. store .begin_close_environment(BeginCloseEnvironment { environment_id: created.environment.environment_id.clone(), - updated_at_ms: 4_000, + reason: EnvironmentCloseReason::Requested, + at_ms: 4_000, }) .await .expect("close"); - let closing = admit_registration(&store, ®ister(&key, &nonce, Some(&secret)), 4_100) + let closed = admit_registration(&store, ®ister(&key, Some(&secret)), 4_100) .await .unwrap_err(); - assert_eq!(closing.code, RegistrationRejectionCode::EnvironmentClosed); + assert!(matches!( + closed.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::EnvironmentClosed + } + )); + assert!(!closed.is_retryable()); + } + + #[tokio::test(flavor = "current_thread")] + async fn a_reconnect_planned_before_a_close_is_refused_as_closed() { + let (store, secret) = + store_with_key("rk", policy(RegisteredIdentityMode::Persistent)).await; + let key = daemon(9); + let created = admit_registration(&store, ®ister(&key, Some(&secret)), 2_000) + .await + .expect("created"); + let plan = plan_admission(&store, ®ister(&key, None), 3_000) + .await + .expect("plan"); store - .finish_close_environment(FinishCloseEnvironment { + .begin_close_environment(BeginCloseEnvironment { environment_id: created.environment.environment_id.clone(), - observed_at_ms: 4_200, + reason: EnvironmentCloseReason::Requested, + at_ms: 3_100, }) .await - .expect("finish"); - let closed = admit_registration(&store, ®ister(&key, &nonce, Some(&secret)), 4_300) - .await - .unwrap_err(); - assert_eq!(closed.code, RegistrationRejectionCode::EnvironmentClosed); - assert!(closed.code.is_terminal()); + .expect("close"); + let refused = commit_admission(&store, plan, 3_200).await.unwrap_err(); + assert!(matches!( + refused.data, + ErrorData::FailedPrecondition { + reason: PreconditionReason::EnvironmentClosed + } + )); + } + + #[tokio::test(flavor = "current_thread")] + async fn only_an_ephemeral_departure_closes_its_environment() { + let (ephemeral_store, ephemeral_secret) = + store_with_key("rk-e", policy(RegisteredIdentityMode::Ephemeral)).await; + let ephemeral = admit_registration( + &ephemeral_store, + ®ister(&daemon(10), Some(&ephemeral_secret)), + 1_000, + ) + .await + .expect("ephemeral"); + assert!(departure_closes(&ephemeral.environment)); + let (persistent_store, persistent_secret) = + store_with_key("rk-p", policy(RegisteredIdentityMode::Persistent)).await; + let persistent = admit_registration( + &persistent_store, + ®ister(&daemon(11), Some(&persistent_secret)), + 1_000, + ) + .await + .expect("persistent"); + assert!(!departure_closes(&persistent.environment)); + } + + #[tokio::test(flavor = "current_thread")] + async fn connection_transitions_of_one_environment_run_one_at_a_time() { + let registrations = Arc::new(RegisteredConnections::new()); + let universe_id = Uuid::nil(); + let held = registrations.transition(universe_id, "env").await; + // Another environment's transition is independent. + drop(registrations.transition(universe_id, "other").await); + let waiting = { + let registrations = registrations.clone(); + tokio::spawn(async move { + drop(registrations.transition(universe_id, "env").await); + }) + }; + tokio::task::yield_now().await; + assert!( + !waiting.is_finished(), + "a second transition waits for the first" + ); + drop(held); + waiting.await.unwrap(); + assert!( + registrations.transitions.lock().unwrap().is_empty(), + "finished transitions leave no lock behind" + ); + } + + #[test] + fn a_failed_admission_hands_the_environment_back_to_the_displaced_connection() { + let registrations = RegisteredConnections::new(); + let universe_id = Uuid::nil(); + let handle = |id: &str| { + let (commands, _) = mpsc::channel(1); + let (supersede, _) = oneshot::channel(); + let (ended, _) = watch::channel(()); + ControlHandle { + connection_id: ControlConnectionId::new(id).unwrap(), + commands, + supersede, + ended, + } + }; + let old = ControlConnectionId::new("connection_old").unwrap(); + let new = ControlConnectionId::new("connection_new").unwrap(); + assert!( + registrations + .install(universe_id, "env".to_owned(), handle("connection_old")) + .is_none() + ); + let previous = + registrations.install(universe_id, "env".to_owned(), handle("connection_new")); + let ended = previous.as_ref().expect("displaced").ended.subscribe(); + assert_eq!(registrations.current(universe_id, "env"), Some(new.clone())); + assert!(registrations.uninstall(universe_id, "env", &new, previous)); + assert_eq!(registrations.current(universe_id, "env"), Some(old.clone())); + assert!( + ended.has_changed().is_ok(), + "the restored handle keeps its routes" + ); + assert!( + !registrations.remove_if_current(universe_id, "env", &new), + "a connection that is no longer current removes nothing" + ); + assert!(registrations.remove_if_current(universe_id, "env", &old)); + assert_eq!(registrations.current(universe_id, "env"), None); + assert!( + ended.has_changed().is_err(), + "dropping a handle ends the routes paired under it" + ); } #[tokio::test(flavor = "current_thread")] - async fn admission_maps_key_policy_refusals_to_typed_codes() { + async fn admission_maps_key_policy_refusals_to_typed_kinds() { let (store, secret) = store_with_key( "rk", RegistrationKeyPolicy { @@ -1054,24 +2246,30 @@ mod tests { }, ) .await; - let nonce = [2u8; 32]; - admit_registration(&store, ®ister(&daemon(4), &nonce, Some(&secret)), 2_000) + admit_registration(&store, ®ister(&daemon(4), Some(&secret)), 2_000) .await .expect("first"); - let full = admit_registration(&store, ®ister(&daemon(5), &nonce, Some(&secret)), 2_100) + let full = admit_registration(&store, ®ister(&daemon(5), Some(&secret)), 2_100) .await .unwrap_err(); - assert_eq!(full.code, RegistrationRejectionCode::CapacityExhausted); - assert!(!full.code.is_terminal()); + assert!(matches!( + full.data, + ErrorData::Busy { + reason: Some(BusyReason::CapacityExhausted), + .. + } + )); + assert!(full.is_retryable()); - let bad = admit_registration( - &store, - ®ister(&daemon(6), &nonce, Some("lsrk_nope")), - 2_200, - ) - .await - .unwrap_err(); - assert_eq!(bad.code, RegistrationRejectionCode::InvalidRegistrationKey); + let bad = admit_registration(&store, ®ister(&daemon(6), Some("lsrk_nope")), 2_200) + .await + .unwrap_err(); + assert!(matches!( + bad.data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::InvalidRegistrationKey) + } + )); store .revoke_registration_key(RevokeEnvironmentRegistrationKey { @@ -1080,19 +2278,20 @@ mod tests { }) .await .expect("revoke"); - let revoked = - admit_registration(&store, ®ister(&daemon(7), &nonce, Some(&secret)), 3_100) - .await - .unwrap_err(); - assert_eq!( - revoked.code, - RegistrationRejectionCode::RegistrationKeyRevoked - ); + let revoked = admit_registration(&store, ®ister(&daemon(7), Some(&secret)), 3_100) + .await + .unwrap_err(); + assert!(matches!( + revoked.data, + ErrorData::PermissionDenied { + reason: Some(PermissionReason::RegistrationKeyRevoked) + } + )); // The already registered daemon still reconnects after revocation. - let reconnect = admit_registration(&store, ®ister(&daemon(4), &nonce, None), 3_200) + let reconnect = admit_registration(&store, ®ister(&daemon(4), None), 3_200) .await .expect("reconnect"); - assert!(!reconnect.created); + assert_eq!(reconnect.admission, Admission::Reconnected); let (expired_store, expired_secret) = store_with_key( "rk-exp", @@ -1104,36 +2303,112 @@ mod tests { .await; let expired = admit_registration( &expired_store, - ®ister(&daemon(8), &nonce, Some(&expired_secret)), + ®ister(&daemon(8), Some(&expired_secret)), 2_000, ) .await .unwrap_err(); + assert_eq!(expired.kind(), ProtocolErrorKind::Expired); + } + + fn text(value: serde_json::Value) -> Message { + Message::Text(value.to_string().into()) + } + + #[test] + fn before_admission_only_register_is_served() { + let register = text(json!({ + "jsonrpc": "2.0", "id": 1, "method": "registration/register", "params": {} + })); + assert!(matches!( + classify_pre_admission(®ister), + PreAdmission::Register(_) + )); + let other = text(json!({"jsonrpc": "2.0", "id": 2, "method": "env/getIdle"})); + let PreAdmission::AnswerAndClose(response) = classify_pre_admission(&other) else { + panic!("a request other than register is refused and closes"); + }; assert_eq!( - expired.code, - RegistrationRejectionCode::RegistrationKeyExpired + response.result.unwrap_err().kind(), + ProtocolErrorKind::NotInitialized ); + let goodbye = text(json!({ + "jsonrpc": "2.0", "method": "registration/goodbye", "params": {"reason": "shutdown"} + })); + assert_eq!(classify_pre_admission(&goodbye), PreAdmission::Close); + let PreAdmission::Answer(response) = + classify_pre_admission(&Message::Text("not json".into())) + else { + panic!("a parse error is answered and the socket stays open"); + }; + assert_eq!(response.id, None); + assert_eq!( + response.result.unwrap_err().kind(), + ProtocolErrorKind::ParseError + ); + assert!(matches!( + classify_pre_admission(&Message::Binary(vec![1_u8].into())), + PreAdmission::Answer(_) + )); + let stray = text(json!({"jsonrpc": "2.0", "id": "g9", "result": {}})); + assert_eq!(classify_pre_admission(&stray), PreAdmission::Ignore); } #[test] - fn data_url_is_the_public_websocket_base_plus_the_data_path() { + fn after_admission_stream_answers_and_goodbye_are_recognized() { + assert_eq!( + classify_admitted(&json!({"jsonrpc": "2.0", "id": "g1", "result": {}}).to_string()), + Admitted::StreamAnswer(Id::String("g1".to_owned()), Ok(())) + ); + let Admitted::StreamAnswer(_, Err(error)) = classify_admitted( + &jsonrpc::Response::failure( + Some(Id::String("g2".to_owned())), + ProtocolError::unavailable("dial failed"), + ) + .to_text(), + ) else { + panic!("a failed dial is an error answer"); + }; + assert_eq!(error.kind(), ProtocolErrorKind::Unavailable); assert_eq!( - data_url("https://lightspeed.example/"), - "wss://lightspeed.example/environment-gateway/data" + classify_admitted( + &json!({ + "jsonrpc": "2.0", "method": "registration/goodbye", + "params": {"reason": "shutdown"} + }) + .to_string() + ), + Admitted::Goodbye ); + let Admitted::Answer(again) = classify_admitted( + &json!({"jsonrpc": "2.0", "id": 1, "method": "registration/register"}).to_string(), + ) else { + panic!("a second register is answered"); + }; assert_eq!( - data_url("http://127.0.0.1:18080"), - "ws://127.0.0.1:18080/environment-gateway/data" + again.result.unwrap_err().kind(), + ProtocolErrorKind::InvalidRequest ); + let Admitted::Answer(unknown) = classify_admitted( + &json!({"jsonrpc": "2.0", "id": 3, "method": "registration/unknown"}).to_string(), + ) else { + panic!("an unknown request is answered"); + }; + assert_eq!( + unknown.result.unwrap_err().kind(), + ProtocolErrorKind::MethodNotFound + ); + assert!(matches!( + classify_admitted(&json!({"jsonrpc": "2.0", "method": "x/y"}).to_string()), + Admitted::Ignore(_) + )); } #[test] - fn failure_accounting_rate_limits_a_subject() { - let registrations = RegisteredConnections::new(); - for _ in 0..MAX_FAILURES_PER_WINDOW { - registrations.record_failure("key:abc"); - } - assert!(registrations.over_failure_limit("key:abc")); - assert!(!registrations.over_failure_limit("key:other")); + fn the_liveness_deadline_is_three_heartbeats() { + assert_eq!( + LIVENESS_TIMEOUT, + HEARTBEAT_INTERVAL * HEARTBEAT_MISSED_INTERVALS + ); } } diff --git a/crates/temporal-runtime/src/gateway/request_context.rs b/crates/temporal-runtime/src/gateway/request_context.rs index 4349766e8..cb6a0165d 100644 --- a/crates/temporal-runtime/src/gateway/request_context.rs +++ b/crates/temporal-runtime/src/gateway/request_context.rs @@ -121,7 +121,7 @@ pub fn initialize_response(caller: CallerAccess) -> api::AgentApiOutcome AgentApiError { + use crate::environments::resolver::EnvironmentResolveError; match error { - crate::environments::resolver::EnvironmentResolveError::Store(error) => { - map_environments_error(error) - } - not_ready @ crate::environments::resolver::EnvironmentResolveError::NotReady { .. } => { - AgentApiError::environment_not_ready(not_ready.to_string()) - } + EnvironmentResolveError::Store(error) => map_environments_error(error), + EnvironmentResolveError::NotReady { + environment_id, + reason, + retry_after_ms, + .. + } => AgentApiError::environment_not_ready(format!( + "environment {environment_id} is not ready ({reason}); retry after {retry_after_ms} ms" + )), + EnvironmentResolveError::Unreachable { + environment_id, + status, + } => AgentApiError::environment_not_ready(format!( + "environment {environment_id} is unreachable ({status}); retry with backoff" + )), other => AgentApiError::rejected(other.to_string()), } } @@ -89,3 +100,26 @@ pub(super) fn activate_environment_command( pub(super) fn deactivate_environment_command() -> CoreAgentCommand { CoreAgentCommand::ClearActiveEnvironment } + +#[cfg(test)] +mod tests { + use api::AgentApiErrorKind; + + use super::map_environment_resolve_error; + use crate::environments::resolver::EnvironmentResolveError; + + #[test] + fn an_unreachable_environment_is_not_ready_and_a_failed_one_is_rejected() { + let unreachable = map_environment_resolve_error(EnvironmentResolveError::Unreachable { + environment_id: "env-1".to_owned(), + status: "connect refused".to_owned(), + }); + assert_eq!(unreachable.kind, AgentApiErrorKind::EnvironmentNotReady); + + let failed = map_environment_resolve_error(EnvironmentResolveError::Failed { + environment_id: "env-1".to_owned(), + message: "boot failed".to_owned(), + }); + assert_eq!(failed.kind, AgentApiErrorKind::Rejected); + } +} diff --git a/crates/temporal-runtime/src/gateway/service/mod.rs b/crates/temporal-runtime/src/gateway/service/mod.rs index 795596b23..f692ae908 100644 --- a/crates/temporal-runtime/src/gateway/service/mod.rs +++ b/crates/temporal-runtime/src/gateway/service/mod.rs @@ -10,7 +10,6 @@ pub(crate) mod channels_api; mod common; mod controller; pub(crate) use crate::environments::lifecycle as environment_lifecycle; -pub(crate) use crate::environments::power as environment_power; pub(crate) use crate::environments::providers as environment_providers; mod environments; mod errors; @@ -46,7 +45,6 @@ use catalogs::parse_client_context_key; use common::now_ms; pub use environment_lifecycle::ReconcileFailureLog; use environment_lifecycle::parse_registry_environment_id; -pub use environment_power::PowerReaperStats; use environment_providers::map_environments_error; use environments::{activate_environment_command, deactivate_environment_command}; use errors::*; @@ -182,8 +180,8 @@ fn env_flag(name: &str) -> bool { pub(super) fn validate_caller_metadata( metadata: &BTreeMap, ) -> Result<(), AgentApiError> { - environment_protocol::registration::validate_registration_metadata(None, metadata) - .map_err(|message| AgentApiError::invalid_request(format!("invalid metadata: {message}"))) + environment_protocol::metadata::validate_caller_metadata(metadata) + .map_err(|error| AgentApiError::invalid_request(format!("invalid metadata: {error}"))) } fn validate_delete_after_close_ms(value: Option) -> Result<(), AgentApiError> { @@ -566,6 +564,7 @@ pub struct GatewayAgentApiBuilder { model_discovery_anthropic: Option>, provider_controller_connector: Arc, environment_gateway: crate::environments::gateway::EnvironmentGatewayClientConfig, + environment_signals: Arc, } impl GatewayAgentApiBuilder { @@ -649,6 +648,15 @@ impl GatewayAgentApiBuilder { self } + /// The process's environment signals; a fresh instance by default. + pub fn with_environment_signals( + mut self, + signals: Arc, + ) -> Self { + self.environment_signals = signals; + self + } + pub fn with_continue_as_new_history_threshold(mut self, threshold: u32) -> Self { self.continue_as_new_history_threshold = Some(threshold); self @@ -772,6 +780,7 @@ impl GatewayAgentApiBuilder { model_discovery, provider_controller_connector: self.provider_controller_connector, environment_gateway: self.environment_gateway, + environment_signals: self.environment_signals, } } } @@ -799,6 +808,7 @@ pub struct GatewayAgentApi { model_discovery: ModelDiscoveryService, provider_controller_connector: Arc, pub(crate) environment_gateway: crate::environments::gateway::EnvironmentGatewayClientConfig, + environment_signals: Arc, } impl GatewayAgentApi { @@ -830,8 +840,9 @@ impl GatewayAgentApi { github_api_client: None, model_discovery_openai: None, model_discovery_anthropic: None, - provider_controller_connector: Arc::new(WebSocketProviderControllerConnector::default()), + provider_controller_connector: Arc::new(WebSocketProviderControllerConnector), environment_gateway, + environment_signals: Arc::default(), } } @@ -3629,6 +3640,21 @@ impl AgentApiService for GatewayAgentApi { .map(AgentApiOutcome::new) } + async fn update_environment( + &self, + params: EnvironmentUpdateParams, + ) -> Result, AgentApiError> { + self.authorize_method( + METHOD_ENVIRONMENTS_UPDATE, + Some(ResourceRef::Environment(params.environment_id.clone())), + ) + .await?; + self.environment_service() + .update_environment_record(params) + .await + .map(AgentApiOutcome::new) + } + async fn list_environments( &self, params: EnvironmentListParams, @@ -3698,6 +3724,30 @@ impl AgentApiService for GatewayAgentApi { .map(AgentApiOutcome::new) } + async fn update_environment_registration_key( + &self, + params: EnvironmentRegistrationKeyUpdateParams, + ) -> Result, AgentApiError> { + self.authorize_method(METHOD_ENVIRONMENTS_REGISTRATION_KEYS_UPDATE, None) + .await?; + self.environment_service() + .update_environment_registration_key_record(params) + .await + .map(AgentApiOutcome::new) + } + + async fn rotate_environment_registration_key( + &self, + params: EnvironmentRegistrationKeyRotateParams, + ) -> Result, AgentApiError> { + self.authorize_method(METHOD_ENVIRONMENTS_REGISTRATION_KEYS_ROTATE, None) + .await?; + self.environment_service() + .rotate_environment_registration_key_record(params) + .await + .map(AgentApiOutcome::new) + } + async fn revoke_environment_registration_key( &self, params: EnvironmentRegistrationKeyRevokeParams, @@ -3737,20 +3787,6 @@ impl AgentApiService for GatewayAgentApi { .map(AgentApiOutcome::new) } - async fn put_environment_idle_policy( - &self, - params: EnvironmentIdlePolicyPutParams, - ) -> Result, AgentApiError> { - self.authorize_method( - METHOD_ENVIRONMENTS_IDLE_POLICY_PUT, - Some(ResourceRef::Environment(params.environment_id.clone())), - ) - .await?; - self.put_environment_idle_policy_record(params) - .await - .map(AgentApiOutcome::new) - } - async fn activate_session_environment( &self, params: SessionEnvironmentActivateParams, @@ -4853,10 +4889,6 @@ pub(crate) fn allocate_environment_id_public() -> ::environments::EnvironmentId environment_lifecycle::allocate_environment_id() } -pub(crate) fn allocate_incarnation_id_public() -> ::environments::EnvironmentIncarnationId { - environment_lifecycle::allocate_incarnation_id() -} - #[cfg(test)] mod tests; @@ -4870,10 +4902,15 @@ impl GatewayAgentApi { pub(crate) fn environment_service(&self) -> crate::environments::EnvironmentService { crate::environments::EnvironmentService { store: self.store.clone(), - environment_gateway: self.environment_gateway.clone(), provider_controller_connector: self.provider_controller_connector.clone(), + signals: self.environment_signals.clone(), } } + + /// The process's environment signals. + pub fn environment_signals(&self) -> &Arc { + &self.environment_signals + } pub(super) async fn put_environment_ingress_record( &self, params: EnvironmentIngressPutParams, @@ -4910,15 +4947,6 @@ impl GatewayAgentApi { .await } - pub(super) async fn put_environment_idle_policy_record( - &self, - params: EnvironmentIdlePolicyPutParams, - ) -> Result { - self.environment_service() - .put_environment_idle_policy_record(params) - .await - } - pub(super) async fn list_environment_records( &self, params: EnvironmentListParams, @@ -4943,12 +4971,6 @@ impl GatewayAgentApi { .await } - pub async fn reap_idle_environments_once(&self) -> Result { - self.environment_service() - .reap_idle_environments_once() - .await - } - pub(super) async fn create_environment_registration_key_record( &self, params: EnvironmentRegistrationKeyCreateParams, diff --git a/crates/temporal-runtime/src/gateway/service/session_jobs.rs b/crates/temporal-runtime/src/gateway/service/session_jobs.rs index 06a74520d..0f0abf97a 100644 --- a/crates/temporal-runtime/src/gateway/service/session_jobs.rs +++ b/crates/temporal-runtime/src/gateway/service/session_jobs.rs @@ -2,29 +2,40 @@ use super::*; use ::environments::{EnvironmentId, EnvironmentJobGroupId, EnvironmentRecord}; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; -use environment_client::{EnvironmentClientError, EnvironmentDataClient, WebSocketConnectOptions}; +use environment_client::{ClientError, DataClient}; use environment_protocol::{ data::{ - handshake::{ - InitializeParams as DataInitializeParams, InitializedParams as DataInitializedParams, + exec::{ + ExitStatus as ProtocolExitStatus, GapReason as ProtocolGapReason, + OutputChunk as ProtocolOutputChunk, OutputGap as ProtocolOutputGap, + OutputStream as ProtocolOutputStream, ReturnOn as ProtocolReturnOn, + StoppedBy as ProtocolStoppedBy, }, - jobs::{ - CancelJobsParams as ProtocolCancelJobsParams, JobArtifact as ProtocolJobArtifact, - JobCancelScope as ProtocolJobCancelScope, JobDependency as ProtocolJobDependency, - JobDependencyPolicy as ProtocolJobDependencyPolicy, - JobOutputChunk as ProtocolJobOutputChunk, JobOutputStream as ProtocolJobOutputStream, - JobReadResult as ProtocolJobReadResult, JobStartSpec as ProtocolJobStartSpec, + handshake::JobLimits, + job::{ + CancelJobsParams as ProtocolCancelJobsParams, JobCancelScope as ProtocolJobCancelScope, + JobReadEntry as ProtocolJobReadEntry, JobReadRequest as ProtocolJobReadRequest, JobStatus as ProtocolJobStatus, JobSummary as ProtocolJobSummary, - ReadJobsParams as ProtocolReadJobsParams, StartJobsParams as ProtocolStartJobsParams, + JobSummaryEntry as ProtocolJobSummaryEntry, ReadJobsParams as ProtocolReadJobsParams, }, }, - error::EnvironmentProtocolErrorCode, - shared::{ - ByteChunk, CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities, EnvironmentDataConnection, - EnvironmentPath, EnvironmentTransport, JobId, - }, + error::ProtocolErrorKind, + ids::{JobId, JobName, JobRequestId, QueueKey}, + paths::EnvironmentPath, +}; +use temporal_workflow::{ + EnvironmentJobCommand, EnvironmentJobDependency, EnvironmentJobDependencyPolicy, + EnvironmentJobId, EnvironmentJobRecord, EnvironmentJobSpec, EnvironmentJobStartPayloadV1, + EnvironmentJobStatus, }; -use harness::validate_general_string_id; +use tools::environment::jobs::DEFAULT_JOB_OUTPUT_BYTES; + +/// `User-Agent` of the job API's worker-route upgrades; `clientInfo` is +/// always the runtime build. +const JOB_API_USER_AGENT: &str = "lightspeed-temporal-runtime"; + +/// Deadline of each data-plane call the job API makes. +const JOB_API_REQUEST_TIMEOUT: Duration = Duration::from_secs(60); impl GatewayAgentApi { pub(super) async fn create_environment_job_records( @@ -32,19 +43,16 @@ impl GatewayAgentApi { params: EnvironmentJobCreateParams, ) -> Result { let environment_id = parse_environment_job_environment_id(params.environment_id)?; - self.start_environment_jobs(environment_id, params.request_id, params.jobs, None) - .await - } - - async fn start_environment_jobs( - &self, - environment_id: EnvironmentId, - request_id: String, - jobs: Vec, - default_cwd: Option, - ) -> Result { - let mut protocol_specs = Vec::with_capacity(jobs.len()); - for (index, spec) in jobs.into_iter().enumerate() { + let request_id = JobRequestId::new(params.request_id).map_err(|error| { + AgentApiError::invalid_request(format!("invalid request_id: {error}")) + })?; + if params.jobs.is_empty() { + return Err(AgentApiError::invalid_request( + "environments/jobs/create requires at least one job", + )); + } + let mut specs = Vec::with_capacity(params.jobs.len()); + for (index, spec) in params.jobs.into_iter().enumerate() { let job_id = spec .job_id .as_deref() @@ -52,68 +60,42 @@ impl GatewayAgentApi { .transpose() .map_err(AgentApiError::invalid_request)? .unwrap_or_else(|| derived_job_id(&environment_id, &request_id, index)); - let mut spec = - api_start_spec_to_protocol(spec, job_id).map_err(AgentApiError::invalid_request)?; - if spec.cwd.is_none() { - spec.cwd = default_cwd.clone(); - } - protocol_specs.push(spec); + specs.push( + api_start_spec_to_workflow(spec, job_id).map_err(AgentApiError::invalid_request)?, + ); } let instance = self.read_job_instance(&environment_id).await?; - self.start_environment_jobs_protocol_specs(instance, request_id, protocol_specs) + // Refuse what the daemon would refuse before a workflow exists for + // it: a daemon without jobs, or more jobs than one start takes. + let client = self.connect_job_client(&instance).await?; + let limits = client.initialize_response().limits.job.clone(); + let _ = client.close().await; + if specs.len() > limits.max_request_jobs as usize { + return Err(AgentApiError::invalid_request(format!( + "environments/jobs/create names {} jobs; the environment starts at most {} at once", + specs.len(), + limits.max_request_jobs + ))); + } + self.start_environment_jobs(instance, request_id, specs) .await } - async fn start_environment_jobs_protocol_specs( + async fn start_environment_jobs( &self, instance: EnvironmentRecord, - request_id: String, - jobs: Vec, + request_id: JobRequestId, + jobs: Vec, ) -> Result { - validate_job_request_id(&request_id)?; - if jobs.is_empty() { - return Err(AgentApiError::invalid_request( - "environments/jobs/create requires at least one job", - )); - } - let key = self - .environment_gateway - .route_key(self.store.config().universe_id, &instance); - let connection = self.environment_gateway.connection(&key); - let (_client, capabilities) = connect_initialized_environment_data_client( - &connection, - self.environment_gateway - .connect_options("lightspeed-temporal-runtime"), - ) - .await - .map_err(|error| AgentApiError::rejected(error.to_string()))?; - if !capabilities.job_start { - return Err(AgentApiError::rejected(format!( - "environment does not support durable job start: {}", - instance.environment_id - ))); - } - let request = ProtocolStartJobsParams { - namespace: instance.environment_id.as_str().to_owned(), - request_id: request_id.clone(), - jobs, - }; - let request_fingerprint = - BlobRef::from_bytes(&serde_json::to_vec(&request).map_err(|error| { - AgentApiError::internal(format!("encode environment job request: {error}")) - })?); - let job_group_id = derived_job_group_id( - &instance.environment_id, - &request_id, - request_fingerprint.as_str(), - ); - let job_ids = request.jobs.iter().map(|job| job.job_id.clone()).collect(); - let start_payload = temporal_workflow::EnvironmentJobStartPayload { request }; + let payload = EnvironmentJobStartPayloadV1::new(request_id.into_string(), jobs); + let encoded = payload.encode().map_err(|error| { + AgentApiError::internal(format!("encode environment job workflow start: {error}")) + })?; + let job_group_id = + derived_job_group_id(&instance.environment_id, &BlobRef::from_bytes(&encoded)); let request_ref = self .store - .put_bytes(serde_json::to_vec(&start_payload).map_err(|error| { - AgentApiError::internal(format!("encode environment job workflow start: {error}")) - })?) + .put_bytes(encoded) .await .map_err(map_blob_store_error)?; let snapshot = self @@ -124,7 +106,7 @@ impl GatewayAgentApi { job_group_id: job_group_id.as_str().to_owned(), request_ref, }, - job_ids, + payload.job_ids(), Vec::new(), ) .await?; @@ -134,22 +116,7 @@ impl GatewayAgentApi { jobs: snapshot .jobs .into_iter() - .map(|summary| SessionJobStartedView { - name: summary.name, - job_id: summary.job_id.as_str().to_owned(), - handle: SessionJobHandleView { - environment_id: instance.environment_id.as_str().to_owned(), - job_id: summary.job_id.as_str().to_owned(), - }, - promise_id: None, - status: api_job_status(summary.status), - dependencies: summary - .dependencies - .into_iter() - .map(|id| id.as_str().to_owned()) - .collect(), - queue_key: summary.queue_key, - }) + .map(|record| api_started_job(&instance.environment_id, record)) .collect(), }) } @@ -157,7 +124,7 @@ impl GatewayAgentApi { async fn start_environment_job_workflow( &self, start: temporal_workflow::EnvironmentJobStartActivityRequest, - job_ids: Vec, + job_ids: Vec, subscriptions: Vec, ) -> Result { let workflow_id = temporal_workflow::compose_environment_job_workflow_id( @@ -230,166 +197,170 @@ impl GatewayAgentApi { &self, params: EnvironmentJobReadParams, ) -> Result { - let jobs = self - .read_jobs( - params.jobs, - params.output_bytes, - params.after_seq, - params.include_artifacts, - ) - .await?; - Ok(EnvironmentJobReadResponse { jobs }) - } - - async fn read_jobs( - &self, - handles: Vec, - output_bytes: Option, - after_seq: Option, - include_artifacts: bool, - ) -> Result, AgentApiError> { - let mut entries = Vec::with_capacity(handles.len()); - for handle in handles { - let resolved = match parse_job_handle(handle) { - Ok(handle) => handle, - Err(error) => { - entries.push(session_job_read_error(None, error)); - continue; - } - }; - let environment_id = match EnvironmentId::try_new(resolved.environment_id.clone()) { - Ok(environment_id) => environment_id, - Err(error) => { - entries.push(session_job_read_error(Some(resolved), error.to_string())); - continue; - } - }; - let job_id = match parse_protocol_job_id(&resolved.job_id) { - Ok(job_id) => job_id, - Err(error) => { - entries.push(session_job_read_error(Some(resolved), error)); - continue; - } - }; - let mut client = match self - .connect_client_for_job_handle(&environment_id, "read") + if params.after.is_some() && params.tail_bytes.is_some() { + return Err(AgentApiError::invalid_request( + "environments/jobs/read takes after or tailBytes, not both", + )); + } + let position = JobReadPosition { + after: params.after, + tail_bytes: params.tail_bytes, + max_bytes: params.max_bytes.unwrap_or(DEFAULT_JOB_OUTPUT_BYTES), + }; + let mut entries: Vec> = vec![None; params.jobs.len()]; + let mut groups: BTreeMap> = + BTreeMap::new(); + for (index, handle) in params.jobs.into_iter().enumerate() { + match parse_job_handle(handle) { + Ok((environment_id, handle, job_id)) => groups + .entry(environment_id) + .or_default() + .push((index, handle, job_id)), + Err(error) => entries[index] = Some(session_job_read_error(None, error)), + } + } + for (environment_id, jobs) in groups { + match self + .read_jobs_in_environment(&environment_id, &jobs, position) .await { - Ok(client) => client, + Ok(read) => { + for ((index, _, _), entry) in jobs.iter().zip(read) { + entries[*index] = Some(entry); + } + } Err(error) => { - entries.push(session_job_read_error(Some(resolved), error)); - continue; + for (index, handle, _) in jobs { + entries[index] = Some(session_job_read_error(Some(handle), error.clone())); + } + } + } + } + Ok(EnvironmentJobReadResponse { + jobs: entries.into_iter().flatten().collect(), + }) + } + + /// One `job/read` entry per job, in order, within the daemon's ceilings. + async fn read_jobs_in_environment( + &self, + environment_id: &EnvironmentId, + jobs: &[(usize, SessionJobHandleView, JobId)], + position: JobReadPosition, + ) -> Result, String> { + let instance = self + .read_job_instance(environment_id) + .await + .map_err(|error| error.message)?; + let client = self + .connect_job_client(&instance) + .await + .map_err(|error| error.message)?; + let limits = client.initialize_response().limits.job.clone(); + let mut entries = Vec::with_capacity(jobs.len()); + let mut result = Ok(()); + for chunk in jobs.chunks((limits.max_request_jobs as usize).max(1)) { + let params = + job_read_params(chunk.iter().map(|(_, _, job_id)| job_id), position, &limits); + match client.read_jobs(¶ms).await { + Ok(response) => { + for ((_, handle, _), entry) in chunk.iter().zip(response.jobs) { + entries.push(session_job_read_entry(handle.clone(), entry)); + } } - }; - match client - .read_jobs(&ProtocolReadJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: vec![job_id], - after_seq, - max_bytes: output_bytes, - include_artifacts, - wait_ms: None, - }) - .await - { - Ok(response) => entries.push(session_job_read_entry_from_response( - resolved, - response.jobs.into_iter().next(), - )), Err(error) => { - entries.push(session_job_read_error(Some(resolved), error.to_string())) + result = Err(error.to_string()); + break; } } } - Ok(entries) + let _ = client.close().await; + result.map(|()| entries) } pub(super) async fn cancel_environment_job_records( &self, params: EnvironmentJobCancelParams, ) -> Result { - let jobs = self - .cancel_jobs(params.jobs, params.scope, params.force) - .await?; - Ok(EnvironmentJobCancelResponse { jobs }) - } - - async fn cancel_jobs( - &self, - handles: Vec, - scope: SessionJobCancelScopeView, - force: bool, - ) -> Result, AgentApiError> { - let mut entries = Vec::with_capacity(handles.len()); - for handle in handles { - let resolved = match parse_job_handle(handle) { - Ok(handle) => handle, - Err(error) => { - entries.push(session_job_cancel_error(None, error)); - continue; - } - }; - let environment_id = match EnvironmentId::try_new(resolved.environment_id.clone()) { - Ok(environment_id) => environment_id, - Err(error) => { - entries.push(session_job_cancel_error(Some(resolved), error.to_string())); - continue; - } - }; - let job_id = match parse_protocol_job_id(&resolved.job_id) { - Ok(job_id) => job_id, - Err(error) => { - entries.push(session_job_cancel_error(Some(resolved), error)); - continue; - } - }; + let scope = match params.scope { + SessionJobCancelScopeView::Job => ProtocolJobCancelScope::Job, + SessionJobCancelScopeView::Dependents => ProtocolJobCancelScope::Dependents, + }; + let mut entries = Vec::with_capacity(params.jobs.len()); + let mut groups: BTreeMap> = + BTreeMap::new(); + for handle in params.jobs { + match parse_job_handle(handle) { + Ok((environment_id, handle, job_id)) => groups + .entry(environment_id) + .or_default() + .push((handle, job_id)), + Err(error) => entries.push(session_job_cancel_error(None, error)), + } + } + for (environment_id, jobs) in groups { match self - .cancel_job_on_provider(&environment_id, job_id, scope, force) + .cancel_jobs_in_environment(&environment_id, &jobs, scope, params.force) .await { - Ok(summary) => entries.push(SessionJobCancelEntryView { - handle: Some(resolved), - summary: Some(api_job_summary(summary)), - error: None, - }), - Err(error) => entries.push(session_job_cancel_error(Some(resolved), error)), + Ok(cancelled) => entries.extend(cancelled), + Err(error) => entries.extend( + jobs.into_iter() + .map(|(handle, _)| session_job_cancel_error(Some(handle), error.clone())), + ), } } - Ok(entries) + Ok(EnvironmentJobCancelResponse { jobs: entries }) } - async fn cancel_job_on_provider( + /// The daemon's entries for the named jobs, then for the dependents a + /// `dependents` cancel reached. + async fn cancel_jobs_in_environment( &self, environment_id: &EnvironmentId, - job_id: JobId, - scope: SessionJobCancelScopeView, + jobs: &[(SessionJobHandleView, JobId)], + scope: ProtocolJobCancelScope, force: bool, - ) -> Result { - let mut client = self - .connect_client_for_job_handle(environment_id, "cancel") - .await?; - let response = client - .cancel_jobs(&ProtocolCancelJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: vec![job_id], - scope: protocol_cancel_scope(scope), - force, - }) + ) -> Result, String> { + let instance = self + .read_job_instance(environment_id) .await - .map_err(|error| error.to_string())?; - response - .jobs - .into_iter() - .next() - .ok_or_else(|| "provider returned no job result".to_owned()) + .map_err(|error| error.message)?; + let client = self + .connect_job_client(&instance) + .await + .map_err(|error| error.message)?; + let per_request = + (client.initialize_response().limits.job.max_request_jobs as usize).max(1); + let mut entries = Vec::new(); + let mut result = Ok(()); + for chunk in jobs.chunks(per_request) { + let params = ProtocolCancelJobsParams { + job_ids: chunk.iter().map(|(_, job_id)| job_id.clone()).collect(), + scope, + force, + }; + match client.cancel_jobs(¶ms).await { + Ok(response) => entries.extend( + response + .jobs + .into_iter() + .map(|entry| session_job_cancel_entry(environment_id, entry)), + ), + Err(error) => { + result = Err(error.to_string()); + break; + } + } + } + let _ = client.close().await; + result.map(|()| entries) } - /// Resolve the environment through the shared status-aware resolver so - /// job calls participate in wake-on-use: a powered-down - /// provisioned environment gets its desired power set to `running` and - /// the call fails typed `environment_not_ready` for the caller to retry - /// — the shape polling automations lean on. Plain record reads would - /// dial a sleeping data plane and fail generically, waking nothing. + /// Resolve the environment through the shared use gate, so a + /// provisioning environment or a reconnecting daemon fails typed + /// `environment_not_ready` for the caller to retry — the shape polling + /// automations lean on — and a closed or failed one is refused. async fn read_job_instance( &self, environment_id: &EnvironmentId, @@ -401,236 +372,276 @@ impl GatewayAgentApi { .map_err(super::environments::map_environment_resolve_error) } - async fn connect_client_for_job_handle( + /// An initialized data client for the environment; its daemon must + /// serve the `job` family. + async fn connect_job_client( &self, - environment_id: &EnvironmentId, - operation: &str, - ) -> Result, String> { - let instance = self - .read_job_instance(environment_id) - .await - .map_err(|error| error.to_string())?; + instance: &EnvironmentRecord, + ) -> Result { let key = self .environment_gateway - .route_key(self.store.config().universe_id, &instance); - let connection = self.environment_gateway.connection(&key); - let (client, capabilities) = connect_initialized_environment_data_client( - &connection, + .route_key(self.store.config().universe_id, instance); + let (client, initialized) = DataClient::connect_initialized( + &self.environment_gateway.endpoint(&key), self.environment_gateway - .connect_options("lightspeed-temporal-runtime"), + .connect_options(JOB_API_USER_AGENT) + .with_request_timeout(JOB_API_REQUEST_TIMEOUT), ) .await - .map_err(|error| error.to_string())?; - let supported = match operation { - "read" => capabilities.job_read, - "cancel" => capabilities.job_cancel, - _ => true, - }; - if !supported { - return Err(format!( - "environment does not support durable job {operation}: {}", - environment_id - )); + .map_err(|error| { + // A powered-down environment wakes on this dial; its typed + // refusal is `environment_not_ready`, like the gate's waits. + match crate::environments::resolver::route_refusal_error(instance, &error) { + Some(refusal) => super::environments::map_environment_resolve_error(refusal), + None => map_environment_client_api_error(error), + } + })?; + if initialized.capabilities.job.is_none() { + let _ = client.close().await; + return Err(AgentApiError::rejected(format!( + "environment does not serve durable jobs: {}", + instance.environment_id + ))); } Ok(client) } } +/// Where each job of an API read starts and how much it shows. +#[derive(Clone, Copy, Debug)] +struct JobReadPosition { + after: Option, + tail_bytes: Option, + /// Per job; never more than the daemon's read ceiling allows. + max_bytes: u64, +} + +/// One `job/read` for `jobs` at `position`, waiting for nothing. The +/// per-job budget shrinks so the total stays within `maxReadBytes`. +fn job_read_params<'a>( + jobs: impl ExactSizeIterator, + position: JobReadPosition, + limits: &JobLimits, +) -> ProtocolReadJobsParams { + let count = jobs.len().max(1) as u64; + let per_job = position.max_bytes.min(limits.max_read_bytes / count); + ProtocolReadJobsParams { + jobs: jobs + .map(|job_id| ProtocolJobReadRequest { + job_id: job_id.clone(), + after: position.after, + tail_bytes: position.tail_bytes, + max_bytes: Some(per_job), + }) + .collect(), + max_bytes: per_job * count, + wait_ms: 0, + return_on: ProtocolReturnOn::AnyOutput, + } +} + fn parse_environment_job_environment_id(value: String) -> Result { EnvironmentId::try_new(value).map_err(|error| { AgentApiError::invalid_request(format!("invalid environment instance id: {error}")) }) } -fn validate_job_request_id(value: &str) -> Result<(), AgentApiError> { - validate_general_string_id("job request_id", value) - .map_err(|error| AgentApiError::invalid_request(format!("invalid request_id: {error}"))) -} - fn parse_protocol_job_id(value: &str) -> Result { - validate_general_string_id("job_id", value) - .map_err(|error| format!("invalid job_id: {error}"))?; - Ok(JobId::new(value.to_owned())) + JobId::new(value).map_err(|error| format!("invalid job_id: {error}")) } -fn parse_job_handle(handle: SessionJobHandleInput) -> Result { +fn parse_job_handle( + handle: SessionJobHandleInput, +) -> Result<(EnvironmentId, SessionJobHandleView, JobId), String> { let environment_id = EnvironmentId::try_new(handle.environment_id) .map_err(|error| format!("invalid job handle environment_id: {error}"))?; let job_id = parse_protocol_job_id(&handle.job_id)?; - Ok(SessionJobHandleView { + let view = SessionJobHandleView { environment_id: environment_id.as_str().to_owned(), job_id: job_id.as_str().to_owned(), - }) + }; + Ok((environment_id, view, job_id)) } -fn api_start_spec_to_protocol( +/// The job workflow's record of an API job spec, validated against the +/// wire grammar so the start activity cannot refuse it later. +fn api_start_spec_to_workflow( spec: SessionJobStartSpecInput, job_id: JobId, -) -> Result { +) -> Result { if spec.argv.is_empty() { return Err("job argv must not be empty".to_owned()); } - Ok(ProtocolJobStartSpec { - job_id, - name: spec.name, - argv: spec.argv, - cwd: spec - .cwd - .as_deref() - .map(EnvironmentPath::new) - .transpose() - .map_err(|error| format!("invalid job cwd: {error}"))?, - env: spec.env, - secret_env: BTreeMap::new(), - stdin: spec.stdin.map(|stdin| ByteChunk::from(stdin.into_bytes())), - timeout_ms: spec.timeout_ms, + let name = spec + .name + .map(|name| { + JobName::new(name) + .map(JobName::into_string) + .map_err(|error| format!("invalid job name: {error}")) + }) + .transpose()?; + let cwd = spec + .cwd + .map(|cwd| { + EnvironmentPath::new(cwd) + .map(|cwd| cwd.as_str().to_owned()) + .map_err(|error| format!("invalid job cwd: {error}")) + }) + .transpose()?; + let queue_key = spec + .queue_key + .map(|key| { + QueueKey::new(key) + .map(QueueKey::into_string) + .map_err(|error| format!("invalid job queue_key: {error}")) + }) + .transpose()?; + Ok(EnvironmentJobSpec { + job_id: EnvironmentJobId::new(job_id.into_string()), + name, + command: EnvironmentJobCommand { + argv: spec.argv, + cwd, + env: spec.env, + stdin: spec.stdin, + timeout_ms: spec.timeout_ms, + }, depends_on: spec .depends_on .into_iter() - .map(api_dependency_to_protocol) + .map(api_dependency_to_workflow) .collect::, _>>()?, dependency_policy: match spec.dependency_policy { SessionJobDependencyPolicyView::AllSucceeded => { - ProtocolJobDependencyPolicy::AllSucceeded + EnvironmentJobDependencyPolicy::AllSucceeded + } + SessionJobDependencyPolicyView::AllTerminal => { + EnvironmentJobDependencyPolicy::AllTerminal } - SessionJobDependencyPolicyView::AllTerminal => ProtocolJobDependencyPolicy::AllTerminal, }, - queue_key: spec.queue_key, + queue_key, }) } -fn api_dependency_to_protocol( +fn api_dependency_to_workflow( dependency: SessionJobDependencyInput, -) -> Result { +) -> Result { match (dependency.job_id, dependency.name) { - (Some(job_id), None) => Ok(ProtocolJobDependency { - job_id: Some(parse_protocol_job_id(&job_id)?), - name: None, + (Some(job_id), None) => Ok(EnvironmentJobDependency::JobId { + job_id: EnvironmentJobId::new(parse_protocol_job_id(&job_id)?.into_string()), }), - (None, Some(name)) if !name.is_empty() => Ok(ProtocolJobDependency { - job_id: None, - name: Some(name), + (None, Some(name)) => Ok(EnvironmentJobDependency::Name { + name: JobName::new(name) + .map_err(|error| format!("invalid job dependency name: {error}"))? + .into_string(), }), (Some(_), Some(_)) => { Err("job dependency must specify job_id or name, not both".to_owned()) } - _ => Err("job dependency must specify job_id or name".to_owned()), + (None, None) => Err("job dependency must specify job_id or name".to_owned()), } } -fn protocol_cancel_scope(scope: SessionJobCancelScopeView) -> ProtocolJobCancelScope { - match scope { - SessionJobCancelScopeView::Job => ProtocolJobCancelScope::Job, - SessionJobCancelScopeView::Dependents => ProtocolJobCancelScope::Dependents, - } -} - -fn derived_job_id(environment_id: &EnvironmentId, request_id: &str, index: usize) -> JobId { +fn derived_job_id( + environment_id: &EnvironmentId, + request_id: &JobRequestId, + index: usize, +) -> JobId { let hash = BlobRef::from_bytes(format!("{environment_id}:{request_id}:{index}").as_bytes()); - JobId::new(format!("job-{}", &hash.as_str()[7..31])) + JobId::new(format!("job-{}", &hash.as_str()[7..31])).expect("hex job id fits the id grammar") } +/// A group is named by its environment and the content address of its +/// encoded start payload, which is also the payload's CAS `request_ref`. fn derived_job_group_id( environment_id: &EnvironmentId, - request_id: &str, - request_fingerprint: &str, + payload_ref: &BlobRef, ) -> EnvironmentJobGroupId { - let hash = BlobRef::from_bytes( - format!("{environment_id}:{request_id}:{request_fingerprint}").as_bytes(), - ); + let hash = BlobRef::from_bytes(format!("{environment_id}:{payload_ref}").as_bytes()); EnvironmentJobGroupId::new(format!("ejg_{}", &hash.as_str()[7..31])) } -async fn connect_initialized_environment_data_client( - connection: &EnvironmentDataConnection, - options: WebSocketConnectOptions, -) -> Result< - ( - EnvironmentDataClient, - EnvironmentCapabilities, - ), - AgentApiError, -> { - let mut client = connect_environment_data_client(connection, options).await?; - let response = client - .initialize(&DataInitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-temporal-runtime".to_owned(), - scope: connection.scope.clone(), - resume_connection_id: None, - }) - .await - .map_err(map_environment_client_api_error)?; - if response.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(AgentApiError::internal(format!( - "unsupported environment data protocol version {}; expected {CURRENT_PROTOCOL_VERSION}", - response.protocol_version - ))); +fn map_environment_client_api_error(error: ClientError) -> AgentApiError { + let message = error.to_string(); + match error.protocol_kind() { + Some(ProtocolErrorKind::InvalidParams | ProtocolErrorKind::InvalidRequest) => { + AgentApiError::invalid_request(message) + } + Some(ProtocolErrorKind::NotFound) => AgentApiError::not_found(message), + Some(ProtocolErrorKind::AlreadyExists) => AgentApiError::conflict(message), + Some( + ProtocolErrorKind::FailedPrecondition + | ProtocolErrorKind::CapabilityUnavailable + | ProtocolErrorKind::PermissionDenied + | ProtocolErrorKind::VersionMismatch, + ) => AgentApiError::rejected(message), + _ => match error { + ClientError::WrongPlane { .. } | ClientError::Configuration { .. } => { + AgentApiError::rejected(message) + } + _ => AgentApiError::internal(message), + }, } - let capabilities = response.capabilities; - client - .initialized(&DataInitializedParams {}) - .await - .map_err(map_environment_client_api_error)?; - Ok((client, capabilities)) } -pub(crate) async fn connect_environment_data_client( - connection: &EnvironmentDataConnection, - options: WebSocketConnectOptions, -) -> Result, AgentApiError> { - match &connection.transport { - EnvironmentTransport::WebSocket => { - EnvironmentDataClient::connect(&connection.endpoint, options) - .await - .map_err(map_environment_client_api_error) - } - other => Err(AgentApiError::rejected(format!( - "unsupported environment data transport for environment jobs: {other:?}" - ))), +/// The started view of a job the workflow recorded. A job the daemon no +/// longer knows (`lost`) is reported as failed: the API has no such status. +fn api_started_job( + environment_id: &EnvironmentId, + record: EnvironmentJobRecord, +) -> SessionJobStartedView { + SessionJobStartedView { + name: record.name, + job_id: record.job_id.as_str().to_owned(), + handle: SessionJobHandleView { + environment_id: environment_id.as_str().to_owned(), + job_id: record.job_id.as_str().to_owned(), + }, + promise_id: None, + status: api_record_status(record.status), + dependencies: record + .depends_on + .into_iter() + .map(EnvironmentJobId::into_string) + .collect(), + queue_key: record.queue_key, } } -fn map_environment_client_api_error(error: EnvironmentClientError) -> AgentApiError { - match error { - EnvironmentClientError::Protocol(error) => match error.code { - EnvironmentProtocolErrorCode::InvalidRequest => { - AgentApiError::invalid_request(error.message) - } - EnvironmentProtocolErrorCode::NotFound => AgentApiError::not_found(error.message), - EnvironmentProtocolErrorCode::Conflict - | EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable => { - AgentApiError::rejected(error.message) - } - _ => AgentApiError::internal(error.message), - }, - other => AgentApiError::internal(other.to_string()), +fn api_record_status(status: EnvironmentJobStatus) -> SessionJobStatusView { + match status { + EnvironmentJobStatus::Queued => SessionJobStatusView::Queued, + EnvironmentJobStatus::Running => SessionJobStatusView::Running, + EnvironmentJobStatus::Succeeded => SessionJobStatusView::Succeeded, + EnvironmentJobStatus::Failed | EnvironmentJobStatus::Lost => SessionJobStatusView::Failed, + EnvironmentJobStatus::TimedOut => SessionJobStatusView::TimedOut, + EnvironmentJobStatus::Cancelled => SessionJobStatusView::Cancelled, + EnvironmentJobStatus::DependencyFailed => SessionJobStatusView::DependencyFailed, + EnvironmentJobStatus::Interrupted => SessionJobStatusView::Interrupted, } } -fn session_job_read_entry_from_response( +fn session_job_read_entry( handle: SessionJobHandleView, - response: Option, + entry: ProtocolJobReadEntry, ) -> SessionJobReadEntryView { - match response { - Some(response) => SessionJobReadEntryView { + match entry { + ProtocolJobReadEntry::Job { summary, output } => SessionJobReadEntryView { handle: Some(handle), - summary: Some(api_job_summary(response.summary)), - output_chunks: response - .output_chunks + summary: Some(api_job_summary(*summary)), + output_chunks: output + .chunks .into_iter() .map(api_job_output_chunk) .collect(), - output_next_seq: response.output_next_seq, - artifacts: response - .artifacts - .into_iter() - .map(api_job_artifact) - .collect(), + output_next: output.next, + output_end: output.end, + output_gaps: output.gaps.into_iter().map(api_job_output_gap).collect(), error: None, }, - None => session_job_read_error(Some(handle), "provider returned no job result".to_owned()), + ProtocolJobReadEntry::Error { error, .. } => { + session_job_read_error(Some(handle), error.to_string()) + } } } @@ -642,12 +653,33 @@ fn session_job_read_error( handle, summary: None, output_chunks: Vec::new(), - output_next_seq: 0, - artifacts: Vec::new(), + output_next: 0, + output_end: 0, + output_gaps: Vec::new(), error: Some(error), } } +fn session_job_cancel_entry( + environment_id: &EnvironmentId, + entry: ProtocolJobSummaryEntry, +) -> SessionJobCancelEntryView { + let handle = |job_id: &JobId| SessionJobHandleView { + environment_id: environment_id.as_str().to_owned(), + job_id: job_id.as_str().to_owned(), + }; + match entry { + ProtocolJobSummaryEntry::Job { summary } => SessionJobCancelEntryView { + handle: Some(handle(&summary.job_id)), + summary: Some(api_job_summary(*summary)), + error: None, + }, + ProtocolJobSummaryEntry::Error { job_id, error } => { + session_job_cancel_error(Some(handle(&job_id)), error.to_string()) + } + } +} + fn session_job_cancel_error( handle: Option, error: String, @@ -661,57 +693,254 @@ fn session_job_cancel_error( fn api_job_summary(summary: ProtocolJobSummary) -> SessionJobSummaryView { SessionJobSummaryView { - namespace: summary.namespace, - job_id: summary.job_id.as_str().to_owned(), - name: summary.name, + job_id: summary.job_id.into_string(), + request_id: summary.request_id.into_string(), + name: summary.name.map(JobName::into_string), status: api_job_status(summary.status), dependencies: summary - .dependencies + .depends_on .into_iter() - .map(|id| id.as_str().to_owned()) + .map(JobId::into_string) .collect(), created_at_ms: summary.created_at_ms, queued_at_ms: summary.queued_at_ms, started_at_ms: summary.started_at_ms, finished_at_ms: summary.finished_at_ms, - exit_code: summary.exit_code, - orphaned_descendants: summary.orphaned_descendants, - failure: summary.failure, - queue_key: summary.queue_key, + cancel_requested_at_ms: summary.cancel_requested_at_ms, + duration_ms: summary.duration_ms, + exit: summary.exit.map(|exit| match exit { + ProtocolExitStatus::Code { code } => SessionJobExitView::Code { code }, + ProtocolExitStatus::Signal { + signal, + core_dumped, + } => SessionJobExitView::Signal { + signal: signal.as_str().to_owned(), + core_dumped, + }, + }), + stopped_by: summary.stopped_by.map(|stopped_by| match stopped_by { + ProtocolStoppedBy::Timeout => SessionJobStoppedByView::Timeout, + ProtocolStoppedBy::Terminate => SessionJobStoppedByView::Terminate, + ProtocolStoppedBy::Kill => SessionJobStoppedByView::Kill, + }), + orphaned_descendants: summary.leftovers.is_some(), + failure: summary.error.map(|error| error.to_string()), + queue_key: summary.queue_key.map(QueueKey::into_string), } } fn api_job_status(status: ProtocolJobStatus) -> SessionJobStatusView { match status { - ProtocolJobStatus::Accepted => SessionJobStatusView::Accepted, ProtocolJobStatus::Queued => SessionJobStatusView::Queued, ProtocolJobStatus::Running => SessionJobStatusView::Running, ProtocolJobStatus::Succeeded => SessionJobStatusView::Succeeded, ProtocolJobStatus::Failed => SessionJobStatusView::Failed, - ProtocolJobStatus::CancelRequested => SessionJobStatusView::CancelRequested, - ProtocolJobStatus::Cancelled => SessionJobStatusView::Cancelled, ProtocolJobStatus::TimedOut => SessionJobStatusView::TimedOut, + ProtocolJobStatus::Cancelled => SessionJobStatusView::Cancelled, ProtocolJobStatus::DependencyFailed => SessionJobStatusView::DependencyFailed, ProtocolJobStatus::Interrupted => SessionJobStatusView::Interrupted, - ProtocolJobStatus::Lost => SessionJobStatusView::Lost, } } -fn api_job_output_chunk(chunk: ProtocolJobOutputChunk) -> SessionJobOutputChunkView { +fn api_job_output_chunk(chunk: ProtocolOutputChunk) -> SessionJobOutputChunkView { SessionJobOutputChunkView { - seq: chunk.seq, + offset: chunk.offset, stream: match chunk.stream { - ProtocolJobOutputStream::Stdout => SessionJobOutputStreamView::Stdout, - ProtocolJobOutputStream::Stderr => SessionJobOutputStreamView::Stderr, + // Jobs never run on a PTY; a merged stream reads as stdout. + ProtocolOutputStream::Stdout | ProtocolOutputStream::Pty => { + SessionJobOutputStreamView::Stdout + } + ProtocolOutputStream::Stderr => SessionJobOutputStreamView::Stderr, }, - data_base64: BASE64_STANDARD.encode(chunk.chunk.into_inner()), + data_base64: BASE64_STANDARD.encode(chunk.data.as_slice()), } } -fn api_job_artifact(artifact: ProtocolJobArtifact) -> SessionJobArtifactView { - SessionJobArtifactView { - path: artifact.path.as_str().to_owned(), - kind: artifact.kind, - metadata: artifact.metadata, +fn api_job_output_gap(gap: ProtocolOutputGap) -> SessionJobOutputGapView { + SessionJobOutputGapView { + offset: gap.offset, + omitted_bytes: gap.omitted_bytes, + reason: match gap.reason { + ProtocolGapReason::Overflow => SessionJobOutputGapReasonView::Overflow, + ProtocolGapReason::Expired => SessionJobOutputGapReasonView::Expired, + }, + } +} + +#[cfg(test)] +mod tests { + use environment_protocol::{bytes::ByteChunk, data::exec::OutputPage, error::ProtocolError}; + + use super::*; + + fn limits(max_read_bytes: u64) -> JobLimits { + JobLimits { + max_read_bytes, + max_request_jobs: 64, + max_retained_output_bytes: 2 * 1024 * 1024, + record_retention_ms: 1, + cancel_grace_ms: 1, + } + } + + fn job_id(value: &str) -> JobId { + JobId::new(value).unwrap() + } + + #[test] + fn reads_split_the_daemon_ceiling_across_jobs() { + let jobs = [job_id("a"), job_id("b")]; + let params = job_read_params( + jobs.iter(), + JobReadPosition { + after: None, + tail_bytes: Some(100), + max_bytes: 1_000, + }, + &limits(600), + ); + assert_eq!(params.max_bytes, 600); + assert_eq!(params.wait_ms, 0); + for (entry, expected) in params.jobs.iter().zip(&jobs) { + assert_eq!(&entry.job_id, expected); + assert_eq!(entry.tail_bytes, Some(100)); + assert_eq!(entry.after, None); + assert_eq!(entry.max_bytes, Some(300)); + } + } + + #[test] + fn api_specs_are_validated_against_the_wire_grammar() { + let spec = |name: Option<&str>, cwd: Option<&str>| SessionJobStartSpecInput { + name: name.map(str::to_owned), + job_id: None, + argv: vec!["make".to_owned()], + cwd: cwd.map(str::to_owned), + env: BTreeMap::new(), + stdin: Some("input".to_owned()), + timeout_ms: None, + depends_on: vec![SessionJobDependencyInput { + job_id: None, + name: Some("lint".to_owned()), + }], + dependency_policy: SessionJobDependencyPolicyView::AllTerminal, + queue_key: Some("ci".to_owned()), + }; + let stored = + api_start_spec_to_workflow(spec(Some("build"), Some("/workspace")), job_id("build")) + .unwrap(); + assert_eq!(stored.command.cwd.as_deref(), Some("/workspace")); + assert_eq!(stored.command.stdin.as_deref(), Some("input")); + assert_eq!( + stored.depends_on, + vec![EnvironmentJobDependency::Name { + name: "lint".to_owned() + }] + ); + assert!(api_start_spec_to_workflow(spec(None, Some("relative")), job_id("x")).is_err()); + assert!(api_start_spec_to_workflow(spec(Some("a\tb"), None), job_id("x")).is_err()); + assert!(parse_protocol_job_id("not a job id").is_err()); + } + + #[test] + fn read_entries_carry_byte_positions_and_entry_errors() { + let summary: ProtocolJobSummary = serde_json::from_value(serde_json::json!({ + "jobId": "build", + "requestId": "req-1", + "status": "failed", + "createdAtMs": 1, + "exit": {"kind": "signal", "signal": "SIGSEGV", "coreDumped": true}, + "stoppedBy": "kill", + })) + .unwrap(); + let handle = SessionJobHandleView { + environment_id: "environment_1".to_owned(), + job_id: "build".to_owned(), + }; + let entry = session_job_read_entry( + handle.clone(), + ProtocolJobReadEntry::Job { + summary: Box::new(summary), + output: OutputPage { + chunks: vec![ProtocolOutputChunk { + offset: 10, + stream: ProtocolOutputStream::Stderr, + data: ByteChunk::from(b"boom".to_vec()), + }], + next: 14, + end: 40, + gaps: vec![ProtocolOutputGap { + offset: 4, + omitted_bytes: 6, + reason: ProtocolGapReason::Overflow, + }], + }, + }, + ); + assert_eq!((entry.output_next, entry.output_end), (14, 40)); + assert_eq!(entry.output_chunks[0].offset, 10); + assert_eq!( + entry.output_chunks[0].stream, + SessionJobOutputStreamView::Stderr + ); + assert_eq!(entry.output_gaps[0].omitted_bytes, 6); + let summary = entry.summary.unwrap(); + assert_eq!(summary.request_id, "req-1"); + assert_eq!( + summary.exit, + Some(SessionJobExitView::Signal { + signal: "SIGSEGV".to_owned(), + core_dumped: true + }) + ); + assert_eq!(summary.stopped_by, Some(SessionJobStoppedByView::Kill)); + + let missing = session_job_read_entry( + handle, + ProtocolJobReadEntry::Error { + job_id: job_id("build"), + error: ProtocolError::not_found("unknown job"), + }, + ); + assert!(missing.summary.is_none()); + assert!(missing.error.is_some()); + } + + #[test] + fn lost_jobs_are_reported_as_failed() { + assert_eq!( + api_record_status(EnvironmentJobStatus::Lost), + SessionJobStatusView::Failed + ); + assert_eq!( + api_record_status(EnvironmentJobStatus::Queued), + SessionJobStatusView::Queued + ); + } + + #[test] + fn client_refusals_keep_their_api_kind() { + let kind = |error: ProtocolError| { + map_environment_client_api_error(ClientError::Protocol(error)).kind + }; + assert_eq!( + kind(ProtocolError::invalid_params("bad")), + AgentApiErrorKind::InvalidRequest + ); + assert_eq!( + kind(ProtocolError::not_found("gone")), + AgentApiErrorKind::NotFound + ); + assert_eq!( + kind(ProtocolError::failed_precondition( + environment_protocol::error::PreconditionReason::JobIdInUse, + "in use" + )), + AgentApiErrorKind::Rejected + ); + assert_eq!( + map_environment_client_api_error(ClientError::DeadlineExceeded { elapsed_ms: 1 }).kind, + AgentApiErrorKind::Internal + ); } } diff --git a/crates/temporal-runtime/src/gateway/service/tests.rs b/crates/temporal-runtime/src/gateway/service/tests.rs index b872f5af7..d7708431a 100644 --- a/crates/temporal-runtime/src/gateway/service/tests.rs +++ b/crates/temporal-runtime/src/gateway/service/tests.rs @@ -2531,6 +2531,7 @@ async fn skill_list_reads_latest_structured_provenance() { environment.availability = EnvironmentSkillAvailability::Stale; environment.warnings = vec!["scan incomplete".into()]; environment.skills.push(EnvironmentSkill { + root: "/skills/system".into(), skill_id: tools::skills::SkillId::new("environment:review"), name: "current".into(), description: "same named skill".into(), diff --git a/crates/temporal-runtime/src/main.rs b/crates/temporal-runtime/src/main.rs index 3250492d4..b1ed64699 100644 --- a/crates/temporal-runtime/src/main.rs +++ b/crates/temporal-runtime/src/main.rs @@ -3,13 +3,13 @@ use std::{env, net::SocketAddr, sync::Arc, time::Duration}; use clap::{Args, Parser, Subcommand}; use temporal_runtime::{ config::{ - DeploymentStores, TaskQueues, cas_sweep_grace_from_env, environment_public_url_from_env, - gateway_auth_mode_from_env, postgres_pool_from_env, task_queues_from_env, + DeploymentStores, TaskQueues, cas_sweep_grace_from_env, gateway_auth_mode_from_env, + postgres_pool_from_env, task_queues_from_env, }, gateway::{ DEFAULT_GATEWAY_BIND, DEFAULT_MAX_REQUEST_BODY_BYTES, DEFAULT_TEMPORAL_NAMESPACE, DEFAULT_TEMPORAL_TARGET, GatewayRoutes, GatewayState, gateway_router, - prewarm_single_universe, + prewarm_single_universe, registration, }, roles::{Role, RoleSet}, universe::UniverseRuntime, @@ -177,9 +177,20 @@ struct RunArgs { max_request_body_bytes: usize, /// Externally reachable base URL of the gateway (OAuth callbacks, - /// webhook ingest URLs). Defaults to http://{bind}. + /// webhook ingest URLs, and the origin registered daemons sign). + /// Defaults to http://{bind}. #[arg(long, env = "LIGHTSPEED_PUBLIC_BASE_URL")] public_base_url: Option, + + /// Further gateway URLs registered daemons may dial and sign, comma + /// separated: for example a cluster-internal Service address. Only + /// their origins count. + #[arg( + long, + env = "LIGHTSPEED_ENVIRONMENT_REGISTRATION_URLS", + value_delimiter = ',' + )] + environment_registration_urls: Vec, } impl RunArgs { @@ -554,6 +565,17 @@ async fn run_roles(args: RunArgs) -> anyhow::Result<()> { .await? .with_blob_cache(temporal_runtime::config::blob_cache_from_env()?); let reaper_stores = stores.clone(); + // Daemons sign the gateway origin they dial; refuse to serve + // registration under an origin the gateway cannot name. + let registration_origins = if roles.has(Role::EnvironmentGateway) { + temporal_runtime::gateway::http::registration_origins( + args.public_base_url.as_deref(), + &args.environment_registration_urls, + args.bind, + )? + } else { + Vec::new() + }; let public_base_url = args .public_base_url .clone() @@ -586,8 +608,18 @@ async fn run_roles(args: RunArgs) -> anyhow::Result<()> { let mut workers: Vec<(Role, temporalio_sdk::Worker)> = Vec::new(); if roles.has(Role::EnvironmentGateway) { + // Before the listener accepts connections: this process held every + // registered connection the rows still call connected. + let repaired = registration::repair_registered_connections(&universes).await?; + tracing::info!( + target: "temporal_runtime", + repaired, + "registered environments marked disconnected until their daemons reconnect" + ); background.push(tokio::spawn(universes.clone().run_environment_reconciler())); - background.push(tokio::spawn(universes.clone().run_power_reaper())); + background.push(tokio::spawn(registration::run_registered_sweep( + universes.clone(), + ))); } if roles.has(Role::Sessions) { let activities = WorkerActivities::with_runtime(universes.clone()); @@ -678,7 +710,7 @@ async fn run_roles(args: RunArgs) -> anyhow::Result<()> { }; let gateway_state = Arc::new( GatewayState::multi(mode, universes, public_base_url) - .with_environment_public_url(environment_public_url_from_env()?), + .with_registration_origins(registration_origins), ); let app = gateway_router(gateway_state, args.max_request_body_bytes, routes); let listener = tokio::net::TcpListener::bind(args.bind).await?; @@ -796,6 +828,20 @@ mod tests { assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument); } + #[test] + fn registration_urls_are_a_comma_separated_list() { + let cli = Cli::try_parse_from([ + "lightspeed-runtime", + "--environment-registration-urls", + "ws://gateway.internal:18080,wss://gateway.example", + ]) + .expect("parse registration URLs"); + assert_eq!( + cli.run.environment_registration_urls, + ["ws://gateway.internal:18080", "wss://gateway.example"] + ); + } + #[test] fn code_role_accepts_its_own_queue_and_interpreter_capacity() { let cli = Cli::try_parse_from([ diff --git a/crates/temporal-runtime/src/roles.rs b/crates/temporal-runtime/src/roles.rs index 109c2a379..608b1e252 100644 --- a/crates/temporal-runtime/src/roles.rs +++ b/crates/temporal-runtime/src/roles.rs @@ -11,8 +11,9 @@ pub enum Role { Gateway, /// The environment data plane: the public routes outbound daemons dial, /// the internal route workers use to reach environments, the lifecycle - /// reconciler, and the power reaper. Registered daemons hold their - /// connection in this process, so run exactly one replica of it. + /// reconciler, and the registered-environment sweep. Registered daemons + /// hold their connection in this process, so run exactly one replica of + /// it. EnvironmentGateway, /// Session, sub-agent, and environment-job workflows with their /// activities and the promise reaper. diff --git a/crates/temporal-runtime/src/universe.rs b/crates/temporal-runtime/src/universe.rs index 274ad807c..183406e8f 100644 --- a/crates/temporal-runtime/src/universe.rs +++ b/crates/temporal-runtime/src/universe.rs @@ -23,6 +23,7 @@ use std::{ }; use auth::{GitHubApiClient, HttpGitHubApiClient, HttpOAuthTokenClient, OAuthTokenClient}; +use futures_util::StreamExt as _; use llm_clients::{ anthropic::messages as am, openai::{audio as oai_audio, completions as oai_completions, responses as oai}, @@ -34,7 +35,7 @@ use uuid::Uuid; use crate::{ config::{DeploymentStores, TaskQueues}, - environments::gateway::EnvironmentGatewayClientConfig, + environments::{gateway::EnvironmentGatewayClientConfig, signals::EnvironmentSignals}, gateway::GatewayAgentApi, subagents::AgentApiSubagentRuntime, worker::{ActivityState, AudioTranscoder}, @@ -44,6 +45,9 @@ use crate::{ /// only needs to be long enough that busy tenants never churn. const UNIVERSE_IDLE_EVICT_MS: u64 = 4 * 60 * 60 * 1000; +/// Universes the environment display sweep observes at once. +const DISPLAY_SWEEP_UNIVERSE_CONCURRENCY: usize = 16; + /// Hard cap on cached universe states, evicted LRU-first. Sized far above any /// expected concurrently-active tenant count; the idle sweep does the real /// work. @@ -141,10 +145,6 @@ struct UniverseEntry { /// pinned universe); the worker never creates, because a workflow for a /// universe the gateway did not create is a routing error, not a provisioning /// request. -/// How often the power reaper polls daemon idle reports. Idle thresholds are -/// minutes to hours, so a coarse cadence keeps the data-plane load negligible. -pub const POWER_REAPER_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60); - /// How often bot trigger Schedules are re-converged in the background. pub const BOT_SCHEDULE_RECONCILE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10 * 60); @@ -156,6 +156,7 @@ pub struct UniverseRuntime { stores: DeploymentStores, clients: DeploymentClients, environment_gateway: EnvironmentGatewayClientConfig, + environment_signals: Arc, states: tokio::sync::Mutex>, } @@ -192,6 +193,7 @@ impl UniverseRuntime { stores, clients: DeploymentClients::from_env()?, environment_gateway, + environment_signals: Arc::new(EnvironmentSignals::default()), states: tokio::sync::Mutex::new(BTreeMap::new()), }) } @@ -224,6 +226,11 @@ impl UniverseRuntime { &self.environment_gateway } + /// This process's environment signals, shared by every universe's API. + pub fn environment_signals(&self) -> &Arc { + &self.environment_signals + } + /// Drop the universe's cached runtime state (operator purge). In-flight /// work holding an `Arc` finishes on the old state; nothing durable is /// lost because states hold no durable data. @@ -262,16 +269,37 @@ impl UniverseRuntime { Ok(state) } - /// Run the one active deployment lifecycle reconciler. Provider calls are - /// idempotent, so process restart safely resumes any persisted intent. + /// Run the one active environment reconciler (environment-gateway + /// role): the work loop and the display sweep, concurrently. Provider + /// calls are idempotent per owner, so a process restart resumes every + /// scheduled row. pub async fn run_environment_reconciler(self: Arc) { - let mut interval = tokio::time::interval(std::time::Duration::from_millis(500)); + tokio::join!( + self.clone().run_environment_work_loop(), + self.run_environment_display_loop(), + ); + } + + /// Every [`WORK_TICK`](crate::environments::lifecycle::WORK_TICK), or at + /// once when nudged: start a work pass for every universe and provider + /// with due provisioning or closing rows and no pass in flight. Passes + /// run as their own tasks, so a slow provider never delays another + /// provider's or universe's rows. + async fn run_environment_work_loop(self: Arc) { + use crate::environments::lifecycle::{ProviderWork, WORK_TICK, clock_ms}; + let mut interval = tokio::time::interval(WORK_TICK); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let work = ProviderWork::default(); let mut failures = crate::gateway::ReconcileFailureLog::default(); loop { - interval.tick().await; - let universe_ids = match store_pg::list_universes_with_pending_environments( + tokio::select! { + _ = interval.tick() => {} + _ = self.environment_signals.work_nudged() => {} + } + let now = clock_ms(); + let universe_ids = match store_pg::list_universes_with_environment_work( self.stores.pool(), + now, ) .await { @@ -292,7 +320,7 @@ impl UniverseRuntime { match state .api .environment_service() - .reconcile_environment_lifecycle_once() + .start_environment_work(now, &work) .await { Ok(_) => failures.succeeded(universe_id), @@ -302,41 +330,50 @@ impl UniverseRuntime { } } - /// Run the one active power reaper. Every pass reads daemon idle - /// reports for ready environments with an idle policy and records power - /// intent; the lifecycle reconciler converges the provider. Failures are - /// logged and retried on the next tick. - pub async fn run_power_reaper(self: Arc) { - let mut interval = tokio::time::interval(POWER_REAPER_INTERVAL); + /// Every [`DISPLAY_SWEEP_INTERVAL`](crate::environments::lifecycle::DISPLAY_SWEEP_INTERVAL): + /// observe the open provisioned environments of every universe. + async fn run_environment_display_loop(self: Arc) { + use crate::environments::lifecycle::{DISPLAY_SWEEP_INTERVAL, clock_ms}; + let mut interval = tokio::time::interval(DISPLAY_SWEEP_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let mut failures = crate::gateway::ReconcileFailureLog::default(); loop { interval.tick().await; - let universe_ids = match store_pg::list_universes_with_idle_policies(self.stores.pool()) + let now = clock_ms(); + match store_pg::list_universes_with_open_provisioned_environments(self.stores.pool()) .await { - Ok(ids) => ids, - Err(error) => { - tracing::warn!(target: "temporal_runtime", %error, "power reaper scan failed"); - continue; - } - }; - for universe_id in universe_ids { - let state = match self.state_for(universe_id, false).await { - Ok(state) => state, - Err(error) => { - tracing::warn!(target: "temporal_runtime", %universe_id, %error, "power reaper could not resolve universe"); - continue; + Ok(universe_ids) => { + // Universes are observed concurrently, so one slow + // provider never holds up the others. + let swept = futures_util::stream::iter(universe_ids) + .map(|universe_id| { + let runtime = self.clone(); + async move { + let result = match runtime.state_for(universe_id, false).await { + Ok(state) => { + state.api.environment_service().run_display_sweep(now).await + } + Err(error) => { + tracing::warn!(target: "temporal_runtime", %universe_id, %error, "environment display sweep could not resolve universe"); + return None; + } + }; + Some((universe_id, result)) + } + }) + .buffer_unordered(DISPLAY_SWEEP_UNIVERSE_CONCURRENCY) + .collect::>() + .await; + for (universe_id, result) in swept.into_iter().flatten() { + match result { + Ok(_) => failures.succeeded(universe_id), + Err(error) => failures.failed(universe_id, &error), + } } - }; - match state - .api - .environment_service() - .reconcile_idle_power_once() - .await - { - Ok(_) => failures.succeeded(universe_id), - Err(error) => failures.failed(universe_id, &error), + } + Err(error) => { + tracing::warn!(target: "temporal_runtime", %error, "environment display sweep scan failed"); } } } @@ -393,7 +430,8 @@ impl UniverseRuntime { self.clients.openai.clone(), self.clients.anthropic.clone(), ) - .with_environment_gateway(self.environment_gateway.clone()); + .with_environment_gateway(self.environment_gateway.clone()) + .with_environment_signals(self.environment_signals.clone()); if let Some(public_base_url) = &self.public_base_url { api = api.with_public_base_url(public_base_url.clone()); } diff --git a/crates/temporal-runtime/src/worker/activities/environment_jobs.rs b/crates/temporal-runtime/src/worker/activities/environment_jobs.rs index 03272f9c8..738d2a4d9 100644 --- a/crates/temporal-runtime/src/worker/activities/environment_jobs.rs +++ b/crates/temporal-runtime/src/worker/activities/environment_jobs.rs @@ -1,34 +1,58 @@ -use std::collections::{BTreeMap, BTreeSet}; +//! Activities of `EnvironmentJobWorkflow`. +//! +//! The workflow keeps its own job records and start payload in history; +//! these activities are the only place that maps them to and from the +//! environment protocol's `job/*` wire types. -use environment_client::{EnvironmentClientError, EnvironmentDataClient}; +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, +}; + +use environment_client::{ClientError, DataClient}; use environment_protocol::{ + bytes::ByteChunk, data::{ - handshake::{InitializeParams, InitializedParams}, - jobs::{CancelJobsParams, JobStatus, ReadJobsParams}, + exec::{CommandSpec, ExecInput, ReturnOn}, + job::{ + CancelJobsParams, JobCancelScope, JobDependency, JobDependencyPolicy, JobReadEntry, + JobReadRequest, JobSpec, JobStatus, JobSummary, JobSummaryEntry, ReadJobsParams, + StartJobsParams, + }, }, - error::EnvironmentProtocolErrorCode, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentDataConnection, EnvironmentTransport}, + error::{ProtocolError, ProtocolErrorKind}, + ids::{JobId, JobName, JobRequestId, QueueKey}, + paths::EnvironmentPath, }; -use environments::{EnvironmentId, EnvironmentJobGroupId}; +use environments::{EnvironmentId, EnvironmentJobGroupId, EnvironmentRecord}; use harness::{BlobRef, PromiseSourceCheckResult, storage::BlobStore}; use temporal_workflow::{ - EnvironmentJobCancelActivityRequest, EnvironmentJobPollActivityRequest, - EnvironmentJobPollActivityResult, EnvironmentJobPrepareWorkflowToolRequest, + EnvironmentJobCancelActivityRequest, EnvironmentJobCancelActivityResult, + EnvironmentJobCancelScope, EnvironmentJobCommand, EnvironmentJobDependency, + EnvironmentJobDependencyPolicy, EnvironmentJobId, EnvironmentJobObservation, + EnvironmentJobPollActivityRequest, EnvironmentJobPollActivityResult, + EnvironmentJobPrepareWorkflowToolRequest, EnvironmentJobRecord, EnvironmentJobSpec, EnvironmentJobStartActivityRequest, EnvironmentJobStartActivityResult, - EnvironmentJobStartPayload, EnvironmentJobSubscription, EnvironmentJobWorkflowArgs, - EnvironmentJobWorkflowToolContext, + EnvironmentJobStartPayloadV1, EnvironmentJobStatus, EnvironmentJobSubscription, + EnvironmentJobWorkflowArgs, EnvironmentJobWorkflowToolContext, }; use temporalio_common::error::ApplicationFailure; use temporalio_sdk::activities::ActivityError; use tools::environment::jobs::{ - JOB_RUN_WORKFLOW_SEMANTIC_TYPE, JOB_RUN_WORKFLOW_TOOL_ID, JOB_SUBMIT_WORKFLOW_SEMANTIC_TYPE, - JOB_SUBMIT_WORKFLOW_TOOL_ID, JobHandle, NormalizeJobResultInput, normalize_job_result, + DEFAULT_JOB_OUTPUT_BYTES, JOB_RUN_WORKFLOW_SEMANTIC_TYPE, JOB_RUN_WORKFLOW_TOOL_ID, + JOB_SUBMIT_WORKFLOW_SEMANTIC_TYPE, JOB_SUBMIT_WORKFLOW_TOOL_ID, JobHandle, + NormalizeJobResultInput, job_output_read, model_job_error, normalize_job_result, store_model_job_result, }; use super::{common::activity_error, state::EnvironmentJobActivityDeps}; -const PROMISE_JOB_OUTPUT_BYTES: usize = 16 * 1024; +/// `User-Agent` of the job activities' worker-route upgrades; `clientInfo` +/// is always the runtime build. +const JOB_ACTIVITY_USER_AGENT: &str = "lightspeed-environment-job-workflow"; + +/// Deadline of every data-plane call these background activities make. +const JOB_REQUEST_TIMEOUT: Duration = Duration::from_secs(60); #[derive(Clone, Copy)] enum WorkflowJobToolKind { @@ -49,11 +73,7 @@ pub(super) async fn prepare_workflow_tool( deps: Option<&EnvironmentJobActivityDeps>, request: EnvironmentJobPrepareWorkflowToolRequest, ) -> Result { - let deps = deps.ok_or_else(|| { - activity_error(anyhow::anyhow!( - "environment job activities are not configured" - )) - })?; + let deps = configured(deps)?; let start = request.start; let expected_holder = temporal_workflow::compose_workflow_id(start.universe_id, &start.invocation.session_id); @@ -112,16 +132,16 @@ pub(super) async fn prepare_workflow_tool( } let environment_id = EnvironmentId::try_new(execution_context.environment_id).map_err(activity_error)?; + let working_directory = execution_context.working_directory.as_deref(); // The executor admitted this call against the session's attachment // grant; the context names the machine it admitted. - let request_id = format!( - "jobreq:{}:{}:{}:{}", + let request_id = workflow_tool_request_id( start.invocation.run_id.as_u64(), start.invocation.turn_id.as_u64(), start.invocation.tool_batch_id.as_u64(), - start.invocation.tool_call_id.as_str() + start.invocation.tool_call_id.as_str(), ); - let mut jobs = match tool_kind { + let wire_jobs = match tool_kind { WorkflowJobToolKind::Submit => { let args: tools::environment::jobs::JobSubmitArgs = serde_json::from_slice(&arguments).map_err(activity_error)?; @@ -134,7 +154,7 @@ pub(super) async fn prepare_workflow_tool( .into_iter() .map(|spec| { let job_id = spec.job_id.clone(); - spec.into_protocol_spec(job_id) + spec.into_job_spec(job_id, working_directory) .map_err(|error| activity_error(anyhow::anyhow!(error.to_string()))) }) .collect::, _>>()? @@ -144,36 +164,17 @@ pub(super) async fn prepare_workflow_tool( serde_json::from_slice(&arguments).map_err(activity_error)?; let job_id = derived_job_run_id(start.invocation.invocation_id.as_str()); vec![ - args.into_protocol_spec(job_id) + args.into_job_spec(job_id, working_directory) .map_err(|error| activity_error(anyhow::anyhow!(error.to_string())))?, ] } }; - if let Some(cwd) = &execution_context.working_directory { - for job in &mut jobs { - let path = tools::environment::sources::absolute( - std::path::Path::new(cwd), - job.cwd.as_ref().map(|path| path.as_str()).unwrap_or("."), - ) - .map_err(|error| activity_error(anyhow::anyhow!(error)))?; - job.cwd = Some( - environment_protocol::shared::EnvironmentPath::new(path.to_string_lossy()) - .map_err(activity_error)?, - ); - } - } - let params = environment_protocol::data::jobs::StartJobsParams { - namespace: environment_id.as_str().to_owned(), - request_id, - jobs, - }; - let request_fingerprint = - BlobRef::from_bytes(&serde_json::to_vec(¶ms).map_err(activity_error)?); - let job_group_id = derived_workflow_tool_job_group_id( - &environment_id, - ¶ms.request_id, - request_fingerprint.as_str(), - ); + let jobs = wire_jobs + .into_iter() + .map(workflow_job_spec) + .collect::, _>>() + .map_err(|error| activity_error(anyhow::anyhow!(error)))?; + let payload = EnvironmentJobStartPayloadV1::new(request_id.into_string(), jobs); let completion_promises = start .invocation .completion_promises @@ -186,12 +187,12 @@ pub(super) async fn prepare_workflow_tool( })?; let subscriptions = match tool_kind { WorkflowJobToolKind::Submit => { - if completion_promises.len() != params.jobs.len() { + if completion_promises.len() != payload.jobs.len() { return Err(activity_error(anyhow::anyhow!( "job_submit completion promise count does not match job count" ))); } - params + payload .jobs .iter() .map(|job| { @@ -232,16 +233,16 @@ pub(super) async fn prepare_workflow_tool( holder_workflow_id: start.holder_workflow_id.clone(), promise_id: promise_id.as_str().to_owned(), completion_key: harness::REPLY_COMPLETION_KEY.to_owned(), - job_id: params.jobs[0].job_id.clone(), + job_id: payload.jobs[0].job_id.clone(), notified: false, }] } }; - let job_ids = params.jobs.iter().map(|job| job.job_id.clone()).collect(); - let payload = EnvironmentJobStartPayload { request: params }; + let encoded = payload.encode().map_err(activity_error)?; + let job_group_id = derived_job_group_id(&environment_id, &BlobRef::from_bytes(&encoded)); let request_ref = deps .blobs - .put_bytes(serde_json::to_vec(&payload).map_err(activity_error)?) + .put_bytes(encoded) .await .map_err(activity_error)?; Ok(EnvironmentJobWorkflowArgs { @@ -252,7 +253,7 @@ pub(super) async fn prepare_workflow_tool( job_group_id: job_group_id.as_str().to_owned(), request_ref, }, - job_ids, + job_ids: payload.job_ids(), subscriptions, started: false, jobs: Vec::new(), @@ -266,202 +267,378 @@ pub(super) async fn prepare_workflow_tool( }) } -fn derived_job_run_id(invocation_id: &str) -> environment_protocol::shared::JobId { +/// The start's idempotency key: one per model tool call, so a retried call +/// replays its jobs and a new call reusing a job id is refused. +fn workflow_tool_request_id(run: u64, turn: u64, batch: u64, call_id: &str) -> JobRequestId { + JobRequestId::new(format!("jobreq:{run}:{turn}:{batch}:{call_id}")).unwrap_or_else(|_| { + // A provider call id outside the id grammar is hashed instead. + let hash = BlobRef::from_bytes(format!("{run}:{turn}:{batch}:{call_id}").as_bytes()); + JobRequestId::new(format!("jobreq-{}", &hash.as_str()[7..39])) + .expect("hex request id fits the id grammar") + }) +} + +fn derived_job_run_id(invocation_id: &str) -> JobId { let hash = BlobRef::from_bytes(invocation_id.as_bytes()); - environment_protocol::shared::JobId::new(format!("job-{}", &hash.as_str()[7..31])) + JobId::new(format!("job-{}", &hash.as_str()[7..31])).expect("hex job id fits the id grammar") } -fn derived_workflow_tool_job_group_id( +/// A group is named by its environment and the content address of its +/// encoded start payload, which is also the payload's CAS `request_ref`. +fn derived_job_group_id( environment_id: &EnvironmentId, - request_id: &str, - request_fingerprint: &str, + payload_ref: &BlobRef, ) -> EnvironmentJobGroupId { - let hash = BlobRef::from_bytes( - format!("{environment_id}:{request_id}:{request_fingerprint}").as_bytes(), - ); + let hash = BlobRef::from_bytes(format!("{environment_id}:{payload_ref}").as_bytes()); EnvironmentJobGroupId::new(format!("ejg_{}", &hash.as_str()[7..31])) } +/// The workflow's record of a wire job spec built by the model-facing +/// arguments. The spec never carries secrets here: the start activity +/// resolves credential-bound variables. +fn workflow_job_spec(spec: JobSpec) -> Result { + let JobSpec { + job_id, + name, + command, + depends_on, + dependency_policy, + queue_key, + } = spec; + let CommandSpec { + argv, + cwd, + env, + secret_env, + input, + timeout_ms, + } = command; + if !secret_env.is_empty() { + return Err(format!( + "job {job_id} carries secret variables before credential resolution" + )); + } + let stdin = match input { + ExecInput::None {} => None, + ExecInput::Bytes { data } => Some( + String::from_utf8(data.into_inner()) + .map_err(|_| format!("job {job_id} stdin is not UTF-8"))?, + ), + ExecInput::Pipe {} => return Err(format!("job {job_id} cannot read a pipe")), + }; + Ok(EnvironmentJobSpec { + job_id: EnvironmentJobId::new(job_id.into_string()), + name: name.map(JobName::into_string), + command: EnvironmentJobCommand { + argv, + cwd: cwd.map(|cwd| cwd.as_str().to_owned()), + env, + stdin, + timeout_ms, + }, + depends_on: depends_on + .into_iter() + .map(|dependency| match dependency { + JobDependency::JobId { job_id } => EnvironmentJobDependency::JobId { + job_id: EnvironmentJobId::new(job_id.into_string()), + }, + JobDependency::Name { name } => EnvironmentJobDependency::Name { + name: name.into_string(), + }, + }) + .collect(), + dependency_policy: match dependency_policy { + JobDependencyPolicy::AllSucceeded => EnvironmentJobDependencyPolicy::AllSucceeded, + JobDependencyPolicy::AllTerminal => EnvironmentJobDependencyPolicy::AllTerminal, + }, + queue_key: queue_key.map(QueueKey::into_string), + }) +} + +/// The wire spec of a stored job, without its secret variables. +fn wire_job_spec(spec: EnvironmentJobSpec) -> Result { + let EnvironmentJobSpec { + job_id, + name, + command, + depends_on, + dependency_policy, + queue_key, + } = spec; + let job_id = wire_job_id(&job_id)?; + Ok(JobSpec { + name: name + .map(|name| JobName::new(name).map_err(|error| format!("job {job_id} name: {error}"))) + .transpose()?, + command: CommandSpec { + argv: command.argv, + cwd: command + .cwd + .map(|cwd| { + EnvironmentPath::new(cwd).map_err(|error| format!("job {job_id} cwd: {error}")) + }) + .transpose()?, + env: command.env, + secret_env: BTreeMap::new(), + input: match command.stdin { + Some(stdin) => ExecInput::Bytes { + data: ByteChunk::from(stdin.into_bytes()), + }, + None => ExecInput::None {}, + }, + timeout_ms: command.timeout_ms, + }, + depends_on: depends_on + .into_iter() + .map(|dependency| match dependency { + EnvironmentJobDependency::JobId { job_id } => { + wire_job_id(&job_id).map(|job_id| JobDependency::JobId { job_id }) + } + EnvironmentJobDependency::Name { name } => JobName::new(name) + .map(|name| JobDependency::Name { name }) + .map_err(|error| format!("job dependency name: {error}")), + }) + .collect::, _>>()?, + dependency_policy: match dependency_policy { + EnvironmentJobDependencyPolicy::AllSucceeded => JobDependencyPolicy::AllSucceeded, + EnvironmentJobDependencyPolicy::AllTerminal => JobDependencyPolicy::AllTerminal, + }, + queue_key: queue_key + .map(|key| { + QueueKey::new(key).map_err(|error| format!("job {job_id} queue key: {error}")) + }) + .transpose()?, + job_id, + }) +} + +fn wire_job_id(job_id: &EnvironmentJobId) -> Result { + JobId::new(job_id.as_str()).map_err(|error| format!("job id {job_id}: {error}")) +} + +/// The workflow's record of a daemon summary. +fn job_record(summary: &JobSummary) -> EnvironmentJobRecord { + EnvironmentJobRecord { + job_id: EnvironmentJobId::new(summary.job_id.as_str()), + name: summary.name.as_ref().map(|name| name.as_str().to_owned()), + status: job_status(summary.status), + depends_on: summary + .depends_on + .iter() + .map(|job_id| EnvironmentJobId::new(job_id.as_str())) + .collect(), + queue_key: summary + .queue_key + .as_ref() + .map(|key| key.as_str().to_owned()), + } +} + +fn job_status(status: JobStatus) -> EnvironmentJobStatus { + match status { + JobStatus::Queued => EnvironmentJobStatus::Queued, + JobStatus::Running => EnvironmentJobStatus::Running, + JobStatus::Succeeded => EnvironmentJobStatus::Succeeded, + JobStatus::Failed => EnvironmentJobStatus::Failed, + JobStatus::TimedOut => EnvironmentJobStatus::TimedOut, + JobStatus::Cancelled => EnvironmentJobStatus::Cancelled, + JobStatus::DependencyFailed => EnvironmentJobStatus::DependencyFailed, + JobStatus::Interrupted => EnvironmentJobStatus::Interrupted, + } +} + +/// A per-entry error: `notFound` is the workflow's `lost` outcome; any +/// other entry error means the daemon broke the contract. +fn entry_error_observation( + job_id: &JobId, + error: ProtocolError, +) -> Result { + if error.kind() == ProtocolErrorKind::NotFound { + Ok(EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new(job_id.as_str()), + }) + } else { + Err(activity_error(anyhow::anyhow!( + "environment answered job {job_id} with {error}" + ))) + } +} + pub(super) async fn start( deps: Option<&EnvironmentJobActivityDeps>, request: EnvironmentJobStartActivityRequest, ) -> Result { - let deps = deps.ok_or_else(|| { - activity_error(anyhow::anyhow!( - "environment job activities are not configured" - )) - })?; + let deps = configured(deps)?; let environment_id = EnvironmentId::try_new(request.environment_id.clone()).map_err(activity_error)?; - let mut payload: EnvironmentJobStartPayload = serde_json::from_slice( + let payload = EnvironmentJobStartPayloadV1::decode( &deps .blobs .read_bytes(&request.request_ref) .await .map_err(activity_error)?, ) - .map_err(activity_error)?; - if payload.request.namespace != environment_id.as_str() { - return Err(activity_error(anyhow::anyhow!( - "environment job start namespace does not match instance {environment_id}" - ))); - } - - for job in &mut payload.request.jobs { + .map_err(non_retryable_activity_error)?; + let request_id = JobRequestId::new(payload.request_id.clone()).map_err(|error| { + non_retryable_activity_error(anyhow::anyhow!("job request id: {error}")) + })?; + let requested_ids = payload.job_ids().into_iter().collect::>(); + let mut jobs = Vec::with_capacity(payload.jobs.len()); + for spec in payload.jobs { + let mut job = wire_job_spec(spec) + .map_err(|error| non_retryable_activity_error(anyhow::anyhow!(error)))?; let secret_env = deps .credentials - .resolve_secret_env(&environment_id, &job.env) + .resolve_secret_env(&environment_id, &job.command.env) .await .map_err(activity_error)?; - job.secret_env.extend(secret_env); + job.command.secret_env.extend(secret_env); + jobs.push(job); } - start_on_provider(deps, &environment_id, &payload).await -} - -async fn start_on_provider( - deps: &EnvironmentJobActivityDeps, - environment_id: &EnvironmentId, - payload: &EnvironmentJobStartPayload, -) -> Result { - let instance = deps - .environments - .read_environment(environment_id) - .await - .map_err(activity_error)?; - if matches!( - instance.status, - environments::EnvironmentStatus::Closing | environments::EnvironmentStatus::Closed - ) { - return Err(non_retryable_activity_error(anyhow::anyhow!( - "cannot start jobs on closing environment instance {environment_id}" - ))); - } - let gateway = deps.gateway.as_ref().ok_or_else(|| { - non_retryable_activity_error(anyhow::anyhow!( - "environment gateway is not configured on this worker" - )) - })?; - let connection = gateway.connection_for(deps.universe_id, &instance); - let (mut client, capabilities) = initialized_client(&connection, gateway).await?; - if !capabilities.job_start { - return Err(non_retryable_activity_error(anyhow::anyhow!( - "environment does not support durable job start: {environment_id}" - ))); - } + let instance = usable_instance(deps, &environment_id).await?; + let client = job_client(deps, &instance).await?; let response = client - .start_jobs(&payload.request) - .await - .map_err(start_environment_activity_error)?; - let requested_ids = payload - .request - .jobs - .iter() - .map(|job| job.job_id.clone()) - .collect::>(); - let returned_ids = response - .jobs + .start_jobs(&StartJobsParams { request_id, jobs }) + .await; + let _ = client.close().await; + let response = response.map_err(client_activity_error)?; + let records = response.jobs.iter().map(job_record).collect::>(); + let returned_ids = records .iter() - .map(|job| job.job_id.clone()) + .map(|record| record.job_id.clone()) .collect::>(); if requested_ids != returned_ids { return Err(non_retryable_activity_error(anyhow::anyhow!( - "environment provider start response job ids do not match the request" + "environment start response job ids do not match the request" ))); } - Ok(EnvironmentJobStartActivityResult { - jobs: response.jobs, - }) -} - -fn start_environment_activity_error(error: EnvironmentClientError) -> ActivityError { - match &error { - EnvironmentClientError::Protocol(error) - if matches!( - error.code, - EnvironmentProtocolErrorCode::InvalidRequest - | EnvironmentProtocolErrorCode::Unauthorized - | EnvironmentProtocolErrorCode::Forbidden - | EnvironmentProtocolErrorCode::NotFound - | EnvironmentProtocolErrorCode::Conflict - | EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable - ) => - { - non_retryable_activity_error(anyhow::anyhow!(error.message.clone())) - } - _ => activity_error(error), - } -} - -fn non_retryable_activity_error(error: impl Into) -> ActivityError { - ActivityError::application(ApplicationFailure::non_retryable(error.into())) + Ok(EnvironmentJobStartActivityResult { jobs: records }) } pub(super) async fn poll( deps: Option<&EnvironmentJobActivityDeps>, request: EnvironmentJobPollActivityRequest, ) -> Result { - let deps = deps.ok_or_else(|| { - activity_error(anyhow::anyhow!( - "environment job activities are not configured" - )) - })?; + let deps = configured(deps)?; let environment_id = EnvironmentId::try_new(request.environment_id).map_err(activity_error)?; - let instance = deps - .environments - .read_environment(&environment_id) - .await - .map_err(activity_error)?; - let gateway = deps.gateway.as_ref().ok_or_else(|| { - non_retryable_activity_error(anyhow::anyhow!( - "environment gateway is not configured on this worker" - )) - })?; - let connection = gateway.connection_for(deps.universe_id, &instance); - let (mut client, _) = initialized_client(&connection, gateway).await?; - let requested_job_ids = request.job_ids.iter().cloned().collect::>(); - let response = client - .read_jobs(&ReadJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: request.job_ids, - after_seq: None, - max_bytes: Some(PROMISE_JOB_OUTPUT_BYTES), - include_artifacts: true, - wait_ms: None, - }) - .await - .map_err(activity_error)?; - let mut jobs = Vec::with_capacity(response.jobs.len()); + let job_ids = request + .job_ids + .iter() + .map(wire_job_id) + .collect::, _>>() + .map_err(|error| non_retryable_activity_error(anyhow::anyhow!(error)))?; + if job_ids.is_empty() { + return Ok(EnvironmentJobPollActivityResult { + jobs: Vec::new(), + resolutions: BTreeMap::new(), + }); + } + let instance = usable_instance(deps, &environment_id).await?; + let client = job_client(deps, &instance).await?; + let result = poll_jobs(deps, &client, &environment_id, &job_ids).await; + let _ = client.close().await; + result +} + +async fn poll_jobs( + deps: &EnvironmentJobActivityDeps, + client: &DataClient, + environment_id: &EnvironmentId, + job_ids: &[JobId], +) -> Result { + let limits = client.initialize_response().limits.job.clone(); + let per_request = (limits.max_request_jobs as usize).max(1); + let mut jobs = Vec::with_capacity(job_ids.len()); + let mut terminal = Vec::new(); + // Summaries first, without output: most polls find running jobs. + for chunk in job_ids.chunks(per_request) { + let response = client + .read_jobs(&ReadJobsParams { + jobs: chunk + .iter() + .map(|job_id| JobReadRequest { + job_id: job_id.clone(), + after: None, + tail_bytes: None, + max_bytes: None, + }) + .collect(), + max_bytes: 0, + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + }) + .await + .map_err(client_activity_error)?; + for (job_id, entry) in chunk.iter().zip(response.jobs) { + match entry { + JobReadEntry::Job { summary, .. } => { + let record = job_record(&summary); + if record.status.is_terminal() { + terminal.push((job_id.clone(), record.status)); + } + jobs.push(EnvironmentJobObservation::Job { record }); + } + JobReadEntry::Error { job_id, error } => { + jobs.push(entry_error_observation(&job_id, error)?); + } + } + } + } + let mut resolutions = BTreeMap::new(); - for result in response.jobs { - let summary = result.summary.clone(); - if summary.status.is_terminal() { - let error = (summary.status != JobStatus::Succeeded).then(|| { - summary.failure.clone().unwrap_or_else(|| { - format!( - "environment job {} ended as {:?}", - summary.job_id, summary.status - ) - }) - }); - let normalized = normalize_job_result( - deps.blobs.as_ref(), - NormalizeJobResultInput { - handle: Some(JobHandle { - environment_id: environment_id.as_str().to_owned(), - job_id: summary.job_id.clone(), - }), - summary: Some(result.summary), - output_chunks: result.output_chunks, - output_next_seq: result.output_next_seq, - artifacts: result.artifacts, - error, - output_bytes: Some(PROMISE_JOB_OUTPUT_BYTES), + for observation in &jobs { + if let EnvironmentJobObservation::NotFound { job_id } = observation { + let handle = job_handle(environment_id, job_id)?; + let result = model_job_error( + Some(handle), + format!("environment no longer knows job {job_id}"), + ); + let result_ref = + store_model_job_result(deps.blobs.as_ref(), deps.blob_graph.as_deref(), &result) + .await + .map_err(activity_error)?; + resolutions.insert( + job_id.as_str().to_owned(), + PromiseSourceCheckResult::Failed { + error_ref: Some(result_ref), }, - ) + ); + } + } + + // Each terminal job's result shows the head and the tail of its output. + let per_read = (per_request / 2).max(1); + for chunk in terminal.chunks(per_read) { + let ids = chunk + .iter() + .map(|(job_id, _)| job_id.clone()) + .collect::>(); + let output_bytes = DEFAULT_JOB_OUTPUT_BYTES.min(limits.max_read_bytes / ids.len() as u64); + let response = client + .read_jobs(&job_output_read(&ids, None, output_bytes)) .await - .map_err(activity_error)?; + .map_err(client_activity_error)?; + let mut entries = response.jobs.into_iter(); + for (job_id, status) in chunk { + let handle = job_handle(environment_id, &EnvironmentJobId::new(job_id.as_str()))?; + let mut input = NormalizeJobResultInput::from_entries( + Some(handle), + entries.by_ref().take(2).collect::>(), + ); + if !status.is_success() && input.error.is_none() { + input.error = Some( + match input.summary.as_ref().and_then(|s| s.error.as_ref()) { + Some(error) => { + format!("environment job {job_id} ended {status:?}: {error}") + } + None => format!("environment job {job_id} ended {status:?}"), + }, + ); + } + let normalized = normalize_job_result(deps.blobs.as_ref(), input) + .await + .map_err(activity_error)?; let result_ref = store_model_job_result( deps.blobs.as_ref(), deps.blob_graph.as_deref(), @@ -469,7 +646,7 @@ pub(super) async fn poll( ) .await .map_err(activity_error)?; - let resolution = if summary.status == JobStatus::Succeeded { + let resolution = if status.is_success() { PromiseSourceCheckResult::Resolved { payload_ref: Some(result_ref), } @@ -478,99 +655,333 @@ pub(super) async fn poll( error_ref: Some(result_ref), } }; - resolutions.insert(summary.job_id.as_str().to_owned(), resolution); + resolutions.insert(job_id.as_str().to_owned(), resolution); } - jobs.push(summary); } - let returned_job_ids = jobs - .iter() - .map(|job| job.job_id.clone()) - .collect::>(); - let terminal = returned_job_ids == requested_job_ids - && !jobs.is_empty() - && jobs.iter().all(|job| job.status.is_terminal()); - Ok(EnvironmentJobPollActivityResult { - jobs, - resolutions, - terminal, + Ok(EnvironmentJobPollActivityResult { jobs, resolutions }) +} + +fn job_handle( + environment_id: &EnvironmentId, + job_id: &EnvironmentJobId, +) -> Result { + Ok(JobHandle { + environment_id: environment_id.as_str().to_owned(), + job_id: wire_job_id(job_id) + .map_err(|error| non_retryable_activity_error(anyhow::anyhow!(error)))?, }) } pub(super) async fn cancel( deps: Option<&EnvironmentJobActivityDeps>, request: EnvironmentJobCancelActivityRequest, -) -> Result, ActivityError> { - let deps = deps.ok_or_else(|| { +) -> Result { + let deps = configured(deps)?; + let environment_id = EnvironmentId::try_new(request.environment_id).map_err(activity_error)?; + let job_ids = request + .jobs + .iter() + .map(wire_job_id) + .collect::, _>>() + .map_err(|error| non_retryable_activity_error(anyhow::anyhow!(error)))?; + if job_ids.is_empty() { + return Ok(EnvironmentJobCancelActivityResult { jobs: Vec::new() }); + } + let instance = usable_instance(deps, &environment_id).await?; + let client = job_client(deps, &instance).await?; + let result = cancel_jobs(&client, &job_ids, request.scope, request.force).await; + let _ = client.close().await; + result +} + +async fn cancel_jobs( + client: &DataClient, + job_ids: &[JobId], + scope: EnvironmentJobCancelScope, + force: bool, +) -> Result { + let per_request = (client.initialize_response().limits.job.max_request_jobs as usize).max(1); + let scope = match scope { + EnvironmentJobCancelScope::Job => JobCancelScope::Job, + EnvironmentJobCancelScope::Dependents => JobCancelScope::Dependents, + }; + let mut jobs = Vec::new(); + for chunk in job_ids.chunks(per_request) { + let response = client + .cancel_jobs(&CancelJobsParams { + job_ids: chunk.to_vec(), + scope, + force, + }) + .await + .map_err(client_activity_error)?; + for entry in response.jobs { + jobs.push(match entry { + JobSummaryEntry::Job { summary } => EnvironmentJobObservation::Job { + record: job_record(&summary), + }, + JobSummaryEntry::Error { job_id, error } => { + entry_error_observation(&job_id, error)? + } + }); + } + } + Ok(EnvironmentJobCancelActivityResult { jobs }) +} + +fn configured( + deps: Option<&EnvironmentJobActivityDeps>, +) -> Result<&EnvironmentJobActivityDeps, ActivityError> { + deps.ok_or_else(|| { activity_error(anyhow::anyhow!( "environment job activities are not configured" )) - })?; - let environment_id = EnvironmentId::try_new(request.environment_id).map_err(activity_error)?; - let instance = deps - .environments - .read_environment(&environment_id) + }) +} + +/// The environment through the single use gate before a job call dials it: +/// the activity retries a wait (provisioning, or a daemon reconnecting) and +/// fails for good on an environment that is closing, closed, failed or away. +async fn usable_instance( + deps: &EnvironmentJobActivityDeps, + environment_id: &EnvironmentId, +) -> Result { + use crate::environments::resolver::{EnvironmentResolveError, EnvironmentResolver}; + + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(activity_error)? + .as_millis(); + EnvironmentResolver::new(deps.environments.clone()) + .resolve_for_connection(environment_id, i64::try_from(now_ms).unwrap_or(i64::MAX)) .await - .map_err(activity_error)?; + .map_err(|error| match error { + EnvironmentResolveError::NotReady { .. } + | EnvironmentResolveError::Unreachable { .. } + | EnvironmentResolveError::Store(environments::EnvironmentRegistryError::Store { + .. + }) => activity_error(error), + other => non_retryable_activity_error(other), + }) +} + +/// An initialized data client for the environment; the daemon must serve +/// the `job` family. +async fn job_client( + deps: &EnvironmentJobActivityDeps, + instance: &EnvironmentRecord, +) -> Result { let gateway = deps.gateway.as_ref().ok_or_else(|| { non_retryable_activity_error(anyhow::anyhow!( "environment gateway is not configured on this worker" )) })?; - let connection = gateway.connection_for(deps.universe_id, &instance); - let (mut client, _) = initialized_client(&connection, gateway).await?; - let response = client - .cancel_jobs(&CancelJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: request.jobs, - scope: request.scope, - force: request.force, - }) - .await - .map_err(activity_error)?; - Ok(response.jobs) + let endpoint = gateway.endpoint_for(deps.universe_id, instance); + let (client, initialized) = DataClient::connect_initialized( + &endpoint, + gateway + .connect_options(JOB_ACTIVITY_USER_AGENT) + .with_request_timeout(JOB_REQUEST_TIMEOUT), + ) + .await + .map_err(client_activity_error)?; + if initialized.capabilities.job.is_none() { + let _ = client.close().await; + return Err(non_retryable_activity_error(anyhow::anyhow!( + "environment does not serve durable jobs: {}", + instance.environment_id + ))); + } + Ok(client) } -async fn initialized_client( - connection: &EnvironmentDataConnection, - gateway: &crate::environments::gateway::EnvironmentGatewayClientConfig, -) -> Result< - ( - EnvironmentDataClient, - environment_protocol::shared::EnvironmentCapabilities, - ), - ActivityError, -> { - if connection.transport != EnvironmentTransport::WebSocket { - return Err(activity_error(anyhow::anyhow!( - "unsupported environment job transport: {:?}", - connection.transport - ))); +/// Retry what may pass on a later attempt (transport trouble, busy or +/// unavailable daemons, server defects); refusals of the request itself, +/// such as `invalidParams` or `failedPrecondition {jobIdInUse}`, and +/// version or plane mismatches fail the activity for good. +fn client_activity_error(error: ClientError) -> ActivityError { + if is_transient(&error) { + activity_error(error) + } else { + non_retryable_activity_error(error) } - let mut client = EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("lightspeed-environment-job-workflow"), - ) - .await - .map_err(activity_error)?; - let response = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-environment-job-workflow".to_owned(), - scope: connection.scope.clone(), - resume_connection_id: None, - }) - .await - .map_err(activity_error)?; - if response.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(activity_error(anyhow::anyhow!( - "unsupported environment protocol version {}", - response.protocol_version +} + +fn is_transient(error: &ClientError) -> bool { + error.is_retryable() + || matches!( + error, + ClientError::Decode(_) | ClientError::InvalidMessage(_) + ) + || error.protocol_kind() == Some(ProtocolErrorKind::Internal) +} + +fn non_retryable_activity_error(error: impl Into) -> ActivityError { + ActivityError::application(ApplicationFailure::non_retryable(error.into())) +} + +#[cfg(test)] +mod tests { + use environment_protocol::error::PreconditionReason; + + use super::*; + + fn submitted_spec() -> JobSpec { + JobSpec { + job_id: JobId::new("build").unwrap(), + name: Some(JobName::new("build & test").unwrap()), + command: CommandSpec { + argv: vec!["make".to_owned(), "test".to_owned()], + cwd: Some(EnvironmentPath::new("/workspace").unwrap()), + env: BTreeMap::from([("CI".to_owned(), "1".to_owned())]), + secret_env: BTreeMap::new(), + input: ExecInput::Bytes { + data: ByteChunk::from(b"input".to_vec()), + }, + timeout_ms: Some(1_000), + }, + depends_on: vec![ + JobDependency::JobId { + job_id: JobId::new("setup").unwrap(), + }, + JobDependency::Name { + name: JobName::new("lint").unwrap(), + }, + ], + dependency_policy: JobDependencyPolicy::AllTerminal, + queue_key: Some(QueueKey::new("ci").unwrap()), + } + } + + #[test] + fn stored_specs_map_back_onto_the_wire_unchanged() { + let spec = submitted_spec(); + let stored = workflow_job_spec(spec.clone()).unwrap(); + assert_eq!(stored.command.stdin.as_deref(), Some("input")); + assert_eq!(stored.command.cwd.as_deref(), Some("/workspace")); + assert_eq!(wire_job_spec(stored).unwrap(), spec); + } + + #[test] + fn stored_specs_never_hold_secrets_or_pipes() { + let mut secret = submitted_spec(); + secret + .command + .secret_env + .insert("TOKEN".to_owned(), "value".into()); + assert!(workflow_job_spec(secret).is_err()); + let mut piped = submitted_spec(); + piped.command.input = ExecInput::Pipe {}; + assert!(workflow_job_spec(piped).is_err()); + } + + #[test] + fn invalid_stored_ids_are_refused_at_the_wire_boundary() { + let mut stored = workflow_job_spec(submitted_spec()).unwrap(); + stored.job_id = EnvironmentJobId::new("not a job id"); + assert!(wire_job_spec(stored).is_err()); + } + + #[test] + fn request_ids_fit_the_id_grammar_for_any_call_id() { + assert_eq!( + workflow_tool_request_id(1, 2, 3, "toolu_01").as_str(), + "jobreq:1:2:3:toolu_01" + ); + let hashed = workflow_tool_request_id(1, 2, 3, "call id/with spaces"); + assert!(hashed.as_str().starts_with("jobreq-")); + assert_eq!( + hashed, + workflow_tool_request_id(1, 2, 3, "call id/with spaces") + ); + assert!( + derived_job_run_id("invocation") + .as_str() + .starts_with("job-") + ); + } + + #[test] + fn job_group_ids_follow_the_encoded_payload() { + let environment_id = EnvironmentId::new("environment_1"); + let payload = EnvironmentJobStartPayloadV1::new( + "jobreq:1:1:1:call", + vec![workflow_job_spec(submitted_spec()).unwrap()], + ); + let encoded = payload.encode().unwrap(); + let first = derived_job_group_id(&environment_id, &BlobRef::from_bytes(&encoded)); + assert_eq!( + first, + derived_job_group_id(&environment_id, &BlobRef::from_bytes(&encoded)) + ); + let other = EnvironmentJobStartPayloadV1::new("jobreq:1:1:1:other", payload.jobs.clone()); + assert_ne!( + first, + derived_job_group_id( + &environment_id, + &BlobRef::from_bytes(&other.encode().unwrap()) + ) + ); + } + + #[test] + fn not_found_entries_become_lost_observations() { + let job_id = JobId::new("build").unwrap(); + assert_eq!( + entry_error_observation(&job_id, ProtocolError::not_found("unknown job")).unwrap(), + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("build") + } + ); + assert!(entry_error_observation(&job_id, ProtocolError::internal("broken")).is_err()); + } + + #[test] + fn refusals_are_terminal_and_transport_trouble_is_retried() { + let retryable = |error: ClientError| is_transient(&error); + assert!(!retryable(ClientError::Protocol( + ProtocolError::failed_precondition(PreconditionReason::JobIdInUse, "job id in use") + ))); + assert!(!retryable(ClientError::Protocol( + ProtocolError::invalid_params("bad") ))); + assert!(retryable(ClientError::Protocol( + ProtocolError::unavailable("daemon restarting") + ))); + assert!(retryable(ClientError::Protocol(ProtocolError::internal( + "bug" + )))); + assert!(retryable(ClientError::Connect { + endpoint: "ws://gateway.test".to_owned(), + message: "refused".to_owned(), + })); + assert!(retryable(ClientError::DeadlineExceeded { + elapsed_ms: 60_000 + })); + } + + #[test] + fn daemon_summaries_map_onto_workflow_records() { + let summary: JobSummary = serde_json::from_value(serde_json::json!({ + "jobId": "build", + "requestId": "jobreq:1:1:1:call", + "name": "build", + "status": "timedOut", + "dependsOn": ["setup"], + "queueKey": "ci", + "createdAtMs": 1, + "stoppedBy": "timeout", + })) + .unwrap(); + assert_eq!( + job_record(&summary), + EnvironmentJobRecord { + job_id: EnvironmentJobId::new("build"), + name: Some("build".to_owned()), + status: EnvironmentJobStatus::TimedOut, + depends_on: vec![EnvironmentJobId::new("setup")], + queue_key: Some("ci".to_owned()), + } + ); } - let capabilities = response.capabilities; - client - .initialized(&InitializedParams {}) - .await - .map_err(activity_error)?; - Ok((client, capabilities)) } diff --git a/crates/temporal-runtime/src/worker/activities/mod.rs b/crates/temporal-runtime/src/worker/activities/mod.rs index 659f12b0b..ebd67a6c8 100644 --- a/crates/temporal-runtime/src/worker/activities/mod.rs +++ b/crates/temporal-runtime/src/worker/activities/mod.rs @@ -708,7 +708,7 @@ impl WorkerActivities { self: Arc, _ctx: ActivityContext, request: EnvironmentJobCancelActivityRequest, - ) -> Result, ActivityError> { + ) -> Result { let state = self.state_for_universe(request.universe_id).await?; environment_jobs::cancel(state.environment_jobs(), request).await } diff --git a/crates/temporal-runtime/src/worker/session_tools.rs b/crates/temporal-runtime/src/worker/session_tools.rs index 0b0369a12..009195c47 100644 --- a/crates/temporal-runtime/src/worker/session_tools.rs +++ b/crates/temporal-runtime/src/worker/session_tools.rs @@ -1,13 +1,10 @@ use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; -use environment_client::{EnvironmentClientError, EnvironmentDataClient, WebSocketConnectOptions}; +use environment_client::{ClientError, DataClient}; use environment_protocol::{ - data::{ - handshake::{InitializeParams, InitializedParams}, - jobs::{JobReadResult as ProtocolJobReadResult, ReadJobsParams}, - }, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentDataConnection, EnvironmentTransport}, + data::fs::{FileKind, GetMetadataParams}, + paths::EnvironmentPath, }; use environments::{EnvironmentId, EnvironmentRecord, EnvironmentRegistryError, EnvironmentStore}; use harness::PromiseIdAllocator; @@ -32,10 +29,10 @@ use tools::{ is_environment_selection_tool, }, environment::jobs::{ - JOB_RUN_WORKFLOW_SEMANTIC_TYPE, JOB_RUN_WORKFLOW_TOOL_ID, + DEFAULT_JOB_OUTPUT_BYTES, JOB_RUN_WORKFLOW_SEMANTIC_TYPE, JOB_RUN_WORKFLOW_TOOL_ID, JOB_SUBMIT_WORKFLOW_SEMANTIC_TYPE, JOB_SUBMIT_WORKFLOW_TOOL_ID, JobHandle, JobHandleArg, JobReadArgs, JobSubmitExecutionContextV1, ModelJobResult, ModelJobResultSet, - NormalizeJobResultInput, normalize_job_result, + NormalizeJobResultInput, job_output_read, model_job_error, normalize_job_result, }, environment_protocol::RemoteEnvironmentConnection, fs::{AttachedVfsFileSystem, FsPath, FsToolContext}, @@ -417,8 +414,7 @@ impl SessionTools { environments, args.jobs, args.output_bytes, - args.after_seq, - args.include_artifacts, + args.after, ) .await?; self.succeeded_job_read_result(call, result.entries).await @@ -442,10 +438,12 @@ impl SessionTools { policy: Option<&harness::EnvironmentsFeature>, environments: &SessionEnvironmentManager, handles: Vec, - output_bytes: Option, - after_seq: Option, - include_artifacts: bool, + output_bytes: Option, + after: Option, ) -> Result { + let output_bytes = output_bytes + .unwrap_or(DEFAULT_JOB_OUTPUT_BYTES) + .min(ToolLimits::default().max_model_visible_output_bytes); let mut entries = Vec::with_capacity(handles.len()); for handle in handles { let resolved = match resolve_job_handle_arg(active_environment_id, policy, handle) { @@ -478,6 +476,28 @@ impl SessionTools { environments.environment(environment_id.as_str()).cloned() { (Ok(environment), false) + } else if let Some(resolver) = self.environment_resolver.as_ref() { + // The single use gate, as for the active environment. + let resource = match now_unix_ms().and_then(|now| { + i64::try_from(now) + .map_err(|_| io_error("current timestamp does not fit in i64")) + }) { + Ok(now) => resolver + .resolve_for_connection(&environment_id, now) + .await + .map_err(RuntimeEnvironmentError::Refused), + Err(error) => Err(error.into()), + }; + ( + match resource { + Ok(resource) => { + self.runtime_environment_for_resource(session_id, resource, None) + .await + } + Err(error) => Err(error), + }, + true, + ) } else if let Some(store) = self.environment_store.as_ref() { ( match store.read_environment(&environment_id).await { @@ -485,15 +505,13 @@ impl SessionTools { self.runtime_environment_for_resource(session_id, resource, None) .await } - Err(error) => Err(map_environments_error(error)), + Err(error) => Err(map_environments_error(error).into()), }, true, ) } else { ( - Err(io_error( - "environment store is not configured on this runtime", - )), + Err(io_error("environment store is not configured on this runtime").into()), false, ) }; @@ -517,26 +535,26 @@ impl SessionTools { } continue; }; + // The head and the tail of the output, or a page from `after`, + // within the daemon's read ceiling. + let budget = jobs.limits().map_or(output_bytes, |limits| { + output_bytes.min(limits.max_read_bytes) + }); match jobs - .read_jobs(ReadJobsParams { - namespace: environment_id.as_str().to_owned(), - jobs: vec![resolved.job_id.clone()], - after_seq, - max_bytes: output_bytes, - include_artifacts, - wait_ms: None, - }) + .read_jobs(job_output_read( + std::slice::from_ref(&resolved.job_id), + after, + budget, + )) .await { Ok(response) => { + let input = + NormalizeJobResultInput::from_entries(Some(resolved), response.jobs); entries.push( - job_read_entry_from_response( - self.blobs.as_ref(), - resolved, - response.jobs.into_iter().next(), - output_bytes, - ) - .await?, + normalize_job_result(self.blobs.as_ref(), input) + .await + .map_err(map_blob_error)?, ); } Err(error) => { @@ -1069,14 +1087,7 @@ impl SessionTools { .await; } }; - let mut output = - environment_model_view(attachment, Some(&environment), active, policy); - if crate::environments::resolver::wake_on_use_applies(&environment) { - output.status_message = Some(format!( - "Environment is {}. Tools that use this environment will automatically wake it and wait until it is ready. You can proceed normally.", - format!("{:?}", environment.status).to_lowercase(), - )); - } + let output = environment_model_view(attachment, Some(&environment), active, policy); self.succeeded_tool_result( call, &output, @@ -1117,7 +1128,8 @@ impl SessionTools { .await; } }; - let ready = environment.status == environments::EnvironmentStatus::Ready; + let status = environment.display_status(); + let ready = status == environments::EnvironmentDisplayStatus::Ready; let reference = tools::environment::handles::environment_reference( environment.environment_id.as_str(), policy @@ -1129,7 +1141,7 @@ impl SessionTools { environment_id: reference.clone(), active: true, ready, - status: format!("{:?}", environment.status).to_lowercase(), + status: status.as_str().to_owned(), access: attachment.access.describe().to_owned(), working_directory: attachment.working_directory.clone(), }; @@ -1144,7 +1156,7 @@ impl SessionTools { "Active environment set to {} (access: {}; currently {}; availability is checked when an environment tool uses it).", reference, attachment.access.describe(), - format!("{:?}", environment.status).to_lowercase() + status ) }; let mut result = self.succeeded_tool_result(call, &output, summary).await?; @@ -1219,12 +1231,13 @@ impl SessionTools { )) => return Err(io_error(message)), Err(crate::environments::resolver::EnvironmentResolveError::NotReady { environment_id, - status, + reason, + .. }) => { return Ok(environments.with_active_blocker( ActiveEnvironmentBlocker::NotReady { environment_id, - status, + reason, }, )); } @@ -1254,13 +1267,7 @@ impl SessionTools { .await { Ok(environment) => environment, - Err(error) => { - return Ok(environments.with_active_blocker( - ActiveEnvironmentBlocker::Unavailable { - message: error.to_string(), - }, - )); - } + Err(error) => return Ok(environments.with_active_blocker(error.into_blocker())), }; environments.insert_environment(environment); Ok(environments) @@ -1271,71 +1278,38 @@ impl SessionTools { session_id: &SessionId, resource: EnvironmentRecord, working_directory: Option<&str>, - ) -> Result { + ) -> Result { let gateway = self .environment_gateway .as_ref() .ok_or_else(|| io_error("environment gateway is not configured on this worker"))?; - let connection = gateway.connection_for( + let endpoint = gateway.endpoint_for( self.environment_resolver .as_ref() .map(|resolver| resolver.universe_id()) .unwrap_or_default(), &resource, ); - let mut client = connect_environment_data_client( - &connection, + let (client, _) = DataClient::connect_initialized( + &endpoint, gateway.connect_options("lightspeed-temporal-runtime"), ) - .await?; - let response = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "lightspeed-temporal-runtime".to_owned(), - scope: connection.scope.clone(), - resume_connection_id: None, - }) - .await - .map_err(map_environment_client_error)?; - if response.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(io_error(format!( - "unsupported environment data protocol version {}; expected {CURRENT_PROTOCOL_VERSION}", - response.protocol_version - ))); - } - client - .initialized(&InitializedParams {}) - .await - .map_err(map_environment_client_error)?; - let cwd = if response.capabilities.filesystem_read { - match crate::environments::sources::working_directory( - &mut client, - working_directory, - response.default_cwd.as_deref(), - ) - .await - { - Ok(cwd) => cwd, - Err(error) => { - let _ = client.close().await; - return Err(io_error(error)); - } + .await + .map_err(|error| { + match crate::environments::resolver::route_refusal_error(&resource, &error) { + Some(refusal) => RuntimeEnvironmentError::Refused(refusal), + None => map_environment_client_error(error).into(), + } + })?; + let cwd = match environment_working_directory(&client, working_directory).await { + Ok(cwd) => cwd, + Err(error) => { + let _ = client.close().await; + return Err(io_error(error).into()); } - } else { - let cwd = working_directory - .or(response.default_cwd.as_deref()) - .ok_or_else(|| io_error("environment has no working directory"))?; - tools::environment::sources::absolute(std::path::Path::new("/"), cwd) - .map_err(io_error)? - .to_string_lossy() - .into_owned() }; - let cwd = Some(FsPath::new(cwd).map_err(io_error)?); - - let mut remote_connection = RemoteEnvironmentConnection::new(client, response.capabilities); - if let Some(cwd) = cwd { - remote_connection = remote_connection.with_cwd(cwd); - } + let remote_connection = RemoteEnvironmentConnection::new(client) + .with_cwd(FsPath::new(cwd.as_str()).map_err(io_error)?); let (_fs_context, mut environment_context) = remote_connection.clone().into_contexts(self.blobs.clone()); if let Some(credentials) = &self.environment_credentials { @@ -1454,16 +1428,36 @@ fn environment_model_view( .iter() .map(|attachment| attachment.environment_id.as_str()), ), - provider_id: environment - .and_then(|environment| environment.provider_id().map(|id| id.as_str().to_owned())), display_name: environment.and_then(|environment| environment.display_name.clone()), - status: environment.map(|environment| format!("{:?}", environment.status).to_lowercase()), + status: environment.map(|environment| environment.display_status().as_str().to_owned()), access: attachment.access.describe().to_owned(), default: attachment.default, working_directory: attachment.working_directory.clone(), active: active.is_some_and(|active| active.as_str() == attachment.environment_id), - observed_at_ms: environment.map(|environment| environment.observed_at_ms()), - status_message: None, + status_message: environment.and_then(environment_status_message), + } +} + +/// What the model should know about an environment that is not ready: a +/// powered-down target wakes when a tool uses it, a disconnected daemon may +/// come back, and a failed environment says why. +fn environment_status_message(environment: &EnvironmentRecord) -> Option { + use environments::EnvironmentDisplayStatus as Status; + match environment.display_status() { + status @ (Status::Paused | Status::Suspended | Status::Stopped) => Some(format!( + "Environment is {status}. Tools that use it wake it and wait until it is ready." + )), + Status::Away => Some( + match environment.registered().map(|source| source.identity_mode) { + Some(environments::RegisteredIdentityMode::Persistent) => { + "The daemon is disconnected; tools wait up to two minutes for it to reconnect." + } + _ => "The daemon is disconnected.", + } + .to_owned(), + ), + Status::Failed => environment.state_message.clone(), + _ => None, } } @@ -1571,73 +1565,46 @@ fn supplied_environment_policy( .ok_or_else(|| io_error("environment grant is missing from the batch request")) } -async fn job_read_entry_from_response( - blobs: &dyn BlobStore, - handle: JobHandle, - response: Option, - output_bytes: Option, -) -> Result { - match response { - Some(response) => normalize_job_result( - blobs, - NormalizeJobResultInput { - handle: Some(handle), - summary: Some(response.summary), - output_chunks: response.output_chunks, - output_next_seq: response.output_next_seq, - artifacts: response.artifacts, - output_bytes, - ..Default::default() - }, - ) - .await - .map_err(map_blob_error), - None => Ok(model_job_error( - Some(handle), - "provider returned no job result".to_owned(), - )), - } -} - -fn model_job_error(handle: Option, error: String) -> ModelJobResult { - ModelJobResult { - handle: handle.map(|mut handle| { - handle.environment_id = - tools::environment::handles::environment_handle(&handle.environment_id); - handle - }), - summary: None, - output: Vec::new(), - output_next_seq: 0, - truncated: false, - artifacts: Vec::new(), - error: Some(error), - } -} - -async fn connect_environment_data_client( - connection: &EnvironmentDataConnection, - options: WebSocketConnectOptions, -) -> Result, CoreAgentIoError> { - match &connection.transport { - EnvironmentTransport::WebSocket => { - EnvironmentDataClient::connect(&connection.endpoint, options) - .await - .map_err(map_environment_client_error) +/// The session's working directory in the environment: the attachment's +/// configured one, else the daemon's `defaultCwd`. It must be a directory, +/// a symlink to one inside the fs root included; a daemon without the +/// filesystem family cannot be asked, so its directory is taken as given. +async fn environment_working_directory( + client: &DataClient, + configured: Option<&str>, +) -> Result { + let initialized = client.initialize_response(); + let cwd = match configured { + Some(value) => { + if !value.starts_with('/') { + return Err(format!( + "environment working directory must be absolute: {value}" + )); + } + let normalized = + tools::environment::sources::absolute(std::path::Path::new("/"), value)?; + EnvironmentPath::new(normalized.to_string_lossy()) + .map_err(|error| format!("environment working directory {value}: {error}"))? } - EnvironmentTransport::Http => Err(unsupported_environment_data_transport("http")), - EnvironmentTransport::Stdio => Err(unsupported_environment_data_transport("stdio")), - EnvironmentTransport::Ssh => Err(unsupported_environment_data_transport("ssh")), - EnvironmentTransport::Provider { provider_type } => Err( - unsupported_environment_data_transport(format!("provider:{provider_type}")), - ), + None => initialized.default_cwd.clone(), + }; + if initialized.capabilities.filesystem.is_none() { + return Ok(cwd); } -} - -fn unsupported_environment_data_transport(transport: impl std::fmt::Display) -> CoreAgentIoError { - io_error(format!( - "environment data transport is not supported by this worker: {transport}" - )) + let response = client + .get_metadata(&GetMetadataParams { + path: cwd.clone(), + follow_symlinks: true, + }) + .await + .map_err(|error| format!("working directory {}: {error}", cwd.as_str()))?; + if response.metadata.kind != FileKind::Directory { + return Err(format!( + "working directory is not a directory: {}", + cwd.as_str() + )); + } + Ok(cwd) } #[derive(Clone, Copy)] @@ -1871,14 +1838,14 @@ impl CoreAgentTools for SessionTools { ToolCallExecution::EnvironmentNotReady { call_id, environment_id, - status, + reason, } => { failed_result( self.blobs.as_ref(), call_id, active_environment_blocker_message(&ActiveEnvironmentBlocker::NotReady { environment_id, - status, + reason, }), ) .await @@ -1892,12 +1859,12 @@ impl CoreAgentTools for SessionTools { pub enum ToolCallExecution { Completed(ToolInvocationResult), /// The call did not execute because the session's active environment is - /// still provisioning or booting. The workflow waits for readiness and - /// re-dispatches the same call. + /// provisioning, waking, booting, or its daemon is reconnecting. The + /// workflow waits for readiness and re-dispatches the same call. EnvironmentNotReady { call_id: harness::ToolCallId, environment_id: String, - status: environments::EnvironmentStatus, + reason: environments::EnvironmentWaitReason, }, } @@ -2042,12 +2009,12 @@ impl SessionTools { match environments.active_blocker() { Some(ActiveEnvironmentBlocker::NotReady { environment_id, - status, + reason, }) => { return Ok(ToolCallExecution::EnvironmentNotReady { call_id: call.call_id, environment_id: environment_id.clone(), - status: *status, + reason: *reason, }); } Some(blocker @ ActiveEnvironmentBlocker::Unavailable { .. }) => { @@ -2130,50 +2097,84 @@ impl SessionTools { } }; let mut last_status; + // A `disconnected` route refusal carries no bound of its own (the row + // still reads connected), so consecutive ones are bounded here; once + // the row records the disconnect the gate's own bound applies. + let mut disconnected_since: Option = None; loop { heartbeat().await; let now = i64::try_from(now_unix_ms().unwrap_or_default()).unwrap_or(i64::MAX); - match resolver.ready_for_use(&environment_id, now).await { + let poll = match resolver.ready_for_use(&environment_id, now).await { Ok(_) => return Outcome::Ready, Err(crate::environments::resolver::EnvironmentResolveError::NotReady { - status, + reason, + retry_after_ms, + until_ms, .. }) => { - last_status = format!("{status:?}").to_lowercase(); + last_status = reason.as_str().to_owned(); + if reason == environments::EnvironmentWaitReason::Disconnected + && until_ms.is_none() + { + let since = + *disconnected_since.get_or_insert_with(tokio::time::Instant::now); + if since.elapsed() >= REGISTERED_RECONNECT_WAIT { + return Outcome::Failed { + message: format!( + "environment {environment_id}: the daemon did not reconnect within {} seconds", + REGISTERED_RECONNECT_WAIT.as_secs() + ), + }; + } + } else { + disconnected_since = None; + } + std::time::Duration::from_millis(retry_after_ms) + .clamp(ENVIRONMENT_RETRY_AFTER_MIN, ENVIRONMENT_RETRY_AFTER_MAX) } - Err( - crate::environments::resolver::EnvironmentResolveError::EnvironmentUnavailable { - status, - .. - }, - ) => { - // Marked ready in the registry but the route probe failed; - // keep polling until the deadline, the daemon may still - // be coming up. + Err(crate::environments::resolver::EnvironmentResolveError::Unreachable { + status, + .. + }) => { + // The dial failed without a typed refusal: a gateway + // restart, a provider 5xx, or a daemon that has not yet + // missed its heartbeat. Keep polling until the deadline. + disconnected_since = None; last_status = status; + temporal_workflow::ENVIRONMENT_READY_POLL_INTERVAL } Err(crate::environments::resolver::EnvironmentResolveError::Store( environments::EnvironmentRegistryError::Store { message }, )) => { // Transient store trouble: keep polling. + disconnected_since = None; last_status = format!("store error: {message}"); + temporal_workflow::ENVIRONMENT_READY_POLL_INTERVAL } Err(error) => { return Outcome::Failed { message: error.to_string(), }; } - } + }; if tokio::time::Instant::now() >= deadline { return Outcome::TimedOut { last_status }; } let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); - tokio::time::sleep(temporal_workflow::ENVIRONMENT_READY_POLL_INTERVAL.min(remaining)) - .await; + tokio::time::sleep(poll.min(remaining)).await; } } } +/// Bounds on a not-ready wait's `Retry-After` between readiness polls. +const ENVIRONMENT_RETRY_AFTER_MIN: std::time::Duration = std::time::Duration::from_millis(250); +const ENVIRONMENT_RETRY_AFTER_MAX: std::time::Duration = std::time::Duration::from_secs(10); +/// How long consecutive `disconnected` route refusals of a persistent +/// daemon may last before the wait fails; an ephemeral daemon's refusal is +/// already terminal. +const REGISTERED_RECONNECT_WAIT: std::time::Duration = + std::time::Duration::from_millis(environments::REGISTERED_RECONNECT_WAIT_MS.unsigned_abs()); + /// Model-facing text for a call that cannot use the active environment. /// A job handle names the session's active environment unless the model /// says otherwise: `job_submit` and `job_run` only ever start jobs there, @@ -2199,10 +2200,9 @@ fn active_environment_blocker_message(blocker: &ActiveEnvironmentBlocker) -> Str match blocker { ActiveEnvironmentBlocker::NotReady { environment_id, - status, + reason, } => format!( - "active environment {environment_id} is {} and not reachable yet; retry once it is ready", - format!("{status:?}").to_lowercase() + "active environment {environment_id} is not reachable yet ({reason}); retry once it is ready" ), ActiveEnvironmentBlocker::Unavailable { message } => { format!("active environment is unavailable: {message}") @@ -2302,10 +2302,52 @@ fn map_environments_error(error: EnvironmentRegistryError) -> CoreAgentIoError { io_error(format!("load session environment bindings: {error}")) } -fn map_environment_client_error(error: EnvironmentClientError) -> CoreAgentIoError { +fn map_environment_client_error(error: ClientError) -> CoreAgentIoError { io_error(format!("environment data-plane call failed: {error}")) } +/// Why an environment record did not become a runtime environment. +#[derive(Debug)] +enum RuntimeEnvironmentError { + /// The use gate or a typed data-route refusal answered. + Refused(crate::environments::resolver::EnvironmentResolveError), + Failed(CoreAgentIoError), +} + +impl RuntimeEnvironmentError { + /// A wait keeps the call for re-dispatch; anything else fails it. + fn into_blocker(self) -> ActiveEnvironmentBlocker { + match self { + Self::Refused(crate::environments::resolver::EnvironmentResolveError::NotReady { + environment_id, + reason, + .. + }) => ActiveEnvironmentBlocker::NotReady { + environment_id, + reason, + }, + other => ActiveEnvironmentBlocker::Unavailable { + message: other.to_string(), + }, + } + } +} + +impl From for RuntimeEnvironmentError { + fn from(error: CoreAgentIoError) -> Self { + Self::Failed(error) + } +} + +impl std::fmt::Display for RuntimeEnvironmentError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Refused(error) => error.fmt(formatter), + Self::Failed(error) => error.fmt(formatter), + } + } +} + fn map_blob_error(error: BlobStoreError) -> CoreAgentIoError { io_error(format!("write tool error blob: {error}")) } @@ -2335,15 +2377,15 @@ mod tests { use tools::environment::control::ENVIRONMENT_LIST_TOOL_NAME; use crate::environments::runtime::RuntimeEnvironment; - use environment_protocol::shared::{EnvironmentTransport, ProviderTargetId}; + use environment_protocol::ids::{JobId, ProviderTargetId}; use environments::{ - CreateEnvironment, EnvironmentConnectionSpec, EnvironmentIncarnationId, - EnvironmentIncarnationRecord, EnvironmentProviderBindingId, - EnvironmentProviderBindingStatus, EnvironmentProviderBindingStore, EnvironmentProviderId, - EnvironmentProviderStore, EnvironmentProvisionRequestId, EnvironmentSource, - EnvironmentStatus, EnvironmentStore, EnvironmentTemplateId, - InMemoryEnvironmentRegistryStore, ObserveProvisionedEnvironment, PutEnvironmentProvider, - PutEnvironmentProviderBinding, + CreateEnvironment, DaemonBuild, EnvironmentEndpoint, EnvironmentProviderBindingStore, + EnvironmentProviderId, EnvironmentProviderStore, EnvironmentProvisionRequestId, + EnvironmentSource, EnvironmentState, EnvironmentStore, EnvironmentTemplateId, + EnvironmentWaitReason, InMemoryEnvironmentRegistryStore, ObservedTarget, OpenEnvironment, + PowerState, ProviderCredentialChange, PutEnvironmentProvider, + PutEnvironmentProviderBinding, RecordTargetObservation, RegisteredIdentityMode, + RegisteredSource, TargetPhase, }; use harness::{ BlobRef, ContextEntryKind, FunctionToolSpec, RunId, SessionId, ToolBatchId, ToolCallId, @@ -2944,9 +2986,7 @@ mod tests { let sessions = InMemorySessionStore::new(); let (session_id, binding) = workflow_tool_session(blobs.as_ref(), &sessions).await; let registry = Arc::new(InMemoryEnvironmentRegistryStore::new()); - let tools = tools.with_environment_resolver( - crate::environments::resolver::EnvironmentResolver::new(registry.clone(), registry), - ); + let tools = tools.with_environment_resolver(test_resolver(registry.clone())); let sleep_args = blobs.put_bytes(br#"{"ms":50}"#.to_vec()).await.unwrap(); let workflow_args = blobs .put_bytes(br#"{"status":"complete"}"#.to_vec()) @@ -4256,35 +4296,7 @@ mod tests { blobs: Arc, process: Arc, ) -> RuntimeEnvironment { - let target_id = ProviderTargetId::new("test"); - let resource = environments::EnvironmentRecord { - environment_id: harness::EnvironmentId::new("test"), - request_id: EnvironmentProvisionRequestId::new("request-test"), - source: EnvironmentSource::Provisioned { - provider_id: EnvironmentProviderId::new("test-provider"), - binding_id: EnvironmentProviderBindingId::new("test-binding"), - }, - display_name: None, - status: EnvironmentStatus::Offline, - desired_power: environments::PowerState::Running, - idle_policy: None, - incarnation: EnvironmentIncarnationRecord { - incarnation_id: EnvironmentIncarnationId::new("incarnation-test"), - provision_request_id: Some(EnvironmentProvisionRequestId::new("request-test")), - provider_target_id: Some(target_id.clone()), - template_id: Some(EnvironmentTemplateId::new("test-template")), - adoption_source_target: None, - power_states: Vec::new(), - created_at_ms: 1, - updated_at_ms: 1, - }, - public_ingress_enabled: false, - public_endpoint: None, - metadata: BTreeMap::new(), - last_seen_at_ms: None, - created_at_ms: 1, - updated_at_ms: 1, - }; + let resource = open_test_record("test", "test-provider"); let fs_context = tools::fs::FsToolContext::new( Arc::new(tools::fs::InMemoryFileSystem::full_access()), blobs.clone(), @@ -4298,23 +4310,31 @@ mod tests { #[cfg(any(target_os = "linux", target_os = "macos"))] mod transfers { use super::*; - use environment_protocol::data::transfer_session::{TransferRequest, TransferResponse}; - - struct LocalTransfer(environment_daemon::filesystem::LocalFileSystem); - - #[async_trait] - impl tools::transfer::EnvironmentTransfer for LocalTransfer { - async fn request( - &self, - request: TransferRequest, - ) -> tools::ToolResult { - self.0 - .transfer(request) - .await - .map_err(|error| tools::ToolError::InvalidRequest { - message: error.message, - }) - } + use environment_protocol::handshake::ImplementationInfo; + + /// envd's data plane over an in-process socket, rooted at `root` + /// with its state dir beside it. + async fn in_process_envd(root: &std::path::Path, state: &std::path::Path) -> DataClient { + let runtime = + environment_daemon::DaemonRuntime::new(environment_daemon::config::DaemonConfig { + listen: None, + cwd: root.into(), + fs_root: root.into(), + state_dir: state.into(), + read_only_fs: false, + registration: None, + scrubbed_env: Vec::new(), + }) + .expect("envd runtime"); + let (client_io, server_io) = tokio::io::duplex(1 << 20); + tokio::spawn(environment_daemon::server::serve_stream(runtime, server_io)); + let options = environment_client::ConnectOptions::new( + ImplementationInfo::new("session-tools-test").expect("client info"), + ); + DataClient::connect_initialized_over(client_io, options) + .await + .expect("connect to in-process envd") + .0 } async fn dispatch( @@ -4368,19 +4388,19 @@ mod tests { ] { let (blobs, tools, session_id, workspace_attachments) = session_tools_with_readme_attachment().await; - let root = tempfile::tempdir().unwrap(); - let fs = environment_daemon::filesystem::LocalFileSystem::new( - root.path().into(), - root.path().into(), - true, - ); + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap().join("root"); + std::fs::create_dir_all(&root).unwrap(); + let client = + in_process_envd(&root, &temp.path().canonicalize().unwrap().join("state")) + .await; let environment = test_environment( blobs.clone(), Arc::new(RecordingProcessExecutor::default()), ); let mut context = environment.tool_context().clone(); - context.process_cwd = Some(FsPath::new(root.path().to_string_lossy()).unwrap()); - context.transfer = Some(Arc::new(LocalTransfer(fs))); + context.process_cwd = Some(FsPath::new(root.to_string_lossy()).unwrap()); + context.transfer = Some(Arc::new(client)); let environment = RuntimeEnvironment::from_resource(environment.resource().clone(), context); let tools = tools @@ -4403,20 +4423,17 @@ mod tests { let result = dispatch(&tools, request.clone(), batch).await; succeeded(&blobs, &result).await; assert_eq!( - std::fs::read(root.path().join("created/nested/README.md")).unwrap(), + std::fs::read(root.join("created/nested/README.md")).unwrap(), b"hello\n" ); // A completed retry through the hosted path must keep its operation // identity and preserve edits made after publication. - std::fs::write( - root.path().join("created/nested/README.md"), - b"environment edit", - ) - .unwrap(); + std::fs::write(root.join("created/nested/README.md"), b"environment edit") + .unwrap(); succeeded(&blobs, &dispatch(&tools, request.clone(), batch).await).await; assert_eq!( - std::fs::read(root.path().join("created/nested/README.md")).unwrap(), + std::fs::read(root.join("created/nested/README.md")).unwrap(), b"environment edit" ); @@ -4503,6 +4520,49 @@ mod tests { } } + fn test_daemon_build() -> DaemonBuild { + DaemonBuild { + version: Some("test".to_owned()), + git_sha: None, + protocol_version: environment_protocol::ENVD_PROTOCOL_VERSION, + } + } + + fn test_create_environment( + environment_id: &str, + provider_id: &str, + created_at_ms: i64, + ) -> CreateEnvironment { + CreateEnvironment { + request_id: EnvironmentProvisionRequestId::new(format!("request-{environment_id}")), + environment_id: EnvironmentId::new(environment_id), + provider_id: EnvironmentProviderId::new(provider_id), + template_id: EnvironmentTemplateId::new("test-template"), + idle: None, + display_name: None, + metadata: BTreeMap::new(), + created_at_ms, + } + } + + /// An open provisioned record, built without a store. + fn open_test_record( + environment_id: &str, + provider_id: &str, + ) -> environments::EnvironmentRecord { + let mut record = test_create_environment(environment_id, provider_id, 1) + .record() + .expect("provisioned record"); + OpenEnvironment { + environment_id: record.environment_id.clone(), + daemon_build: test_daemon_build(), + at_ms: 2, + } + .apply(&mut record) + .expect("open record"); + record + } + async fn register_test_environment_provider( store: &InMemoryEnvironmentRegistryStore, provider_id: &str, @@ -4511,11 +4571,12 @@ mod tests { .put_provider(PutEnvironmentProvider { provider_id: EnvironmentProviderId::new(provider_id), display_name: None, - controller_connection: EnvironmentConnectionSpec::new( - "http://controller.test", - EnvironmentTransport::Http, - ), - metadata: BTreeMap::new(), + controller_endpoint: EnvironmentEndpoint::parse("ws://controller.test/control") + .unwrap(), + data_route_base: None, + credential: ProviderCredentialChange::Set(auth::SecretValue::new( + "provider-credential", + )), updated_at_ms: 10, }) .await @@ -4523,113 +4584,162 @@ mod tests { store .put_provider_binding(PutEnvironmentProviderBinding { universe_id: store.universe_id(), - binding_id: EnvironmentProviderBindingId::new(format!("binding-{provider_id}")), provider_id: EnvironmentProviderId::new(provider_id), - status: EnvironmentProviderBindingStatus::Enabled, - expected_revision: None, - metadata: BTreeMap::new(), + enabled: true, updated_at_ms: 10, }) .await .expect("register provider binding"); } - async fn observe_test_environment( + /// Create a provisioned environment and open it, as the reconciler does + /// after its first handshake. + async fn open_test_environment( store: &InMemoryEnvironmentRegistryStore, environment_id: &str, provider_id: &str, - observed_at_ms: i64, + opened_at_ms: i64, ) { - let target_id = ProviderTargetId::new(format!("target-{environment_id}")); - let environment_id = EnvironmentId::new(environment_id); + let created = store + .create_environment(test_create_environment( + environment_id, + provider_id, + opened_at_ms.saturating_sub(1), + )) + .await + .expect("create environment"); store - .create_environment(CreateEnvironment { - request_id: EnvironmentProvisionRequestId::new(format!("request-{environment_id}")), - environment_id: environment_id.clone(), - incarnation_id: EnvironmentIncarnationId::new(format!( - "incarnation-{environment_id}" - )), - binding_id: EnvironmentProviderBindingId::new(format!("binding-{provider_id}")), - template_id: EnvironmentTemplateId::new("test-template"), - display_name: None, - metadata: BTreeMap::new(), - - idle_policy: None, - created_at_ms: observed_at_ms.saturating_sub(1), + .record_target_observation(RecordTargetObservation { + environment_id: created.environment_id.clone(), + provider_target_id: Some( + ProviderTargetId::new(format!("target-{environment_id}")).unwrap(), + ), + observed: ObservedTarget { + phase: TargetPhase::Active, + power: Some(PowerState::Running), + message: None, + observed_at_ms: opened_at_ms, + }, + public_endpoint: None, + observed_at_ms: opened_at_ms, }) .await - .expect("create environment"); + .expect("record target"); store - .observe_provisioned_environment(ObserveProvisionedEnvironment { - environment_id, - provider_target_id: target_id, - status: EnvironmentStatus::Offline, - power_states: Vec::new(), - observed_at_ms, + .open_environment(OpenEnvironment { + environment_id: created.environment_id, + daemon_build: test_daemon_build(), + at_ms: opened_at_ms, }) .await - .expect("observe environment"); + .expect("open environment"); } - #[tokio::test(flavor = "current_thread")] - async fn environment_read_status_message_only_describes_supported_wake_on_use() { - use environments::PowerState; + fn test_resolver( + registry: Arc, + ) -> crate::environments::resolver::EnvironmentResolver { + crate::environments::resolver::EnvironmentResolver::new(registry) + } + + #[test] + fn status_messages_tell_the_model_what_a_tool_call_will_meet() { + let open = open_test_record("environment-1", "allowed"); + assert_eq!(environment_status_message(&open), None); + + let mut paused = open.clone(); + if let EnvironmentSource::Provisioned(source) = &mut paused.source { + source.observed = Some(ObservedTarget { + phase: TargetPhase::Active, + power: Some(PowerState::Paused), + message: None, + observed_at_ms: 3, + }); + } + assert_eq!( + environment_status_message(&paused).as_deref(), + Some("Environment is paused. Tools that use it wake it and wait until it is ready.") + ); - for (status, power_states, expected_status, expected_message) in [ + let mut failed = open.clone(); + failed.state = EnvironmentState::Failed; + failed.state_message = Some("no capacity".to_owned()); + assert_eq!( + environment_status_message(&failed).as_deref(), + Some("no capacity") + ); + + let away = |identity_mode| { + let mut record = open.clone(); + record.source = EnvironmentSource::Registered(RegisteredSource { + registration_key_id: environments::EnvironmentRegistrationKeyId::new("key"), + daemon_public_key: "00".repeat(32), + identity_mode, + connected: false, + connection_changed_at_ms: 3, + last_seen_at_ms: 3, + }); + record + }; + assert_eq!( + environment_status_message(&away(RegisteredIdentityMode::Persistent)).as_deref(), + Some("The daemon is disconnected; tools wait up to two minutes for it to reconnect.") + ); + assert_eq!( + environment_status_message(&away(RegisteredIdentityMode::Ephemeral)).as_deref(), + Some("The daemon is disconnected.") + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn environment_views_report_display_status_and_wake_messages() { + for (power, expected_status, expected_message) in [ + (Some(PowerState::Running), "ready", None), ( - EnvironmentStatus::Paused, - vec![PowerState::Running], + Some(PowerState::Paused), "paused", - true, + Some( + "Environment is paused. Tools that use it wake it and wait until it is ready.", + ), ), ( - EnvironmentStatus::Suspended, - vec![PowerState::Running], + Some(PowerState::Suspended), "suspended", - true, - ), - ( - EnvironmentStatus::Offline, - vec![PowerState::Running], - "offline", - true, - ), - (EnvironmentStatus::Paused, Vec::new(), "paused", false), - (EnvironmentStatus::Suspended, Vec::new(), "suspended", false), - (EnvironmentStatus::Offline, Vec::new(), "offline", false), - ( - EnvironmentStatus::Ready, - vec![PowerState::Running], - "ready", - false, + Some( + "Environment is suspended. Tools that use it wake it and wait until it is ready.", + ), ), ( - EnvironmentStatus::Failed, - vec![PowerState::Running], - "failed", - false, + Some(PowerState::Stopped), + "stopped", + Some( + "Environment is stopped. Tools that use it wake it and wait until it is ready.", + ), ), ] { let blobs = Arc::new(InMemoryBlobStore::new()); let registry = Arc::new(InMemoryEnvironmentRegistryStore::new()); register_test_environment_provider(registry.as_ref(), "allowed").await; let environment_id = EnvironmentId::new("environment-allowed-1"); - observe_test_environment(registry.as_ref(), environment_id.as_str(), "allowed", 10) - .await; + open_test_environment(registry.as_ref(), environment_id.as_str(), "allowed", 10).await; registry - .observe_provisioned_environment(ObserveProvisionedEnvironment { + .record_target_observation(RecordTargetObservation { environment_id: environment_id.clone(), - provider_target_id: ProviderTargetId::new("target-environment-allowed-1"), - status, - power_states, + provider_target_id: Some( + ProviderTargetId::new("target-environment-allowed-1").unwrap(), + ), + observed: ObservedTarget { + phase: TargetPhase::Active, + power, + message: None, + observed_at_ms: 11, + }, + public_endpoint: None, observed_at_ms: 11, }) .await - .expect("observe status and power support"); - let resolver = - crate::environments::resolver::EnvironmentResolver::new(registry.clone(), registry); + .expect("observe power"); let tools = SessionTools::new(blobs.clone(), Arc::new(TestCatalog::default())) - .with_environment_resolver(resolver); + .with_environment_resolver(test_resolver(registry)); blobs.put_bytes(b"{}".to_vec()).await.expect("arguments"); for tool_name in ["environment_read", "environment_list"] { @@ -4664,15 +4774,11 @@ mod tests { &output["environments"][0] }; assert_eq!(view["status"], expected_status); - if tool_name == "environment_read" && expected_message { - assert_eq!( - view["status_message"], - format!( - "Environment is {expected_status}. Tools that use this environment will automatically wake it and wait until it is ready. You can proceed normally." - ), - ); - } else { - assert!(view.get("status_message").is_none(), "{tool_name}: {view}"); + assert!(view.get("provider_id").is_none(), "{tool_name}: {view}"); + assert!(view.get("observed_at_ms").is_none(), "{tool_name}: {view}"); + match expected_message { + Some(message) => assert_eq!(view["status_message"], message), + None => assert!(view.get("status_message").is_none(), "{tool_name}: {view}"), } } } @@ -4685,9 +4791,8 @@ mod tests { let blobs = Arc::new(InMemoryBlobStore::new()); let registry = Arc::new(InMemoryEnvironmentRegistryStore::new()); register_test_environment_provider(registry.as_ref(), "allowed").await; - observe_test_environment(registry.as_ref(), id, "allowed", 10).await; - let resolver = - crate::environments::resolver::EnvironmentResolver::new(registry.clone(), registry); + open_test_environment(registry.as_ref(), id, "allowed", 10).await; + let resolver = test_resolver(registry.clone()); let tools = SessionTools::new(blobs.clone(), Arc::new(TestCatalog::default())) .with_environment_resolver(resolver); let policy = test_environment_policy(&[id]); @@ -4723,7 +4828,7 @@ mod tests { Some(&policy), JobHandleArg { environment_id: Some(handle.into()), - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), }, ) .unwrap(); @@ -4744,7 +4849,7 @@ mod tests { Some(&test_environment_policy(&["other"])), JobHandleArg { environment_id: Some(handle.into()), - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), } ) .is_err() @@ -4757,11 +4862,8 @@ mod tests { let catalog = Arc::new(TestCatalog::default()); let registry = Arc::new(InMemoryEnvironmentRegistryStore::new()); register_test_environment_provider(registry.as_ref(), "allowed").await; - observe_test_environment(registry.as_ref(), "environment-allowed-1", "allowed", 10).await; - let resolver = crate::environments::resolver::EnvironmentResolver::new( - registry.clone(), - registry.clone(), - ); + open_test_environment(registry.as_ref(), "environment-allowed-1", "allowed", 10).await; + let resolver = test_resolver(registry.clone()); let tools = SessionTools::new(blobs.clone(), catalog).with_environment_resolver(resolver); let arguments_ref = blobs .put_bytes(br#"{}"#.to_vec()) @@ -4820,24 +4922,14 @@ mod tests { register_test_environment_provider(registry.as_ref(), "allowed").await; // Created but never observed: the environment is still provisioning. registry - .create_environment(CreateEnvironment { - request_id: EnvironmentProvisionRequestId::new("request-pending"), - environment_id: EnvironmentId::new("environment-pending"), - incarnation_id: EnvironmentIncarnationId::new("incarnation-pending"), - binding_id: EnvironmentProviderBindingId::new("binding-allowed"), - template_id: EnvironmentTemplateId::new("test-template"), - display_name: None, - metadata: BTreeMap::new(), - - idle_policy: None, - created_at_ms: 10, - }) + .create_environment(test_create_environment( + "environment-pending", + "allowed", + 10, + )) .await .expect("create environment"); - let resolver = crate::environments::resolver::EnvironmentResolver::new( - registry.clone(), - registry.clone(), - ); + let resolver = test_resolver(registry.clone()); let tools = SessionTools::new(blobs.clone(), catalog).with_environment_resolver(resolver); let arguments_ref = blobs .put_bytes(br#"{"path":"README.md"}"#.to_vec()) @@ -4887,7 +4979,7 @@ mod tests { execution, ToolCallExecution::EnvironmentNotReady { ref environment_id, - status: EnvironmentStatus::Provisioning, + reason: EnvironmentWaitReason::Provisioning, .. } if environment_id == "environment-pending" )); @@ -4896,18 +4988,16 @@ mod tests { let result = tools.invoke_call(request).await.expect("invoke call"); assert_eq!(result.status, harness::ToolCallStatus::Failed); - // Once observed ready, resolution no longer blocks (the route probe - // itself is exercised by live tests). + // Once open, the gate no longer blocks (the route probe itself is + // exercised by live tests). registry - .observe_provisioned_environment(ObserveProvisionedEnvironment { + .open_environment(OpenEnvironment { environment_id: EnvironmentId::new("environment-pending"), - provider_target_id: ProviderTargetId::new("target-pending"), - status: EnvironmentStatus::Ready, - power_states: Vec::new(), - observed_at_ms: 20, + daemon_build: test_daemon_build(), + at_ms: 20, }) .await - .expect("observe ready"); + .expect("open environment"); let outcome = tools .await_environment_ready( &temporal_workflow::AwaitEnvironmentReadyActivityRequest { @@ -4932,19 +5022,17 @@ mod tests { let blobs = Arc::new(InMemoryBlobStore::new()); let registry = Arc::new(InMemoryEnvironmentRegistryStore::new()); register_test_environment_provider(registry.as_ref(), "allowed").await; - observe_test_environment(registry.as_ref(), "environment-failed", "allowed", 10).await; + open_test_environment(registry.as_ref(), "environment-failed", "allowed", 10).await; registry - .fail_environment_lifecycle(environments::FailEnvironmentLifecycle { + .fail_environment(environments::FailEnvironment { environment_id: EnvironmentId::new("environment-failed"), message: "no capacity".to_owned(), - observed_at_ms: 11, + daemon_build: None, + at_ms: 11, }) .await .expect("fail environment"); - let resolver = crate::environments::resolver::EnvironmentResolver::new( - registry.clone(), - registry.clone(), - ); + let resolver = test_resolver(registry.clone()); let tools = SessionTools::new(blobs, Arc::new(TestCatalog::default())) .with_environment_resolver(resolver); let outcome = tools @@ -5016,12 +5104,9 @@ mod tests { let registry = Arc::new(InMemoryEnvironmentRegistryStore::new()); register_test_environment_provider(registry.as_ref(), "allowed").await; register_test_environment_provider(registry.as_ref(), "denied").await; - observe_test_environment(registry.as_ref(), "environment-allowed-1", "allowed", 10).await; - observe_test_environment(registry.as_ref(), "environment-denied", "denied", 10).await; - let resolver = crate::environments::resolver::EnvironmentResolver::new( - registry.clone(), - registry.clone(), - ); + open_test_environment(registry.as_ref(), "environment-allowed-1", "allowed", 10).await; + open_test_environment(registry.as_ref(), "environment-denied", "denied", 10).await; + let resolver = test_resolver(registry.clone()); let tools = SessionTools::new(blobs.clone(), catalog).with_environment_resolver(resolver); let arguments_ref = blobs .put_bytes(br#"{}"#.to_vec()) @@ -5079,7 +5164,7 @@ mod tests { ); assert_eq!(first_environments[0]["active"], true); assert_eq!(first_environments[0]["access"], "read, edit, exec, jobs"); - assert_eq!(first_environments[0]["status"], "offline"); + assert_eq!(first_environments[0]["status"], "ready"); assert_eq!( first_environments[1]["environment_id"], "environment-allowed-2" @@ -5091,7 +5176,7 @@ mod tests { .any(|environment| environment["environment_id"] == "environment-denied") ); - observe_test_environment(registry.as_ref(), "environment-allowed-2", "allowed", 20).await; + open_test_environment(registry.as_ref(), "environment-allowed-2", "allowed", 20).await; let second = tools .invoke_batch(request) .await @@ -5114,7 +5199,7 @@ mod tests { .as_array() .expect("second environments"); assert_eq!(second_environments.len(), 2); - assert_eq!(second_environments[1]["status"], "offline"); + assert_eq!(second_environments[1]["status"], "ready"); } #[tokio::test(flavor = "current_thread")] @@ -5742,7 +5827,7 @@ mod tests { ])), JobHandleArg { environment_id: None, - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), }, ) .expect("defaults to the active environment"); @@ -5757,7 +5842,7 @@ mod tests { ])), JobHandleArg { environment_id: Some("environment_other".to_owned()), - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), }, ) .expect("explicit environment wins"); @@ -5769,7 +5854,7 @@ mod tests { None, JobHandleArg { environment_id: None, - job_id: environment_protocol::shared::JobId::new("build"), + job_id: JobId::new("build").unwrap(), }, ) .is_err(), diff --git a/crates/temporal-runtime/tests/code_capabilities_live.rs b/crates/temporal-runtime/tests/code_capabilities_live.rs index 328f22ff1..afcb60895 100644 --- a/crates/temporal-runtime/tests/code_capabilities_live.rs +++ b/crates/temporal-runtime/tests/code_capabilities_live.rs @@ -313,6 +313,7 @@ where let mut tasks = Tasks(vec![task.abort_handle()]); let directory = tempfile::tempdir()?; let root = directory.path().canonicalize()?; + let daemon_state = tempfile::tempdir()?; let context = support::live::local_request_context_for(api::AccessScope::Universe { universe_id: universe, }) @@ -324,20 +325,20 @@ where let environment_id = if environment { let key = state.api.create_environment_registration_key(api::EnvironmentRegistrationKeyCreateParams { display_name:"Code capability test".into(), identity_mode:api::EnvironmentIdentityModeView::Ephemeral, - max_active_environments:Some(1), ephemeral_disconnect_grace_ms:None, expires_at_ms:None, + max_active_environments:Some(1), ephemeral_disconnect_grace_ms:None, offline_close_after_ms:None, expires_at_ms:None, }).await?.result; - let connect_url = format!("ws://{address}/environment-gateway/connect"); + let connect_url = format!("ws://{address}{}", environment_protocol::routes::CONNECT_PATH); let mut registration = environment_daemon::config::RegistrationConfig::new(connect_url.clone(), environment_daemon::upgrade::resolve_discovery_url(Some(&connect_url), None)?); - registration.registration_key = Some(environment_protocol::shared::SecretString::new(key.secret.0)); + registration.registration_key = Some(environment_protocol::bytes::SecretString::new(key.secret.0)); let daemon = environment_daemon::DaemonRuntime::new(environment_daemon::config::DaemonConfig { - listen:None, cwd:root.clone(), fs_root:root.clone(), state_dir:root.join(".envd"), read_only_fs:false, registration:Some(registration), scrubbed_env:vec![], + listen:None, cwd:root.clone(), fs_root:root.clone(), state_dir:daemon_state.path().into(), read_only_fs:false, registration:Some(registration), scrubbed_env:vec![], })?; let task = tokio::spawn(environment_daemon::server::run(daemon)); tasks.0.push(task.abort_handle()); Some(tokio::time::timeout(Duration::from_secs(30), async { loop { let records = state.api.list_environments(api::EnvironmentListParams::default()).await?.result.environments; - if let Some(record) = records.into_iter().find(|record| record.status == api::EnvironmentLifecycleStatusView::Ready) { return anyhow::Ok(record.environment_id); } + if let Some(record) = records.into_iter().find(|record| record.state == api::EnvironmentStateView::Open) { return anyhow::Ok(record.environment_id); } tokio::time::sleep(Duration::from_millis(100)).await; } }).await??) @@ -643,9 +644,11 @@ async fn jobs_and_subagents_case(fixture: Fixture) -> anyhow::Result<()> { .as_ref() .is_some_and(|handle| handle.environment_id == tools::environment::handles::environment_handle(environment)) - && actual.summary.as_ref().is_some_and(|summary| summary.status - == environment_protocol::data::jobs::JobStatus::Succeeded - && &summary.namespace == environment), + && actual + .summary + .as_ref() + .is_some_and(|summary| summary.status + == tools::environment::jobs::ModelJobStatus::Succeeded), "job must finish on the admitted environment: {actual:?}" ); } diff --git a/crates/temporal-runtime/tests/environment_provider_live.rs b/crates/temporal-runtime/tests/environment_provider_live.rs index 6f2c35b1b..25dc0884e 100644 --- a/crates/temporal-runtime/tests/environment_provider_live.rs +++ b/crates/temporal-runtime/tests/environment_provider_live.rs @@ -1,44 +1,49 @@ -//! Minimal live environment control-plane acceptance test. +//! Live environment provider acceptance tests. //! -//! This deliberately uses the in-process provider with real Postgres and the -//! real deployment reconciler. It covers the durable provider and ingress seams -//! without requiring an Incus installation; the separate Incus live smoke test -//! owns backend connectivity and topology. +//! These use the fake controller, served on loopback and registered like +//! any provider, with real Postgres and the real reconciler. They cover the +//! provider seams without requiring an Incus installation: create until +//! accepted and opened by one handshake, close until gone, the +//! synchronous power and ingress pass-throughs, wake on a data route dial, +//! the display sweep's provider-ended close, and per-row backoff while a +//! provider is down. The separate Incus live smoke test owns backend +//! connectivity and topology. mod support; use std::{collections::BTreeMap, sync::Arc, time::Duration}; use api::{ - AgentApiService, DeploymentApiService, DeploymentEnvironmentAdoptParams, - DeploymentEnvironmentProviderConnection, DeploymentEnvironmentProviderDeleteParams, - DeploymentEnvironmentProviderPutParams, DeploymentEnvironmentProviderTransport, + AgentApiService, DeploymentApiService, DeploymentEnvironmentProviderDeleteParams, + DeploymentEnvironmentProviderPutParams, DeploymentProviderBindingDeleteParams, DeploymentProviderBindingPutParams, DeploymentUniverseCreateParams, EnvironmentCloseParams, - EnvironmentCreateParams, EnvironmentIngressPutParams, EnvironmentLifecycleStatusView, - EnvironmentListParams, EnvironmentProviderBindingStatusView, EnvironmentTemplateListParams, - SessionConfig, SessionStartParams, + EnvironmentCloseReasonView, EnvironmentCreateParams, EnvironmentIngressPutParams, + EnvironmentListParams, EnvironmentPowerPutParams, EnvironmentPowerStateView, + EnvironmentSourceView, EnvironmentStateView, EnvironmentStatusView, + EnvironmentTemplateListParams, EnvironmentView, }; -use api_projection::model_to_api; -use harness::SessionId; -use support::live::{ - LIVE_TEST_LOCK, fake_worker_activities, read_session_view, require_storage_live_env, - run_with_live_worker, wait_for_environment_status, +use environment_client::conformance::data_route::expect_refusal; +use environment_protocol::{ + control::target::TargetOwner, + ids::{EnvironmentId as ProtocolEnvironmentId, ProviderTargetId, UniverseId}, + routes::{DataRouteBase, DataRouteRefusal, provider_data_route}, }; +use environments::EnvironmentStore; +use support::live::wait_for_environment_state; use temporal_runtime::{ DeploymentStores, UniverseRuntime, - gateway::{GatewayAgentApi, GatewayDeploymentApi}, - pg_store_from_env, + environments::fake_controller::{ + FAKE_INGRESS_ENDPOINT, FAKE_TEMPLATE_ID, FakeController, FakeControllerOptions, + }, + gateway::{ + GatewayAgentApi, GatewayDeploymentApi, + request_context::{RequestContext, with_request_context as calling}, + }, }; use temporal_workflow::{DEFAULT_TEMPORAL_NAMESPACE, DEFAULT_TEMPORAL_TARGET, connect_temporal}; -use temporalio_client::Client; use uuid::Uuid; -#[tokio::test(flavor = "current_thread")] -#[ignore = "requires ./dev.sh infra or compatible Temporal + Postgres env"] -async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Result<()> { - let _ = dotenvy::dotenv(); - require_live_env()?; - +async fn live_runtime(name: &str) -> anyhow::Result<(Arc, sqlx::PgPool)> { let temporal_target = std::env::var("TEMPORAL_ADDRESS").unwrap_or_else(|_| DEFAULT_TEMPORAL_TARGET.to_owned()); let namespace = std::env::var("TEMPORAL_NAMESPACE") @@ -46,277 +51,596 @@ async fn environment_provider_lifecycle_and_adoption_round_trip() -> anyhow::Res let client = connect_temporal(&temporal_target, &namespace).await?; let stores = DeploymentStores::from_env().await?; let pool = stores.pool().clone(); - let suffix = Uuid::new_v4().simple().to_string(); - let provider_id = format!("fake-live-{suffix}"); - let universe_a = Uuid::new_v4(); - let universe_b = Uuid::new_v4(); let runtime = Arc::new(UniverseRuntime::new( client, - format!("environment-provider-live-{suffix}"), + format!("{name}-{}", Uuid::new_v4().simple()), Some("http://127.0.0.1:18080".to_owned()), stores, )?); + Ok((runtime, pool)) +} + +fn deployment_caller() -> RequestContext { + RequestContext::local(api::AccessScope::Deployment) +} + +fn universe_caller(universe_id: Uuid) -> RequestContext { + RequestContext::local(api::AccessScope::Universe { universe_id }) +} + +/// Create a universe, register `controller` as `provider_id` and bind it. +async fn provider_universe( + operator: &GatewayDeploymentApi, + universe_id: Uuid, + provider_id: &str, + controller: &FakeController, +) -> anyhow::Result<()> { + calling( + deployment_caller(), + operator.create_universe(DeploymentUniverseCreateParams { + slug: None, + universe_id: universe_id.to_string(), + }), + ) + .await?; + calling( + deployment_caller(), + operator.put_environment_provider(DeploymentEnvironmentProviderPutParams { + provider_id: provider_id.to_owned(), + display_name: Some("Live fake provider".to_owned()), + controller_endpoint: controller.controller_url().to_owned(), + data_route_base: None, + credential: controller.credential().map(str::to_owned), + clear_credential: false, + }), + ) + .await?; + calling( + deployment_caller(), + operator.put_environment_provider_binding(DeploymentProviderBindingPutParams { + universe_id: universe_id.to_string(), + provider_id: provider_id.to_owned(), + enabled: true, + }), + ) + .await?; + Ok(()) +} + +/// Remove what [`provider_universe`] made, best effort. +async fn cleanup( + runtime: &UniverseRuntime, + operator: &GatewayDeploymentApi, + pool: &sqlx::PgPool, + universe_id: Uuid, + provider_id: &str, +) { + let _ = calling( + deployment_caller(), + operator.delete_environment_provider_binding(DeploymentProviderBindingDeleteParams { + universe_id: universe_id.to_string(), + provider_id: provider_id.to_owned(), + }), + ) + .await; + runtime.evict(universe_id).await; + let _ = store_pg::delete_universe(pool, universe_id).await; + let _ = calling( + deployment_caller(), + operator.delete_environment_provider(DeploymentEnvironmentProviderDeleteParams { + provider_id: provider_id.to_owned(), + }), + ) + .await; +} + +fn create_params(request_id: &str, provider_id: &str) -> EnvironmentCreateParams { + EnvironmentCreateParams { + request_id: request_id.to_owned(), + provider_id: provider_id.to_owned(), + template_id: FAKE_TEMPLATE_ID.to_owned(), + display_name: Some("Provisioned live VM".to_owned()), + metadata: BTreeMap::new(), + idle: None, + } +} + +/// Poll a read until `done` holds, with the background reconciler doing +/// the work. +async fn wait_for( + api: &GatewayAgentApi, + environment_id: &str, + what: &str, + done: impl Fn(&EnvironmentView) -> bool, +) -> anyhow::Result { + let started = std::time::Instant::now(); + loop { + let environment = api + .read_environment(api::EnvironmentReadParams { + environment_id: environment_id.to_owned(), + }) + .await? + .result + .environment; + if done(&environment) { + return Ok(environment); + } + if started.elapsed() > Duration::from_secs(15) { + anyhow::bail!( + "timed out waiting for environment {environment_id} to {what}; it is {:?} ({:?})", + environment.state, + environment.state_message + ); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +fn target_id(environment: &EnvironmentView) -> Option { + match &environment.source { + EnvironmentSourceView::Provisioned { target_id, .. } => target_id.clone(), + _ => None, + } +} + +fn public_endpoint(environment: &EnvironmentView) -> Option { + match &environment.source { + EnvironmentSourceView::Provisioned { + public_endpoint, .. + } => public_endpoint.clone(), + _ => None, + } +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Temporal + Postgres env"] +async fn environment_provider_lifecycle_round_trip() -> anyhow::Result<()> { + let _ = dotenvy::dotenv(); + require_live_env()?; + let (runtime, pool) = live_runtime("environment-provider-live").await?; let operator = GatewayDeploymentApi::new(runtime.clone()); + let suffix = Uuid::new_v4().simple().to_string(); + let guarded_id = format!("fake-live-{suffix}"); + let open_id = format!("fake-open-{suffix}"); + let universe_a = Uuid::new_v4(); + let universe_b = Uuid::new_v4(); + let guarded = FakeController::start().await?; + let open = FakeController::start_with(FakeControllerOptions { + credential: false, + ..FakeControllerOptions::default() + }) + .await?; let reconciler = tokio::spawn(runtime.clone().run_environment_reconciler()); - // Direct service calls carry an explicit local request context. - use temporal_runtime::gateway::request_context::{ - RequestContext, with_request_context as calling, - }; - let deployment_caller = RequestContext::local(api::AccessScope::Deployment); let result = async { - for universe_id in [universe_a, universe_b] { + provider_universe(&operator, universe_a, &guarded_id, &guarded).await?; + provider_universe(&operator, universe_b, &open_id, &open).await?; + + // Registration ran the handshake; nothing it advertised is stored, + // and the data route base is derived from the controller endpoint. + let read = |provider_id: String| { calling( - deployment_caller.clone(), - operator.create_universe(DeploymentUniverseCreateParams { - slug: None, - universe_id: universe_id.to_string(), + deployment_caller(), + operator.read_environment_provider(api::DeploymentEnvironmentProviderReadParams { + provider_id, }), ) - .await?; + }; + let guarded_view = read(guarded_id.clone()).await?.result.provider; + assert!(guarded_view.has_credential); + assert_eq!(guarded_view.data_route_base, None); + let effective = guarded_view + .effective_data_route_base + .clone() + .expect("a ws:// controller has a derivable data route base"); + assert!(effective.ends_with("/routes"), "{effective}"); + assert!(!read(open_id.clone()).await?.result.provider.has_credential); + // A controller URL that names no controller is refused outright, and + // so is a guarded controller registered without its credential. + for (controller_endpoint, credential) in [ + ( + guarded.controller_url().replace("/control", "/elsewhere"), + guarded.credential().map(str::to_owned), + ), + (guarded.controller_url().to_owned(), None), + ] { + let refused = calling( + deployment_caller(), + operator.put_environment_provider(DeploymentEnvironmentProviderPutParams { + provider_id: format!("{guarded_id}-refused"), + display_name: None, + controller_endpoint, + data_route_base: None, + credential, + clear_credential: false, + }), + ) + .await + .expect_err("a definitive handshake refusal is refused"); + assert_eq!(refused.kind, api::AgentApiErrorKind::InvalidRequest); } - calling( - deployment_caller.clone(), + let exclusive = calling( + deployment_caller(), operator.put_environment_provider(DeploymentEnvironmentProviderPutParams { - provider_id: provider_id.clone(), - display_name: Some("Live fake provider".to_owned()), - controller_connection: DeploymentEnvironmentProviderConnection { - endpoint: "in-process".to_owned(), - transport: DeploymentEnvironmentProviderTransport::Provider { - provider_type: "fake".to_owned(), - }, - }, - metadata: BTreeMap::new(), + provider_id: guarded_id.clone(), + display_name: None, + controller_endpoint: guarded.controller_url().to_owned(), + data_route_base: None, + credential: Some("token".to_owned()), + clear_credential: true, }), ) - .await?; - for (universe_id, binding_id) in [(universe_a, "primary-a"), (universe_b, "primary-b")] { - calling( - deployment_caller.clone(), - operator.put_environment_provider_binding(DeploymentProviderBindingPutParams { - universe_id: universe_id.to_string(), - binding_id: binding_id.to_owned(), - provider_id: provider_id.clone(), - status: EnvironmentProviderBindingStatusView::Enabled, - metadata: BTreeMap::new(), - expected_revision: None, - }), - ) - .await?; - } + .await + .expect_err("credential and clearCredential exclude each other"); + assert_eq!(exclusive.kind, api::AgentApiErrorKind::InvalidRequest); let state_a = runtime.state_for(universe_a, false).await?; let state_b = runtime.state_for(universe_b, false).await?; - let caller_a = RequestContext::local(api::AccessScope::Universe { - universe_id: universe_a, - }); - let caller_b = RequestContext::local(api::AccessScope::Universe { - universe_id: universe_b, - }); + let caller_a = universe_caller(universe_a); + let caller_b = universe_caller(universe_b); + let api_a = state_a.api.as_ref(); + let api_b = state_b.api.as_ref(); + let templates = calling( caller_a.clone(), - state_a - .api - .list_environment_templates(EnvironmentTemplateListParams { - binding_id: Some("primary-a".to_owned()), - }), + api_a.list_environment_templates(EnvironmentTemplateListParams { + provider_id: Some(guarded_id.clone()), + }), ) - .await?; - assert_eq!(templates.result.templates.len(), 1); - assert_eq!(templates.result.templates[0].template_id, "rust-v1"); - + .await? + .result + .templates; + assert_eq!(templates.len(), 1); + assert_eq!(templates[0].template_id, FAKE_TEMPLATE_ID); + assert_eq!(templates[0].provider_id, guarded_id); + assert_eq!(templates[0].power_states.len(), 4); + assert!(templates[0].public_ingress); + + // Create until accepted, open after one handshake. let created = calling( caller_a.clone(), - state_a.api.create_environment(EnvironmentCreateParams { - request_id: format!("create-{suffix}"), - binding_id: "primary-a".to_owned(), - template_id: "rust-v1".to_owned(), - display_name: Some("Provisioned live VM".to_owned()), - metadata: BTreeMap::new(), - idle_policy: None, - }), + api_a.create_environment(create_params(&format!("create-{suffix}"), &guarded_id)), ) .await? .result .environment; - let create_retry = calling( + assert_eq!(created.state, EnvironmentStateView::Provisioning); + let retried = calling( caller_a.clone(), - state_a.api.create_environment(EnvironmentCreateParams { - request_id: format!("create-{suffix}"), - binding_id: "primary-a".to_owned(), - template_id: "rust-v1".to_owned(), - display_name: None, - metadata: BTreeMap::new(), - idle_policy: None, - }), + api_a.create_environment(create_params(&format!("create-{suffix}"), &guarded_id)), ) .await? .result .environment; - assert_eq!(create_retry.environment_id, created.environment_id); - - let adopted = calling( - deployment_caller.clone(), - operator.adopt_environment(DeploymentEnvironmentAdoptParams { - universe_id: universe_b.to_string(), - request_id: format!("adopt-{suffix}"), - binding_id: "primary-b".to_owned(), - source_target: "legacy/hand-built-vm".to_owned(), - take_ownership: true, - display_name: Some("Adopted live VM".to_owned()), - metadata: BTreeMap::new(), + assert_eq!(retried.environment_id, created.environment_id); + let environment_id = created.environment_id.clone(); + let opened = calling( + caller_a.clone(), + wait_for(api_a, &environment_id, "open", |environment| { + environment.state == EnvironmentStateView::Open }), ) + .await?; + assert_eq!(opened.status, EnvironmentStatusView::Ready); + assert_eq!( + opened.daemon.as_ref().map(|daemon| daemon.protocol_version), + Some(environment_protocol::ENVD_PROTOCOL_VERSION) + ); + let target = target_id(&opened).expect("an open environment has a target"); + + // A provider without a credential works the same way. + let open_created = calling( + caller_b.clone(), + api_b.create_environment(create_params(&format!("open-{suffix}"), &open_id)), + ) .await? .result .environment; - assert_eq!(adopted.incarnation.template_id, None); - let adopt_retry = calling( - deployment_caller.clone(), - operator.adopt_environment(DeploymentEnvironmentAdoptParams { - universe_id: universe_b.to_string(), - request_id: format!("adopt-{suffix}"), - binding_id: "primary-b".to_owned(), - source_target: "legacy/hand-built-vm".to_owned(), - take_ownership: true, - display_name: None, - metadata: BTreeMap::new(), + calling( + caller_b.clone(), + wait_for(api_b, &open_created.environment_id, "open", |environment| { + environment.state == EnvironmentStateView::Open }), ) + .await?; + + // Pages of one; each universe sees only its own environments. + let second = calling( + caller_a.clone(), + api_a.create_environment(create_params(&format!("second-{suffix}"), &guarded_id)), + ) .await? .result .environment; - assert_eq!(adopt_retry.environment_id, adopted.environment_id); - - let created_ready = calling( + calling( caller_a.clone(), - wait_for_status( - state_a.api.as_ref(), - &created.environment_id, - EnvironmentLifecycleStatusView::Ready, - ), + wait_for(api_a, &second.environment_id, "open", |environment| { + environment.state == EnvironmentStateView::Open + }), ) .await?; - let adopted_ready = calling( - caller_b.clone(), - wait_for_status( - state_b.api.as_ref(), - &adopted.environment_id, - EnvironmentLifecycleStatusView::Ready, - ), + let first_page = calling( + caller_a.clone(), + api_a.list_environments(EnvironmentListParams { + limit: Some(1), + ..EnvironmentListParams::default() + }), ) - .await?; - assert!(created_ready.incarnation.provider_target_id.is_some()); - assert!(adopted_ready.incarnation.provider_target_id.is_some()); - - let listed_a = calling( + .await? + .result; + assert_eq!(first_page.environments.len(), 1); + assert!(first_page.next_after.is_some()); + let next_page = calling( caller_a.clone(), - state_a - .api - .list_environments(EnvironmentListParams::default()), + api_a.list_environments(EnvironmentListParams { + limit: Some(1), + after: first_page.next_after.clone(), + ..EnvironmentListParams::default() + }), ) - .await?; + .await? + .result; + assert_eq!(next_page.environments.len(), 1); + assert_ne!( + next_page.environments[0].environment_id, + first_page.environments[0].environment_id + ); + // The last page carries no cursor. + assert_eq!(next_page.next_after, None); let listed_b = calling( caller_b.clone(), - state_b - .api - .list_environments(EnvironmentListParams::default()), + api_b.list_environments(EnvironmentListParams::default()), ) - .await?; - assert_eq!(listed_a.result.environments.len(), 1); - assert_eq!(listed_b.result.environments.len(), 1); + .await? + .result; + assert_eq!(listed_b.environments.len(), 1); + + // Power is a synchronous pass-through: the answer is recorded. + let paused = calling( + caller_a.clone(), + api_a.put_environment_power(EnvironmentPowerPutParams { + environment_id: environment_id.clone(), + power: EnvironmentPowerStateView::Paused, + }), + ) + .await? + .result + .environment; + assert_eq!(paused.state, EnvironmentStateView::Open); + assert_eq!(paused.status, EnvironmentStatusView::Paused); + // The data route dial is the wake signal: a paused target refuses + // `waking` while it resumes, then upgrades. + let route = provider_data_route( + &DataRouteBase::new(effective.clone())?, + &TargetOwner { + universe_id: UniverseId::new(universe_a.to_string())?, + environment_id: ProtocolEnvironmentId::new(environment_id.clone())?, + }, + &ProviderTargetId::new(target.clone())?, + ); + let options = || { + environment_client::ConnectOptions::new( + temporal_runtime::environments::gateway::runtime_client_info(), + ) + .with_bearer_token(guarded.credential().expect("guarded credential")) + }; + assert_eq!( + expect_refusal(&route, options()) + .await + .map_err(anyhow::Error::msg)?, + DataRouteRefusal::Waking {} + ); + let (client, _) = environment_client::DataClient::connect_initialized(&route, options()) + .await + .expect("the woken target upgrades"); + let _ = client.close().await; + // Ingress is a synchronous pass-through too; the endpoint is cached. let ingress = calling( caller_a.clone(), - state_a - .api - .put_environment_ingress(EnvironmentIngressPutParams { - environment_id: created.environment_id.clone(), - enabled: true, - }), + api_a.put_environment_ingress(EnvironmentIngressPutParams { + environment_id: environment_id.clone(), + enabled: true, + }), ) - .await?; + .await? + .result + .environment; assert_eq!( - ingress.result.environment.public_endpoint.as_deref(), - Some("https://fake.env.test") + public_endpoint(&ingress).as_deref(), + Some(FAKE_INGRESS_ENDPOINT) ); - calling( + let removed = calling( caller_a.clone(), - state_a - .api - .put_environment_ingress(EnvironmentIngressPutParams { - environment_id: created.environment_id.clone(), - enabled: false, - }), + api_a.put_environment_ingress(EnvironmentIngressPutParams { + environment_id: environment_id.clone(), + enabled: false, + }), + ) + .await? + .result + .environment; + assert_eq!(public_endpoint(&removed), None); + + // External environments have no provider to pass through to. + let external = calling( + caller_a.clone(), + api_a.create_external_environment(api::EnvironmentExternalCreateParams { + request_id: format!("external-{suffix}"), + endpoint: format!("ws://127.0.0.1:1/{suffix}"), + display_name: None, + metadata: BTreeMap::new(), + }), + ) + .await? + .result + .environment; + let external_power = calling( + caller_a.clone(), + api_a.put_environment_power(EnvironmentPowerPutParams { + environment_id: external.environment_id.clone(), + power: EnvironmentPowerStateView::Paused, + }), + ) + .await + .expect_err("external environments have no power control"); + assert_eq!(external_power.kind, api::AgentApiErrorKind::Rejected); + let closed_external = calling( + caller_a.clone(), + api_a.close_environment(EnvironmentCloseParams { + environment_id: external.environment_id.clone(), + }), + ) + .await? + .result + .environment; + assert_eq!(closed_external.state, EnvironmentStateView::Closed); + + // A target the provider removed on its own ends its environment at + // the next display sweep. + assert!(guarded.end_target(&second.environment_id).await); + calling(caller_a.clone(), api_a.reconcile_environments_once()).await?; + let ended = calling( + caller_a.clone(), + wait_for(api_a, &second.environment_id, "end", |environment| { + environment.state == EnvironmentStateView::Closed + }), ) .await?; + assert_eq!( + ended.close_reason, + Some(EnvironmentCloseReasonView::ProviderEnded) + ); + // Close until gone. for (api, caller, environment_id) in [ - ( - state_a.api.as_ref(), - &caller_a, - created.environment_id.as_str(), - ), - ( - state_b.api.as_ref(), - &caller_b, - adopted.environment_id.as_str(), - ), + (api_a, &caller_a, environment_id.as_str()), + (api_b, &caller_b, open_created.environment_id.as_str()), ] { calling(caller.clone(), async { api.close_environment(EnvironmentCloseParams { environment_id: environment_id.to_owned(), }) .await?; - wait_for_status(api, environment_id, EnvironmentLifecycleStatusView::Closed).await + let closed = wait_for(api, environment_id, "close", |environment| { + environment.state == EnvironmentStateView::Closed + }) + .await?; + assert_eq!( + closed.close_reason, + Some(EnvironmentCloseReasonView::Requested) + ); + anyhow::Ok(()) }) .await?; } + assert_eq!(guarded.target_count().await, 0); + assert_eq!(open.target_count().await, 0); + let closed_power = calling( + caller_a.clone(), + api_a.put_environment_power(EnvironmentPowerPutParams { + environment_id: environment_id.clone(), + power: EnvironmentPowerStateView::Running, + }), + ) + .await + .expect_err("a closed environment has no power control"); + assert_eq!(closed_power.kind, api::AgentApiErrorKind::Rejected); anyhow::Ok(()) } .await; reconciler.abort(); let _ = reconciler.await; - runtime.evict(universe_a).await; - runtime.evict(universe_b).await; - for universe_id in [universe_a, universe_b] { - let _ = store_pg::delete_universe(&pool, universe_id).await; - } - let _ = calling( - deployment_caller.clone(), - operator - .delete_environment_provider(DeploymentEnvironmentProviderDeleteParams { provider_id }), - ) - .await; + cleanup(&runtime, &operator, &pool, universe_a, &guarded_id).await; + cleanup(&runtime, &operator, &pool, universe_b, &open_id).await; result } -async fn wait_for_status( - api: &temporal_runtime::gateway::GatewayAgentApi, - environment_id: &str, - expected: EnvironmentLifecycleStatusView, -) -> anyhow::Result { - let started = std::time::Instant::now(); - loop { - let environment = api - .read_environment(api::EnvironmentReadParams { - environment_id: environment_id.to_owned(), - }) - .await? - .result - .environment; - if environment.status == expected { - return Ok(environment); - } - if started.elapsed() > Duration::from_secs(10) { - anyhow::bail!( - "timed out waiting for environment {environment_id} to reach {expected:?}; current status is {:?}", - environment.status +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires ./dev.sh infra or compatible Temporal + Postgres env"] +async fn environment_provider_outage_backs_off_per_row() -> anyhow::Result<()> { + let _ = dotenvy::dotenv(); + require_live_env()?; + let (runtime, pool) = live_runtime("environment-provider-outage-live").await?; + let operator = GatewayDeploymentApi::new(runtime.clone()); + let suffix = Uuid::new_v4().simple().to_string(); + let provider_id = format!("fake-outage-{suffix}"); + let universe_id = Uuid::new_v4(); + let controller = FakeController::start().await?; + + let result = async { + provider_universe(&operator, universe_id, &provider_id, &controller).await?; + let state = runtime.state_for(universe_id, false).await?; + let caller = universe_caller(universe_id); + let api = state.api.as_ref(); + controller.set_reachable(false); + let created = calling( + caller.clone(), + api.create_environment(create_params(&format!("outage-{suffix}"), &provider_id)), + ) + .await? + .result + .environment; + let environment_id = environments::EnvironmentId::new(created.environment_id.clone()); + // Each pass that cannot reach the provider counts one attempt and + // pushes the row's next attempt out; the row stays provisioning. + for expected in 1..=2_u32 { + calling(caller.clone(), api.reconcile_environments_once()).await?; + let record = state.store.read_environment(&environment_id).await?; + assert_eq!(record.state, environments::EnvironmentState::Provisioning); + let source = record.provisioned().expect("provisioned"); + assert_eq!(source.attempts, expected); + assert!( + source.next_attempt_at_ms.expect("scheduled") > record.updated_at_ms, + "the next attempt is backed off" + ); + assert!( + record + .state_message + .as_deref() + .is_some_and(|message| message.contains("unreachable")), + "{:?}", + record.state_message ); } - tokio::time::sleep(Duration::from_millis(100)).await; + controller.set_reachable(true); + calling( + caller.clone(), + wait_for_environment_state(api, &created.environment_id, EnvironmentStateView::Open), + ) + .await?; + let opened = state.store.read_environment(&environment_id).await?; + assert_eq!(opened.state_message, None); + assert_eq!(opened.provisioned().expect("provisioned").attempts, 0); + + // A close while the provider is down waits for it; nothing is + // abandoned. + controller.set_reachable(false); + calling( + caller.clone(), + api.close_environment(EnvironmentCloseParams { + environment_id: created.environment_id.clone(), + }), + ) + .await?; + calling(caller.clone(), api.reconcile_environments_once()).await?; + let closing = state.store.read_environment(&environment_id).await?; + assert_eq!(closing.state, environments::EnvironmentState::Closing); + assert_eq!(closing.provisioned().expect("provisioned").attempts, 1); + controller.set_reachable(true); + calling( + caller.clone(), + wait_for_environment_state(api, &created.environment_id, EnvironmentStateView::Closed), + ) + .await?; + assert_eq!(controller.target_count().await, 0); + anyhow::Ok(()) } + .await; + + cleanup(&runtime, &operator, &pool, universe_id, &provider_id).await; + result } fn require_live_env() -> anyhow::Result<()> { @@ -329,410 +653,3 @@ fn require_live_env() -> anyhow::Result<()> { } Ok(()) } - -#[tokio::test(flavor = "current_thread")] -#[ignore = "requires ./dev.sh infra or compatible Temporal + Postgres env"] -async fn temporal_live_environment_power_intent_converges_and_wakes_on_use() -> anyhow::Result<()> { - let _lock = LIVE_TEST_LOCK.lock().await; - let _ = dotenvy::dotenv(); - require_storage_live_env()?; - - let activities = fake_worker_activities().await?; - run_with_live_worker(activities, run_environment_power_live_client).await -} - -/// Power intent is recorded through the API, converged by the lifecycle -/// reconciler against the provider, and a powered-down environment wakes -/// transparently when a session selects it. Idle policy round-trips and the -/// power reaper treats an environment without a reachable daemon as -/// untouchable. -async fn run_environment_power_live_client( - client: Client, - task_queue: String, - session_id: SessionId, -) -> anyhow::Result<()> { - use std::collections::BTreeMap; - - use environment_protocol::shared::EnvironmentTransport; - use environments::{ - EnvironmentConnectionSpec, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, - EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, - PutEnvironmentProvider, PutEnvironmentProviderBinding, - }; - - let store = pg_store_from_env().await?; - let model = support::live::openai_live_model(); - support::live::seed_agent_default(&store, &model).await?; - let api = GatewayAgentApi::builder(client.clone(), store.clone()) - .with_task_queue(task_queue) - .build(); - let suffix = uuid::Uuid::new_v4().simple().to_string(); - let provider_id = format!("fake-power-{suffix}"); - let binding_id = format!("binding-power-{suffix}"); - store - .put_provider(PutEnvironmentProvider { - provider_id: EnvironmentProviderId::new(provider_id.clone()), - display_name: Some("Live fake provider (power)".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "in-process", - EnvironmentTransport::Provider { - provider_type: "fake".to_owned(), - }, - ), - metadata: BTreeMap::new(), - updated_at_ms: 1, - }) - .await?; - store - .put_provider_binding(PutEnvironmentProviderBinding { - universe_id: store.config().universe_id, - binding_id: EnvironmentProviderBindingId::new(binding_id.clone()), - provider_id: EnvironmentProviderId::new(provider_id.clone()), - status: EnvironmentProviderBindingStatus::Enabled, - expected_revision: None, - metadata: BTreeMap::new(), - updated_at_ms: 1, - }) - .await?; - - let policy = api::EnvironmentIdlePolicyView { - pause_after_ms: Some(60_000), - suspend_after_ms: None, - stop_after_ms: Some(3_600_000), - close_after_ms: None, - }; - let created = api - .create_environment(api::EnvironmentCreateParams { - request_id: format!("power-{suffix}"), - binding_id: binding_id.clone(), - template_id: "rust-v1".to_owned(), - display_name: Some("Power VM".to_owned()), - metadata: BTreeMap::new(), - idle_policy: Some(policy.clone()), - }) - .await? - .result - .environment; - let environment_id = created.environment_id.clone(); - assert_eq!( - created.desired_power, - api::EnvironmentPowerStateView::Running - ); - assert_eq!(created.idle_policy.as_ref(), Some(&policy)); - assert!(created.incarnation.power_states.is_empty()); - - // Before the provider reported power support, power changes are refused. - let premature = api - .put_environment_power(api::EnvironmentPowerPutParams { - environment_id: environment_id.clone(), - power: api::EnvironmentPowerStateView::Paused, - }) - .await - .expect_err("power change before observation is rejected"); - assert_eq!(premature.kind, api::AgentApiErrorKind::Rejected); - - let ready = wait_for_environment_status( - &api, - &environment_id, - api::EnvironmentLifecycleStatusView::Ready, - ) - .await?; - assert_eq!( - ready.incarnation.power_states, - vec![ - api::EnvironmentPowerStateView::Running, - api::EnvironmentPowerStateView::Paused, - api::EnvironmentPowerStateView::Suspended, - api::EnvironmentPowerStateView::Stopped, - ] - ); - - // A malformed idle policy is rejected; a valid replacement and a clear - // round-trip. - let bad_policy = api - .put_environment_idle_policy(api::EnvironmentIdlePolicyPutParams { - environment_id: environment_id.clone(), - idle_policy: Some(api::EnvironmentIdlePolicyView { - pause_after_ms: Some(10), - stop_after_ms: Some(5), - ..api::EnvironmentIdlePolicyView::default() - }), - }) - .await - .expect_err("non-monotone idle policy is rejected"); - assert_eq!(bad_policy.kind, api::AgentApiErrorKind::InvalidRequest); - let cleared = api - .put_environment_idle_policy(api::EnvironmentIdlePolicyPutParams { - environment_id: environment_id.clone(), - idle_policy: None, - }) - .await? - .result - .environment; - assert!(cleared.idle_policy.is_none()); - let restored = api - .put_environment_idle_policy(api::EnvironmentIdlePolicyPutParams { - environment_id: environment_id.clone(), - idle_policy: Some(policy.clone()), - }) - .await? - .result - .environment; - assert_eq!(restored.idle_policy.as_ref(), Some(&policy)); - - // The reaper sees the candidate but cannot reach a daemon through the - // fake provider, so it leaves the environment alone. - let stats = api.reap_idle_environments_once().await?; - assert_eq!(stats.candidates, 1); - assert_eq!(stats.unreachable, 1); - assert_eq!(stats.powered_down, 0); - assert_eq!(stats.closed, 0); - - // Pause intent: recorded immediately, converged by the reconciler. - let paused_intent = api - .put_environment_power(api::EnvironmentPowerPutParams { - environment_id: environment_id.clone(), - power: api::EnvironmentPowerStateView::Paused, - }) - .await? - .result - .environment; - assert_eq!( - paused_intent.desired_power, - api::EnvironmentPowerStateView::Paused - ); - assert_eq!( - paused_intent.status, - api::EnvironmentLifecycleStatusView::Ready - ); - let paused = wait_for_environment_status( - &api, - &environment_id, - api::EnvironmentLifecycleStatusView::Paused, - ) - .await?; - assert_eq!(paused.desired_power, api::EnvironmentPowerStateView::Paused); - // Paused is a filterable lifecycle status. - assert!( - api.list_environments(api::EnvironmentListParams { - metadata: Default::default(), - status: Some(api::EnvironmentLifecycleStatusView::Paused), - ..api::EnvironmentListParams::default() - }) - .await? - .result - .environments - .iter() - .any(|environment| environment.environment_id == environment_id) - ); - - // Selection only records the environment. A paused machine stays paused - // until an operation actually needs to use it. - api.start_session(SessionStartParams { - access: None, - metadata: Default::default(), - session_id: Some(session_id.as_str().to_owned()), - display_name: None, - config: Some(SessionConfig { - model: Some(model_to_api(&model)), - features: Some(api::FeaturesConfig { - environments: Some(api::EnvironmentsFeature { - version: api::CURRENT_FEATURE_VERSION, - selection: false, - prompts: None, - skills: None, - environments: vec![api::EnvironmentAttachment { - environment_id: Some(environment_id.clone()), - inherit: false, - default: false, - access: api::EnvironmentAccess::Exec, - working_directory: None, - }], - }), - ..api::FeaturesConfig::default() - }), - ..SessionConfig::default() - }), - profile: None, - delete_after_close_ms: None, - }) - .await?; - let started_view = read_session_view(&api, &session_id).await?; - assert!(started_view.active_environment_id.is_none()); - api.activate_session_environment(api::SessionEnvironmentActivateParams { - session_id: session_id.as_str().to_owned(), - environment_id: environment_id.clone(), - }) - .await?; - let activated_view = read_session_view(&api, &session_id).await?; - assert_eq!( - activated_view.active_environment_id.as_deref(), - Some(environment_id.as_str()) - ); - let selected = api - .read_environment(api::EnvironmentReadParams { - environment_id: environment_id.clone(), - }) - .await? - .result - .environment; - assert_eq!( - selected.desired_power, - api::EnvironmentPowerStateView::Paused - ); - assert_eq!(selected.status, api::EnvironmentLifecycleStatusView::Paused); - - // Suspend and stop are ordinary intents on a provider that supports them. - api.put_environment_power(api::EnvironmentPowerPutParams { - environment_id: environment_id.clone(), - power: api::EnvironmentPowerStateView::Suspended, - }) - .await?; - wait_for_environment_status( - &api, - &environment_id, - api::EnvironmentLifecycleStatusView::Suspended, - ) - .await?; - // Wake-on-use through the jobs API: creating a job against the - // suspended environment fails typed `environment_not_ready` (never a - // generic rejection) and flips desired power back to running — the - // retry-with-backoff contract polling automations lean on. - let job_not_ready = api - .create_environment_jobs(api::EnvironmentJobCreateParams { - environment_id: environment_id.clone(), - request_id: format!("wake-probe-{suffix}"), - jobs: vec![api::SessionJobStartSpecInput { - name: Some("wake-probe".to_owned()), - job_id: None, - argv: vec!["true".to_owned()], - cwd: None, - env: BTreeMap::new(), - stdin: None, - timeout_ms: None, - depends_on: Vec::new(), - dependency_policy: api::SessionJobDependencyPolicyView::default(), - queue_key: None, - }], - }) - .await - .expect_err("jobs/create against a suspended environment is not ready"); - assert_eq!( - job_not_ready.kind, - api::AgentApiErrorKind::EnvironmentNotReady - ); - let waking = api - .read_environment(api::EnvironmentReadParams { - environment_id: environment_id.clone(), - }) - .await? - .result - .environment; - assert_eq!( - waking.desired_power, - api::EnvironmentPowerStateView::Running - ); - - api.put_environment_power(api::EnvironmentPowerPutParams { - environment_id: environment_id.clone(), - power: api::EnvironmentPowerStateView::Stopped, - }) - .await?; - wait_for_environment_status( - &api, - &environment_id, - api::EnvironmentLifecycleStatusView::Offline, - ) - .await?; - api.put_environment_power(api::EnvironmentPowerPutParams { - environment_id: environment_id.clone(), - power: api::EnvironmentPowerStateView::Running, - }) - .await?; - wait_for_environment_status( - &api, - &environment_id, - api::EnvironmentLifecycleStatusView::Ready, - ) - .await?; - - // External environments have no power control. - let external = api - .create_external_environment(api::EnvironmentExternalCreateParams { - request_id: format!("power-external-{suffix}"), - connection: api::EnvironmentConnectionView { - endpoint: format!("ws://127.0.0.1:1/{suffix}"), - transport: api::EnvironmentConnectionTransportView::WebSocket, - }, - display_name: None, - metadata: BTreeMap::new(), - }) - .await? - .result - .environment; - let external_rejected = api - .put_environment_power(api::EnvironmentPowerPutParams { - environment_id: external.environment_id.clone(), - power: api::EnvironmentPowerStateView::Paused, - }) - .await - .expect_err("external environments have no power control"); - assert_eq!(external_rejected.kind, api::AgentApiErrorKind::Rejected); - let external_policy_rejected = api - .put_environment_idle_policy(api::EnvironmentIdlePolicyPutParams { - environment_id: external.environment_id.clone(), - idle_policy: Some(policy.clone()), - }) - .await - .expect_err("external environments have no idle policy"); - assert_eq!( - external_policy_rejected.kind, - api::AgentApiErrorKind::InvalidRequest - ); - - // Closing wins over power intent. - api.close_session(api::SessionCloseParams { - session_id: session_id.as_str().to_owned(), - force: false, - }) - .await?; - api.close_environment(api::EnvironmentCloseParams { - environment_id: environment_id.clone(), - }) - .await?; - wait_for_environment_status( - &api, - &environment_id, - api::EnvironmentLifecycleStatusView::Closed, - ) - .await?; - let closed_rejected = api - .put_environment_power(api::EnvironmentPowerPutParams { - environment_id: environment_id.clone(), - power: api::EnvironmentPowerStateView::Running, - }) - .await - .expect_err("closed environments cannot change power"); - assert_eq!(closed_rejected.kind, api::AgentApiErrorKind::InvalidRequest); - api.close_environment(api::EnvironmentCloseParams { - environment_id: external.environment_id.clone(), - }) - .await?; - wait_for_environment_status( - &api, - &external.environment_id, - api::EnvironmentLifecycleStatusView::Closed, - ) - .await?; - - let _ = store - .delete_provider_binding( - store.config().universe_id, - &EnvironmentProviderBindingId::new(binding_id.clone()), - ) - .await; - let _ = store - .delete_provider(&EnvironmentProviderId::new(provider_id.clone())) - .await; - Ok(()) -} diff --git a/crates/temporal-runtime/tests/environment_registration_live.rs b/crates/temporal-runtime/tests/environment_registration_live.rs index 8a45ff53c..d3dd9a1ac 100644 --- a/crates/temporal-runtime/tests/environment_registration_live.rs +++ b/crates/temporal-runtime/tests/environment_registration_live.rs @@ -1,10 +1,20 @@ //! Key-based outbound registration, end to end against the real gateway: //! an in-process `lightspeed-envd` dials the environment gateway with a -//! registration key, becomes a `Ready` registered environment, serves a -//! worker route through a reverse-dialed data socket, goes `Offline` when -//! its control connection drops, reconnects by identity alone, keeps -//! reconnecting after its key is revoked, and is refused for good once its -//! environment is closed. +//! registration key and becomes an open, connected registered environment +//! that serves worker routes through reverse-dialed stream sockets. Its +//! connection is written on transitions only: a dropped control connection +//! marks the row disconnected, and the worker route then refuses with a +//! typed refusal (`away` at once for an ephemeral key, a bounded +//! `disconnected` wait for a persistent one). The daemon reconnects by +//! identity alone, loses to a newer process with the same identity, keeps +//! reconnecting after its key is revoked, closes its ephemeral environment +//! at once when it says goodbye (`departed`), and is told the environment +//! closed by an in-process close at once and by a close made elsewhere at +//! the next revocation sweep. The registered sweep closes ephemeral rows +//! after their grace and persistent rows after the key's offline bound; a +//! parked daemon waits for its stop signal instead of exiting. Rotation +//! keeps the key's group, startup repair marks connected rows disconnected, +//! and a configured registration URL is an accepted signature origin. mod support; @@ -12,35 +22,51 @@ use std::{collections::BTreeMap, path::Path, sync::Arc, time::Duration}; use api::{ AgentApiService, DeploymentApiService, DeploymentUniverseCreateParams, EnvironmentCloseParams, - EnvironmentIdentityModeView, EnvironmentLifecycleStatusView, EnvironmentListParams, + EnvironmentCloseReasonView, EnvironmentIdentityModeView, EnvironmentListParams, EnvironmentReadParams, EnvironmentRegistrationKeyCreateParams, EnvironmentRegistrationKeyReadParams, EnvironmentRegistrationKeyRevokeParams, - EnvironmentSourceView, EnvironmentView, + EnvironmentRegistrationKeyRotateParams, EnvironmentRegistrationKeyUpdateParams, + EnvironmentSourceView, EnvironmentStateView, EnvironmentStatusView, EnvironmentView, }; -use environment_client::{EnvironmentDataClient, JsonRpcTransport}; +use environment_client::{ClientError, DataClient}; use environment_daemon::{ DaemonRuntime, - config::{DaemonConfig, RegistrationConfig}, + config::{DaemonConfig, OnClosed, RegistrationConfig}, + server::DaemonExit, }; use environment_protocol::{ + ENVD_PROTOCOL_VERSION, + bytes::{ByteChunk, SecretString}, data::{ + exec::{ + CommandSpec, ExecInput, ExecState, ExitStatus, OutputChunk, PtySize, ReturnOn, + StoppedBy, + }, fs::{ReadDirectoryParams, WriteFileParams}, - handshake::InitializeParams, - idle::IdleParams, + idle::GetIdleParams, process::{ - ProcessOutputChunk, ProcessSignal, ReadProcessParams, StartProcessParams, + ProcessSignal, ReadProcessParams, ReadProcessResponse, StartProcessParams, TerminateProcessParams, WriteProcessParams, }, }, - shared::{ByteChunk, CURRENT_PROTOCOL_VERSION, EnvironmentPath, ProcessId, SecretString}, + error::ProtocolErrorKind, + ids::ProcessId, + metadata::{DisplayName, Metadata}, + paths::EnvironmentPath, + routes::{DataRouteRefusal, canonical_origin}, +}; +use environments::{ + BeginCloseEnvironment, EnvironmentCloseReason, EnvironmentGoneReason, EnvironmentStore as _, + EnvironmentUse, EnvironmentWaitReason, REGISTERED_LIVENESS_STAMP_MS, + REGISTERED_RECONNECT_WAIT_MS, StampRegisteredSeen, }; -use environments::EnvironmentStore as _; use support::live::{LIVE_TEST_LOCK, require_storage_live_env}; use temporal_runtime::{ DeploymentStores, GatewayAuthMode, UniverseRuntime, + environments::gateway::{EnvironmentGatewayClientConfig, RouteKey}, gateway::{ DEFAULT_MAX_REQUEST_BODY_BYTES, GatewayAgentApi, GatewayDeploymentApi, GatewayRoutes, - GatewayState, gateway_router, + GatewayState, gateway_router, registration, request_context::{RequestContext, with_request_context as calling}, }, }; @@ -56,6 +82,8 @@ use uuid::Uuid; const POLL: Duration = Duration::from_millis(200); const SETTLE: Duration = Duration::from_secs(30); +/// An in-process close reaches the daemon without any polling interval. +const IN_PROCESS_NOTIFY: Duration = Duration::from_secs(3); #[tokio::test(flavor = "current_thread")] #[ignore = "requires ./dev.sh infra or compatible Temporal + Postgres env"] @@ -75,10 +103,14 @@ async fn registered_envd_dials_out_serves_routes_reconnects_and_is_spent_on_clos let universe_id = Uuid::new_v4(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let base_url = format!("http://{}", listener.local_addr()?); - let connect_url = format!( - "ws://{}/environment-gateway/connect", - listener.local_addr()? + let address = listener.local_addr()?; + let base_url = format!("http://{address}"); + let connect_url = format!("ws://{address}/environment-gateway/connect"); + // A second name for the same listener: accepted only because it is a + // configured registration URL. + let alias_connect_url = format!( + "ws://localhost:{}/environment-gateway/connect", + address.port() ); let runtime = Arc::new(UniverseRuntime::new( client, @@ -97,11 +129,26 @@ async fn registered_envd_dials_out_serves_routes_reconnects_and_is_spent_on_clos ) .await?; let caller = RequestContext::local(api::AccessScope::Universe { universe_id }); - let state = Arc::new(GatewayState::multi( - GatewayAuthMode::Single { universe_id }, - runtime.clone(), - base_url.clone(), - )); + let origins = temporal_runtime::gateway::http::registration_origins( + Some(&base_url), + std::slice::from_ref(&alias_connect_url), + address, + )?; + assert_eq!( + origins, + [ + canonical_origin(&base_url)?, + canonical_origin(&alias_connect_url)? + ] + ); + let state = Arc::new( + GatewayState::multi( + GatewayAuthMode::Single { universe_id }, + runtime.clone(), + base_url.clone(), + ) + .with_registration_origins(origins), + ); let gateway = tokio::spawn(async move { axum::serve( listener, @@ -162,41 +209,67 @@ async fn registered_envd_dials_out_serves_routes_reconnects_and_is_spent_on_clos "the environment-gateway process serves the connect route" ); api_only.abort(); - let reconciler = tokio::spawn(runtime.clone().run_environment_reconciler()); let api = runtime.state_for(universe_id, false).await?.api.clone(); let sandbox = tempfile::tempdir()?; + let urls = Urls { + base: &base_url, + connect: &connect_url, + alias_connect: &alias_connect_url, + }; let result = calling( caller, - scenario( - &runtime, - &api, - universe_id, - &base_url, - &connect_url, - sandbox.path(), - ), + scenario(&runtime, &api, universe_id, &urls, sandbox.path()), ) .await; - reconciler.abort(); gateway.abort(); result } +struct Urls<'a> { + base: &'a str, + connect: &'a str, + alias_connect: &'a str, +} + async fn scenario( runtime: &Arc, api: &Arc, universe_id: Uuid, - base_url: &str, - connect_url: &str, + urls: &Urls<'_>, + sandbox: &Path, +) -> anyhow::Result<()> { + // Workers reach this test's gateway, not whatever the sourced dev + // environment points LIGHTSPEED_ENVIRONMENT_GATEWAY_URL at; the route + // bearer is the deployment token the gateway state checks. + let gateway = EnvironmentGatewayClientConfig::new( + urls.base, + runtime.environment_gateway().deployment_token(), + ); + ephemeral_scenario(runtime, api, universe_id, &gateway, urls, sandbox).await?; + persistent_scenario(runtime, api, universe_id, &gateway, urls, sandbox).await?; + ephemeral_grace_scenario(runtime, api, universe_id, urls, sandbox).await +} + +/// An ephemeral key: admission, the data path, the coarse stamp, immediate +/// `away` while disconnected, reconnect, supersede, revocation, departure +/// and the in-process close notify. +async fn ephemeral_scenario( + runtime: &Arc, + api: &Arc, + universe_id: Uuid, + gateway: &EnvironmentGatewayClientConfig, + urls: &Urls<'_>, sandbox: &Path, ) -> anyhow::Result<()> { + let store = runtime.state_for(universe_id, false).await?.store.clone(); let minted = api .create_environment_registration_key(EnvironmentRegistrationKeyCreateParams { display_name: "registration live pool".to_owned(), identity_mode: EnvironmentIdentityModeView::Ephemeral, max_active_environments: Some(2), ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, expires_at_ms: None, }) .await? @@ -206,187 +279,246 @@ async fn scenario( assert!(secret.starts_with("lsrk_")); assert_eq!(minted.registration_key.active_environment_count, 0); - // First daemon: a fresh identity admitted by the key. - let root_a = sandbox.join("a"); - std::fs::create_dir_all(&root_a)?; - let receipt_a = root_a.join("receipt.json"); + // First daemon: a fresh identity admitted by the key, open and + // connected, with both connection stamps written at admission. + let root_a = workspace(sandbox, "a")?; + let receipt_a = sandbox.join("receipt-a.json"); let daemon_a = spawn_daemon(daemon_config( &root_a, - connect_url, + &sandbox.join("state-a"), + urls.connect, Some(&secret), Some(&receipt_a), - )?); - let environment_a = - wait_for_registered(api, &key_id, 1, EnvironmentLifecycleStatusView::Ready).await?; - let EnvironmentSourceView::Registered { - registration_key_id, - daemon_id, - identity_mode, - } = &environment_a.source - else { - anyhow::bail!( - "expected a registered source, got {:?}", - environment_a.source - ) - }; - assert_eq!(registration_key_id, &key_id); - assert_eq!(*identity_mode, EnvironmentIdentityModeView::Ephemeral); - assert!(daemon_id.starts_with("daemon_")); - assert!(environment_a.last_seen_at_ms.is_some()); + OnClosed::Exit, + )?)?; + let environment_a = wait_for_registered(api, &key_id, 1, is_connected).await?; + let registered = registered_source(&environment_a)?; + assert_eq!(registered.registration_key_id, key_id); assert_eq!( - environment_a - .metadata - .get("harbor.trialId") - .map(String::as_str), - Some("trial-1") + registered.identity_mode, + EnvironmentIdentityModeView::Ephemeral ); + assert!(registered.daemon_id.starts_with("daemon_")); assert_eq!( - environment_a - .metadata - .get("lightspeed.envd.version") - .map(String::as_str), - Some(release_info::VERSION) + registered.connection_changed_at_ms, + registered.last_seen_at_ms ); + assert_eq!(environment_a.state, EnvironmentStateView::Open); + assert_eq!(environment_a.status, EnvironmentStatusView::Ready); + assert_eq!(environment_a.request_id, None); assert_eq!( environment_a .metadata - .get("lightspeed.envd.gitSha") + .get("harbor.trialId") .map(String::as_str), - Some(release_info::GIT_SHA) + Some("trial-1") ); - assert_eq!( + assert!( environment_a .metadata - .get("lightspeed.envd.protocolVersion") - .map(String::as_str), - Some(CURRENT_PROTOCOL_VERSION.to_string().as_str()) + .keys() + .all(|key| !key.starts_with("lightspeed.")), + "build facts are not metadata: {:?}", + environment_a.metadata ); - let receipt: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&receipt_a)?)?; + let daemon = environment_a.daemon.as_ref().expect("daemon build"); + assert_eq!(daemon.version.as_deref(), Some(release_info::VERSION)); + assert_eq!(daemon.git_sha.as_deref(), Some(release_info::GIT_SHA)); + assert_eq!(daemon.protocol_version, ENVD_PROTOCOL_VERSION); + let receipt = read_receipt(&receipt_a)?; assert_eq!(receipt["environmentId"], environment_a.environment_id); - assert_eq!(receipt["daemonId"], daemon_id.as_str()); + assert_eq!(receipt["daemonId"], registered.daemon_id.as_str()); assert_eq!(receipt["identityMode"], "ephemeral"); + assert_eq!(receipt["admission"], "created"); + assert!(receipt.get("incarnationId").is_none()); + assert!( + receipt["controlConnectionId"] + .as_str() + .is_some_and(|id| id.starts_with("connection_")) + ); assert!(receipt.get("registrationKey").is_none()); - let key_view = api - .read_environment_registration_key(EnvironmentRegistrationKeyReadParams { - registration_key_id: key_id.clone(), - }) - .await? - .result - .registration_key; + let key_view = read_key(api, &key_id).await?; assert_eq!(key_view.active_environment_count, 1); assert_eq!(key_view.registered_environment_count, 1); - // The worker route reaches the daemon through a reverse-dialed data - // socket; two concurrent routes each get their own socket. - let record = runtime - .state_for(universe_id, false) - .await? - .store - .read_environment(&environments::EnvironmentId::new( - environment_a.environment_id.clone(), - )) - .await?; - // Workers reach this test's gateway, not whatever the sourced dev - // environment points LIGHTSPEED_ENVIRONMENT_GATEWAY_URL at; the route - // bearer is the deployment token the gateway state checks. - let gateway = temporal_runtime::environments::gateway::EnvironmentGatewayClientConfig::new( - base_url, - runtime.environment_gateway().deployment_token(), + // The liveness stamp is coarse: a stamp within the interval of the + // last one writes nothing, one a full interval later writes. + let environment_id_a = environments::EnvironmentId::new(environment_a.environment_id.clone()); + assert!( + !store + .stamp_registered_seen(StampRegisteredSeen { + environment_id: environment_id_a.clone(), + at_ms: registered.last_seen_at_ms + 1_000, + }) + .await? ); - let connection = gateway.connection_for(universe_id, &record); - let mut first = EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("registration-live"), - ) - .await?; - let mut second = EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("registration-live"), - ) - .await?; - let mut third = EnvironmentDataClient::connect( - &connection.endpoint, - gateway.connect_options("registration-live"), - ) - .await?; - let mut capabilities = None; - for client in [&mut first, &mut second, &mut third] { - let initialized = client - .initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, - client_name: "registration-live".to_owned(), - scope: environment_protocol::shared::EnvironmentScope::Default, - resume_connection_id: None, + assert!( + store + .stamp_registered_seen(StampRegisteredSeen { + environment_id: environment_id_a.clone(), + at_ms: registered.last_seen_at_ms + REGISTERED_LIVENESS_STAMP_MS, }) - .await?; - assert_eq!(initialized.protocol_version, CURRENT_PROTOCOL_VERSION); - capabilities = Some(initialized.capabilities); - } - let capabilities = capabilities.expect("negotiated capabilities"); + .await? + ); + + // The worker route reaches the daemon through a reverse-dialed stream + // socket; concurrent routes each get their own socket. + let key = RouteKey { + universe_id, + environment_id: environment_a.environment_id.clone(), + }; + assert_eq!( + gateway.route_key( + universe_id, + &store.read_environment(&environment_id_a).await? + ), + key + ); + let options = || gateway.connect_options("registration-live"); + let (first, initialized) = gateway.connect(&key, options()).await?; + let (second, _) = gateway.connect(&key, options()).await?; + let (third, _) = gateway.connect(&key, options()).await?; + assert_eq!(initialized.protocol_version, ENVD_PROTOCOL_VERSION); + assert_eq!( + initialized.default_cwd.as_str(), + root_a.to_str().expect("utf-8 root") + ); + let written = path(&root_a, "registered.txt")?; first .write_file(&WriteFileParams { - path: EnvironmentPath::new("registered.txt")?, + path: written.clone(), data: ByteChunk::new(b"through the reverse dial".to_vec()), + create: Default::default(), + if_match: None, + executable: false, }) .await?; let listing = second .read_directory(&ReadDirectoryParams { - path: EnvironmentPath::new(".")?, + path: initialized.default_cwd.clone(), + limit: 100, + cursor: None, }) .await?; assert!( listing .entries .iter() - .any(|entry| entry.file_name == "registered.txt"), + .any(|entry| entry.name.as_str() == "registered.txt"), "second data socket sees the file the first wrote: {listing:?}" ); assert_eq!( std::fs::read_to_string(root_a.join("registered.txt"))?, "through the reverse dial" ); - process_scenario(&mut first, &mut second, &root_a).await?; - executor_scenario(third, capabilities).await?; + process_scenario(&first, &second, &root_a).await?; + executor_scenario(third).await?; first.close().await?; second.close().await?; - // Dropping the control connection makes the environment Offline; the - // same identity, now without any key, reconnects the same environment. - daemon_a.abort(); - wait_for_status( - api, - &environment_a.environment_id, - EnvironmentLifecycleStatusView::Offline, - ) + // Dropping the control connection marks the environment disconnected. + // An ephemeral daemon is away at once: the gate refuses the route for + // good instead of waiting. + daemon_a.exit.abort(); + let away = wait_for_environment(api, &environment_a.environment_id, "disconnected", |view| { + is_connected(view) == Some(false) + }) .await?; - let daemon_a = spawn_daemon(daemon_config(&root_a, connect_url, None, Some(&receipt_a))?); - wait_for_status( - api, - &environment_a.environment_id, - EnvironmentLifecycleStatusView::Ready, - ) + assert_eq!(away.state, EnvironmentStateView::Open); + assert_eq!(away.status, EnvironmentStatusView::Away); + assert!( + registered_source(&away)?.connection_changed_at_ms >= registered.connection_changed_at_ms + ); + let refused = expect_route_refusal(gateway, &key).await?; + assert!(matches!( + refused, + ClientError::UpgradeRejected { + status: 410, + refusal: Some(DataRouteRefusal::Away {}), + retry_after_ms: None, + } + )); + + // The same identity, now without any key, reconnects the same + // environment. + let daemon_a = spawn_daemon(daemon_config( + &root_a, + &sandbox.join("state-a"), + urls.connect, + None, + Some(&receipt_a), + OnClosed::Exit, + )?)?; + wait_for_environment(api, &environment_a.environment_id, "reconnected", |view| { + is_connected(view) == Some(true) + }) .await?; - let receipt: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&receipt_a)?)?; - assert_eq!(receipt["environmentId"], environment_a.environment_id); + wait_for_receipt_admission(&receipt_a, "reconnected").await?; + assert_eq!( + read_receipt(&receipt_a)?["environmentId"], + environment_a.environment_id + ); assert_eq!( - wait_for_registered(api, &key_id, 1, EnvironmentLifecycleStatusView::Ready) + wait_for_registered(api, &key_id, 1, is_connected) .await? .environment_id, environment_a.environment_id, "reconnect must not create a second environment" ); + // A second process with the same identity (a cloned state directory) + // wins; the older one is told it was superseded and exits instead of + // fighting for the identity. The row stays connected throughout. + let twin_state = sandbox.join("state-a-twin"); + std::fs::create_dir_all(&twin_state)?; + std::fs::copy( + sandbox + .join("state-a") + .join(environment_daemon::identity::DAEMON_KEY_FILE), + twin_state.join(environment_daemon::identity::DAEMON_KEY_FILE), + )?; + let twin_a = spawn_daemon(daemon_config( + &root_a, + &twin_state, + urls.connect, + None, + None, + OnClosed::Exit, + )?)?; + let superseded = tokio::time::timeout(SETTLE, daemon_a.exit).await??; + assert!( + matches!(superseded, DaemonExit::Superseded { .. }), + "unexpected exit: {superseded:?}" + ); + assert_eq!(superseded.status(), 5); + wait_for_environment( + api, + &environment_a.environment_id, + "connected twin", + |view| is_connected(view) == Some(true), + ) + .await?; + let (after_supersede, _) = gateway.connect(&key, options()).await?; + after_supersede.get_idle(&GetIdleParams {}).await?; + after_supersede.close().await?; + // A second identity with the same key is a second environment. - let root_b = sandbox.join("b"); - std::fs::create_dir_all(&root_b)?; - let daemon_b = spawn_daemon(daemon_config(&root_b, connect_url, Some(&secret), None)?); + let root_b = workspace(sandbox, "b")?; + let daemon_b = spawn_daemon(daemon_config( + &root_b, + &sandbox.join("state-b"), + urls.connect, + Some(&secret), + None, + OnClosed::Exit, + )?)?; let environments = wait_for_registered_count(api, &key_id, 2).await?; let environment_b = environments .iter() .find(|environment| environment.environment_id != environment_a.environment_id) .cloned() .expect("second environment"); - assert_eq!(environment_b.status, EnvironmentLifecycleStatusView::Ready); + assert_eq!(is_connected(&environment_b), Some(true)); // Revocation without cascade stops new identities but not reconnects. api.revoke_environment_registration_key(EnvironmentRegistrationKeyRevokeParams { @@ -394,111 +526,443 @@ async fn scenario( close_environments: false, }) .await?; - let root_c = sandbox.join("c"); - std::fs::create_dir_all(&root_c)?; - let daemon_c = spawn_daemon(daemon_config(&root_c, connect_url, Some(&secret), None)?); - let refused = tokio::time::timeout(SETTLE, daemon_c).await??; - let refused = refused.expect_err("a revoked key is a terminal refusal for a new daemon"); + let root_c = workspace(sandbox, "c")?; + let daemon_c = spawn_daemon(daemon_config( + &root_c, + &sandbox.join("state-c"), + urls.connect, + Some(&secret), + None, + OnClosed::Exit, + )?)?; + let refused = tokio::time::timeout(SETTLE, daemon_c.exit).await??; assert!( - refused.to_string().contains("RegistrationKeyRevoked"), - "unexpected refusal: {refused:#}" + matches!(refused, DaemonExit::Refused(_)), + "a revoked key is a terminal refusal for a new daemon: {refused:?}" ); - daemon_b.abort(); - wait_for_status( - api, - &environment_b.environment_id, - EnvironmentLifecycleStatusView::Offline, - ) - .await?; - let daemon_b = spawn_daemon(daemon_config(&root_b, connect_url, None, None)?); - wait_for_status( + assert_eq!(refused.status(), 3); + + // A deliberate stop says goodbye and the daemon exits cleanly. The + // sandbox is finished, so its ephemeral environment closes at once + // (`departed`) instead of waiting out the key's disconnect grace. + daemon_b.runtime.shutdown().cancel(); + let stopped = tokio::time::timeout(SETTLE, daemon_b.exit).await??; + assert!( + matches!(stopped, DaemonExit::Shutdown), + "unexpected exit: {stopped:?}" + ); + assert_eq!(stopped.status(), 0); + let departed = wait_for_state( api, &environment_b.environment_id, - EnvironmentLifecycleStatusView::Ready, + EnvironmentStateView::Closed, ) .await?; + assert_eq!( + departed.close_reason, + Some(EnvironmentCloseReasonView::Departed) + ); + assert!(departed.closed_at_ms.is_some()); + // The departed identity is spent: a restart is refused at register. + let restarted_b = spawn_daemon(daemon_config( + &root_b, + &sandbox.join("state-b"), + urls.connect, + None, + None, + OnClosed::Exit, + )?)?; + let refused_b = tokio::time::timeout(SETTLE, restarted_b.exit).await??; + assert!( + matches!(refused_b, DaemonExit::EnvironmentClosed), + "a departed ephemeral identity cannot reconnect: {refused_b:?}" + ); + assert_eq!(refused_b.status(), 4); - // Closing the environment spends the identity: the daemon is told to go - // and its reconnect is refused for good. + // Closing the environment in this process notifies its control + // connection at once: the daemon is told the environment closed and + // exits for good, and its routes end. api.close_environment(EnvironmentCloseParams { environment_id: environment_a.environment_id.clone(), }) .await?; - let spent = tokio::time::timeout(SETTLE, daemon_a).await??; - let spent = spent.expect_err("a closed environment is a terminal refusal"); + let spent = tokio::time::timeout(IN_PROCESS_NOTIFY, twin_a.exit).await??; assert!( - spent.to_string().contains("EnvironmentClosed"), - "unexpected refusal: {spent:#}" + matches!(spent, DaemonExit::EnvironmentClosed), + "a closed environment ends its daemon: {spent:?}" ); - wait_for_status( + assert_eq!(spent.status(), 4); + let closed = wait_for_state( api, &environment_a.environment_id, - EnvironmentLifecycleStatusView::Closed, + EnvironmentStateView::Closed, ) .await?; - let key_view = api - .read_environment_registration_key(EnvironmentRegistrationKeyReadParams { - registration_key_id: key_id.clone(), - }) - .await? - .result - .registration_key; + assert_eq!( + closed.close_reason, + Some(EnvironmentCloseReasonView::Requested) + ); + let refused = expect_route_refusal(gateway, &key).await?; + assert!(matches!( + refused, + ClientError::UpgradeRejected { + status: 410, + refusal: Some(DataRouteRefusal::Gone {}), + .. + } + )); + assert!( + list_by_key(api, &key_id, false).await?.is_empty(), + "closed environments are hidden by default" + ); + assert_eq!(list_by_key(api, &key_id, true).await?.len(), 2); + let key_view = read_key(api, &key_id).await?; assert_eq!(key_view.registered_environment_count, 2); - assert_eq!(key_view.active_environment_count, 1); + assert_eq!(key_view.active_environment_count, 0); assert_eq!( key_view.status, api::EnvironmentRegistrationKeyStatusView::Revoked ); + Ok(()) +} - daemon_b.abort(); - api.close_environment(EnvironmentCloseParams { - environment_id: environment_b.environment_id.clone(), +/// A persistent key, dialed under the configured registration URL: the +/// bounded reconnect wait, startup repair, rotation, the cross-process +/// close at the revocation sweep, and the offline bound with a parked +/// daemon. +async fn persistent_scenario( + runtime: &Arc, + api: &Arc, + universe_id: Uuid, + gateway: &EnvironmentGatewayClientConfig, + urls: &Urls<'_>, + sandbox: &Path, +) -> anyhow::Result<()> { + let store = runtime.state_for(universe_id, false).await?.store.clone(); + let minted = api + .create_environment_registration_key(EnvironmentRegistrationKeyCreateParams { + display_name: "registration live fleet".to_owned(), + identity_mode: EnvironmentIdentityModeView::Persistent, + max_active_environments: Some(3), + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, + expires_at_ms: None, + }) + .await? + .result; + let key_id = minted.registration_key.registration_key_id.clone(); + let first_secret = minted.secret.0.clone(); + + // The daemon signs the alias origin; the gateway accepts it because + // the alias is a configured registration URL. + let root_p = workspace(sandbox, "p")?; + let state_p = sandbox.join("state-p"); + let receipt_p = sandbox.join("receipt-p.json"); + let spawn_p = |key: Option<&str>| { + spawn_daemon(daemon_config( + &root_p, + &state_p, + urls.alias_connect, + key, + Some(&receipt_p), + OnClosed::Exit, + )?) + }; + let daemon_p = spawn_p(Some(&first_secret))?; + let environment_p = wait_for_registered(api, &key_id, 1, is_connected).await?; + assert_eq!(read_receipt(&receipt_p)?["identityMode"], "persistent"); + let key = RouteKey { + universe_id, + environment_id: environment_p.environment_id.clone(), + }; + let environment_id_p = environments::EnvironmentId::new(environment_p.environment_id.clone()); + let (client, _) = gateway + .connect(&key, gateway.connect_options("registration-live")) + .await?; + client.get_idle(&GetIdleParams {}).await?; + client.close().await?; + + // A persistent daemon that drops is waited for, within a bound: the + // route refuses `disconnected` with a retry hint, and the gate turns + // into `away` once the bound passes. + daemon_p.exit.abort(); + let away = wait_for_environment(api, &environment_p.environment_id, "disconnected", |view| { + is_connected(view) == Some(false) + }) + .await?; + assert_eq!(away.status, EnvironmentStatusView::Away); + let refused = expect_route_refusal(gateway, &key).await?; + assert!(matches!( + refused, + ClientError::UpgradeRejected { + status: 503, + refusal: Some(DataRouteRefusal::Disconnected {}), + retry_after_ms: Some(_), + } + )); + let record = store.read_environment(&environment_id_p).await?; + let since = record + .registered() + .expect("registered") + .connection_changed_at_ms; + assert_eq!( + record.use_gate(since + 1_000), + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Disconnected, + retry_after_ms: environments::WAIT_RETRY_AFTER_MS, + until_ms: Some(since + REGISTERED_RECONNECT_WAIT_MS), + } + ); + assert!(matches!( + record.use_gate(since + REGISTERED_RECONNECT_WAIT_MS), + EnvironmentUse::Gone { + reason: EnvironmentGoneReason::Away, + .. + } + )); + + // Back by identity alone. + let daemon_p = spawn_p(None)?; + wait_for_environment(api, &environment_p.environment_id, "reconnected", |view| { + is_connected(view) == Some(true) }) .await?; - wait_for_status( + wait_for_receipt_admission(&receipt_p, "reconnected").await?; + + // Startup repair (here for this universe only): a gateway that + // restarted held none of the connections its rows still call + // connected. The disconnect counts from now, so persistent rows get + // their full reconnect wait. + let before_repair = now_ms(); + assert!(store.repair_registered_connections(before_repair).await? >= 1); + let repaired = store.read_environment(&environment_id_p).await?; + let source = repaired.registered().expect("registered"); + assert!(!source.connected); + assert_eq!(source.connection_changed_at_ms, before_repair); + assert!(matches!( + repaired.use_gate(before_repair + 1_000), + EnvironmentUse::Wait { + reason: EnvironmentWaitReason::Disconnected, + .. + } + )); + // The daemon's next connection records it again. + daemon_p.exit.abort(); + let daemon_p = spawn_p(None)?; + wait_for_environment( api, - &environment_b.environment_id, - EnvironmentLifecycleStatusView::Closed, + &environment_p.environment_id, + "reconnected after repair", + |view| is_connected(view) == Some(true), + ) + .await?; + + // Rotation retires the live secret at once and keeps the key: its + // group, identity mode and environments. Admitted daemons are + // unaffected; a new daemon needs the new secret. + let rotated = api + .rotate_environment_registration_key(EnvironmentRegistrationKeyRotateParams { + registration_key_id: key_id.clone(), + }) + .await? + .result; + assert_eq!(rotated.registration_key.registration_key_id, key_id); + assert_eq!( + rotated.registration_key.identity_mode, + EnvironmentIdentityModeView::Persistent + ); + assert_ne!( + rotated.registration_key.key_prefix, + minted.registration_key.key_prefix + ); + assert_ne!(rotated.secret.0, first_secret); + let root_r = workspace(sandbox, "r")?; + let stale = spawn_daemon(daemon_config( + &root_r, + &sandbox.join("state-r"), + urls.connect, + Some(&first_secret), + None, + OnClosed::Exit, + )?)?; + let stale_exit = tokio::time::timeout(SETTLE, stale.exit).await??; + assert!( + matches!(stale_exit, DaemonExit::Refused(_)), + "a retired secret admits nobody: {stale_exit:?}" + ); + let root_s = workspace(sandbox, "s")?; + let state_s = sandbox.join("state-s"); + let daemon_s = spawn_daemon(daemon_config( + &root_s, + &state_s, + urls.connect, + Some(&rotated.secret.0), + None, + OnClosed::Exit, + )?)?; + let environments = wait_for_registered_count(api, &key_id, 2).await?; + let environment_s = environments + .iter() + .find(|environment| environment.environment_id != environment_p.environment_id) + .cloned() + .expect("environment admitted by the rotated secret"); + assert_eq!( + registered_source(&environment_s)?.registration_key_id, + key_id + ); + assert_eq!( + read_key(api, &key_id).await?.registered_environment_count, + 2 + ); + + // A close made outside this process (here written to the store + // directly) reaches the daemon at the next revocation sweep. + store + .begin_close_environment(BeginCloseEnvironment { + environment_id: environment_id_p.clone(), + reason: EnvironmentCloseReason::Requested, + at_ms: now_ms(), + }) + .await?; + assert!(registration::revoke_ended_routes(runtime).await? >= 1); + let spent = tokio::time::timeout(SETTLE, daemon_p.exit).await??; + assert!( + matches!(spent, DaemonExit::EnvironmentClosed), + "the revocation sweep ends the daemon of an environment closed elsewhere: {spent:?}" + ); + + // The key's offline bound closes a persistent environment whose daemon + // stays away (`offline_timeout`). The daemon, configured to park, + // then waits for its stop signal instead of exiting or crash-looping. + let updated = api + .update_environment_registration_key(EnvironmentRegistrationKeyUpdateParams { + registration_key_id: key_id.clone(), + display_name: "registration live fleet (offline bound)".to_owned(), + max_active_environments: Some(3), + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: Some(1_000), + expires_at_ms: None, + }) + .await? + .result; + assert_eq!(updated.registration_key.offline_close_after_ms, Some(1_000)); + assert_eq!( + updated.registration_key.display_name, + "registration live fleet (offline bound)" + ); + daemon_s.exit.abort(); + wait_for_environment(api, &environment_s.environment_id, "disconnected", |view| { + is_connected(view) == Some(false) + }) + .await?; + tokio::time::sleep(Duration::from_millis(1_500)).await; + let sweep = registration::sweep_registered_universe(runtime, universe_id, now_ms()).await?; + assert!( + sweep + .closed + .iter() + .any(|(id, reason)| id.as_str() == environment_s.environment_id + && *reason == EnvironmentCloseReason::OfflineTimeout), + "{sweep:?}" + ); + let closed = wait_for_state( + api, + &environment_s.environment_id, + EnvironmentStateView::Closed, ) .await?; + assert_eq!( + closed.close_reason, + Some(EnvironmentCloseReasonView::OfflineTimeout) + ); + let parked = spawn_daemon(daemon_config( + &root_s, + &state_s, + urls.connect, + None, + None, + OnClosed::Park, + )?)?; + tokio::time::sleep(Duration::from_secs(2)).await; + assert!( + !parked.exit.is_finished(), + "a parked daemon waits for its stop signal" + ); + parked.runtime.shutdown().cancel(); + let stopped = tokio::time::timeout(SETTLE, parked.exit).await??; + assert!( + matches!(stopped, DaemonExit::Shutdown), + "a parked daemon exits cleanly when stopped: {stopped:?}" + ); + assert_eq!(stopped.status(), 0); + Ok(()) +} - // Ephemeral grace: a key with a short grace closes an environment whose - // daemon stays away, through the lifecycle reconciler alone. +/// Ephemeral grace: the registered sweep closes an environment whose +/// ephemeral daemon stays away longer than its key's grace. +async fn ephemeral_grace_scenario( + runtime: &Arc, + api: &Arc, + universe_id: Uuid, + urls: &Urls<'_>, + sandbox: &Path, +) -> anyhow::Result<()> { let short = api .create_environment_registration_key(EnvironmentRegistrationKeyCreateParams { display_name: "registration live short grace".to_owned(), identity_mode: EnvironmentIdentityModeView::Ephemeral, max_active_environments: None, ephemeral_disconnect_grace_ms: Some(1_500), + offline_close_after_ms: None, expires_at_ms: None, }) .await? .result; let short_key_id = short.registration_key.registration_key_id.clone(); - let root_d = sandbox.join("d"); - std::fs::create_dir_all(&root_d)?; + let root_d = workspace(sandbox, "d")?; let daemon_d = spawn_daemon(daemon_config( &root_d, - connect_url, + &sandbox.join("state-d"), + urls.connect, Some(&short.secret.0), None, - )?); - let environment_d = - wait_for_registered(api, &short_key_id, 1, EnvironmentLifecycleStatusView::Ready).await?; - daemon_d.abort(); - wait_for_status( + OnClosed::Exit, + )?)?; + let environment_d = wait_for_registered(api, &short_key_id, 1, is_connected).await?; + daemon_d.exit.abort(); + wait_for_environment(api, &environment_d.environment_id, "disconnected", |view| { + is_connected(view) == Some(false) + }) + .await?; + let early = registration::sweep_registered_universe(runtime, universe_id, now_ms()).await?; + assert!( + early + .closed + .iter() + .all(|(id, _)| id.as_str() != environment_d.environment_id), + "the grace has not passed yet: {early:?}" + ); + tokio::time::sleep(Duration::from_millis(2_000)).await; + let sweep = registration::sweep_registered_universe(runtime, universe_id, now_ms()).await?; + assert!( + sweep + .closed + .iter() + .any(|(id, reason)| id.as_str() == environment_d.environment_id + && *reason == EnvironmentCloseReason::EphemeralGrace), + "{sweep:?}" + ); + let closed = wait_for_state( api, &environment_d.environment_id, - EnvironmentLifecycleStatusView::Closed, + EnvironmentStateView::Closed, ) .await?; - let short_view = api - .read_environment_registration_key(EnvironmentRegistrationKeyReadParams { - registration_key_id: short_key_id, - }) - .await? - .result - .registration_key; + assert_eq!( + closed.close_reason, + Some(EnvironmentCloseReasonView::EphemeralGrace) + ); + let short_view = read_key(api, &short_key_id).await?; assert_eq!(short_view.active_environment_count, 0); assert_eq!(short_view.registered_environment_count, 1); Ok(()) @@ -508,88 +972,115 @@ async fn scenario( /// at once: a normal exit leaves its background child running, reported /// and counted by the idle report and stoppable through the handle; the /// read cursor is the daemon's, so the second socket continues where the -/// first stopped and an empty write is a wait; a PTY takes input; and an -/// interrupt reaches the group. -async fn process_scenario( - first: &mut EnvironmentDataClient, - second: &mut EnvironmentDataClient, +/// first stopped while positioned reads consume nothing; a PTY takes input; +/// an interrupt reaches the group; and terminate escalates gracefully. +async fn process_scenario( + first: &DataClient, + second: &DataClient, root: &Path, -) -> anyhow::Result<()> -where - T: JsonRpcTransport + Send + 'static, -{ +) -> anyhow::Result<()> { // A service the command left behind survives its exit. first .start_process(&shell_process( "live-leftover", "nohup sleep 60 >/dev/null 2>&1 & echo $! > leftover.pid; echo started; exit 0", false, - )) + )?) .await?; let leftover_pid = wait_for_pid_file(&root.join("leftover.pid")).await?; - tokio::time::sleep(Duration::from_millis(100)).await; let read = first - .read_process(&read_params("live-leftover", Some(10_000))) + .read_process(&read_until_exit("live-leftover")?) .await?; - assert!(read.exited); - assert_eq!(read.exit_code, Some(0)); - assert!(read.pid.is_some(), "the response names the root pid"); - assert_eq!(chunk_text(&read.chunks), "started\n"); + assert_eq!(read.state, ExecState::Exited); + assert_eq!(read.exit, Some(ExitStatus::Code { code: 0 })); + assert!(read.pid > 0, "the response names the root pid"); + assert_eq!(page_text(&read.output.chunks), "started\n"); assert!( - read.leftover_processes + read.leftovers.as_ref().is_some_and(|leftovers| leftovers + .processes .iter() - .any(|member| i32::try_from(member.pid) == Ok(leftover_pid)), + .any(|member| i32::try_from(member.pid) == Ok(leftover_pid))), "the leftover is reported with its pid: {:?}", - read.leftover_processes + read.leftovers ); assert!( process_alive(leftover_pid), "the leftover survives the call that started it" ); - let idle = second.idle(&IdleParams {}).await?; - assert_eq!(idle.running_processes, 0); - assert_eq!(idle.leftover_process_groups, 1); - assert!(idle.is_quiescent(), "a leftover is not running work"); + let idle = second.get_idle(&GetIdleParams {}).await?; + assert_eq!(idle.running_process_count, 0); + assert_eq!(idle.leftover_process_group_count, 1); second .terminate_process(&TerminateProcessParams { - process_id: ProcessId::new("live-leftover"), + process_id: process_id("live-leftover")?, signal: ProcessSignal::Kill, }) .await?; wait_for_process_gone(leftover_pid).await?; - assert_eq!(first.idle(&IdleParams {}).await?.leftover_process_groups, 0); + assert_eq!( + first + .get_idle(&GetIdleParams {}) + .await? + .leftover_process_group_count, + 0 + ); // The daemon owns the cursor: the second socket reads only what the - // first did not, and an empty write after exit is a wait. + // first did not, a positioned read consumes nothing, and writing to a + // process without input is refused. first .start_process(&shell_process( "live-cursor", "echo one; sleep 0.4; echo two; sleep 0.4; echo three", false, - )) + )?) .await?; let head = first - .read_process(&read_params("live-cursor", Some(150))) + .read_process(&ReadProcessParams { + return_on: ReturnOn::AnyOutput, + wait_ms: 10_000, + ..read_until_exit("live-cursor")? + }) .await?; - assert!(!head.exited); - assert_eq!(chunk_text(&head.chunks), "one\n"); + assert_eq!(head.state, ExecState::Running); + assert_eq!(page_text(&head.output.chunks), "one\n"); let rest = second - .read_process(&read_params("live-cursor", Some(10_000))) + .read_process(&read_until_exit("live-cursor")?) .await?; - assert!(rest.exited); - assert_eq!(chunk_text(&rest.chunks), "two\nthree\n"); - let last_head_seq = head.chunks.last().expect("head chunk").seq; - assert!(rest.chunks.iter().all(|chunk| chunk.seq > last_head_seq)); - let wait = first - .write_process(&WriteProcessParams { - process_id: ProcessId::new("live-cursor"), - chunk: None, - close_stdin: false, + assert_eq!(rest.state, ExecState::Exited); + assert_eq!(page_text(&rest.output.chunks), "two\nthree\n"); + assert!( + rest.output + .chunks + .iter() + .all(|chunk| chunk.offset >= head.output.next) + ); + let whole = first + .read_process(&ReadProcessParams { + after: Some(0), + ..read_until_exit("live-cursor")? }) .await?; + assert_eq!(page_text(&whole.output.chunks), "one\ntwo\nthree\n"); + assert_eq!(whole.output.next, whole.output.end); + let tail = second + .read_process(&ReadProcessParams { + tail_bytes: Some(6), + ..read_until_exit("live-cursor")? + }) + .await?; + assert_eq!(page_text(&tail.output.chunks), "three\n"); + let refused = first + .write_process(&WriteProcessParams { + process_id: process_id("live-cursor")?, + data: ByteChunk::new(b"late".to_vec()), + close_input: false, + }) + .await + .expect_err("an exited process takes no input"); assert_eq!( - wait.status, - environment_protocol::data::process::WriteProcessStatus::Accepted + refused.protocol_kind(), + Some(ProtocolErrorKind::FailedPrecondition) ); // A PTY takes input written from another socket. @@ -598,27 +1089,22 @@ where "live-pty", "read line; echo got:$line", true, - )) + )?) .await?; let write = second .write_process(&WriteProcessParams { - process_id: ProcessId::new("live-pty"), - chunk: Some(ByteChunk::new(b"hello\n".to_vec())), - close_stdin: false, + process_id: process_id("live-pty")?, + data: ByteChunk::new(b"hello\n".to_vec()), + close_input: false, }) .await?; - assert_eq!( - write.status, - environment_protocol::data::process::WriteProcessStatus::Accepted - ); - let pty = first - .read_process(&read_params("live-pty", Some(10_000))) - .await?; - assert!(pty.exited); + assert_eq!(write.accepted_bytes, 6); + let pty = first.read_process(&read_until_exit("live-pty")?).await?; + assert_eq!(pty.state, ExecState::Exited); assert!( - chunk_text(&pty.chunks).contains("got:hello"), + page_text(&pty.output.chunks).contains("got:hello"), "{:?}", - chunk_text(&pty.chunks) + page_text(&pty.output.chunks) ); // An interrupt reaches the group; the next read observes the trap. @@ -627,39 +1113,63 @@ where "live-interrupt", "trap 'echo caught; exit 3' INT; sleep 100 & wait $!", false, - )) + )?) .await?; tokio::time::sleep(Duration::from_millis(300)).await; let signalled = first .terminate_process(&TerminateProcessParams { - process_id: ProcessId::new("live-interrupt"), + process_id: process_id("live-interrupt")?, signal: ProcessSignal::Interrupt, }) .await?; assert!(signalled.running); let caught = second - .read_process(&read_params("live-interrupt", Some(10_000))) + .read_process(&read_until_exit("live-interrupt")?) + .await?; + assert_exited_with(&caught, 3, None, "caught\n"); + + // Terminate is graceful first: the process may clean up on SIGTERM. + first + .start_process(&shell_process( + "live-terminate", + "trap 'echo stopping; exit 7' TERM; sleep 100 & wait $!", + false, + )?) + .await?; + tokio::time::sleep(Duration::from_millis(300)).await; + second + .terminate_process(&TerminateProcessParams { + process_id: process_id("live-terminate")?, + signal: ProcessSignal::Terminate, + }) .await?; - assert!(caught.exited); - assert_eq!(caught.exit_code, Some(3)); - assert_eq!(chunk_text(&caught.chunks), "caught\n"); + let terminated = first + .read_process(&read_until_exit("live-terminate")?) + .await?; + assert_exited_with(&terminated, 7, Some(StoppedBy::Terminate), "stopping\n"); Ok(()) } +fn assert_exited_with( + read: &ReadProcessResponse, + code: i32, + stopped_by: Option, + output: &str, +) { + assert_eq!(read.state, ExecState::Exited); + assert_eq!(read.exit, Some(ExitStatus::Code { code })); + assert_eq!(read.stopped_by, stopped_by); + assert_eq!(page_text(&read.output.chunks), output); +} + /// The substrate executor over the real transport: a yielded run returns /// a handle and pid, a continue reads only new output, and a kill through /// the handle ends the group. -async fn executor_scenario( - client: EnvironmentDataClient, - capabilities: environment_protocol::shared::EnvironmentCapabilities, -) -> anyhow::Result<()> -where - T: JsonRpcTransport + Send + 'static, -{ - let connection = RemoteEnvironmentConnection::new(client, capabilities); +async fn executor_scenario(client: DataClient) -> anyhow::Result<()> { + let connection = RemoteEnvironmentConnection::new(client); let executor = connection .process_executor() - .expect("the daemon negotiates process execution"); + .expect("the daemon serves the process family"); let mut request = ProcessRequest::argv(["/bin/sh", "-c", "echo hi; sleep 60"]); request.yield_ms = Some(300); request.max_output_bytes = Some(4096); @@ -679,7 +1189,7 @@ where more.stdout.text_lossy() ); - let root_pid = running.pid.expect("pid") as i32; + let root_pid = i32::try_from(running.pid.expect("pid"))?; let killed = executor .continue_process(ContinueProcessRequest { handle, @@ -697,32 +1207,50 @@ where Ok(()) } -fn shell_process(process_id: &str, script: &str, tty: bool) -> StartProcessParams { - StartProcessParams { - process_id: ProcessId::new(process_id), - argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], - cwd: None, - env: BTreeMap::new(), - secret_env: BTreeMap::new(), - stdin: None, - timeout_ms: Some(60_000), - tty, - } +fn process_id(id: &str) -> anyhow::Result { + Ok(ProcessId::new(id)?) } -fn read_params(process_id: &str, wait_ms: Option) -> ReadProcessParams { - ReadProcessParams { - process_id: ProcessId::new(process_id), - after_seq: None, - max_bytes: None, - wait_ms, - } +fn path(root: &Path, name: &str) -> anyhow::Result { + let path = root.join(name); + Ok(EnvironmentPath::new(path.to_str().ok_or_else(|| { + anyhow::anyhow!("non-utf-8 path {}", path.display()) + })?)?) +} + +fn shell_process(process_id: &str, script: &str, pty: bool) -> anyhow::Result { + Ok(StartProcessParams { + process_id: self::process_id(process_id)?, + command: CommandSpec { + argv: vec!["/bin/sh".to_owned(), "-c".to_owned(), script.to_owned()], + cwd: None, + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input: ExecInput::None {}, + timeout_ms: Some(60_000), + }, + pty: pty.then_some(PtySize { rows: 24, cols: 80 }), + }) +} + +/// A cursor read that returns once the process exited and its output +/// settled. +fn read_until_exit(process_id: &str) -> anyhow::Result { + Ok(ReadProcessParams { + process_id: self::process_id(process_id)?, + after: None, + tail_bytes: None, + ack: None, + max_bytes: 65_536, + wait_ms: 10_000, + return_on: ReturnOn::Exit, + }) } -fn chunk_text(chunks: &[ProcessOutputChunk]) -> String { +fn page_text(chunks: &[OutputChunk]) -> String { let bytes = chunks .iter() - .flat_map(|chunk| chunk.chunk.as_slice().to_vec()) + .flat_map(|chunk| chunk.data.as_slice().to_vec()) .collect::>(); String::from_utf8_lossy(&bytes).into_owned() } @@ -764,68 +1292,182 @@ async fn wait_for_process_gone(pid: i32) -> anyhow::Result<()> { } } +fn read_receipt(path: &Path) -> anyhow::Result { + Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?) +} + +async fn wait_for_receipt_admission(path: &Path, admission: &str) -> anyhow::Result<()> { + let deadline = tokio::time::Instant::now() + SETTLE; + loop { + if read_receipt(path).is_ok_and(|receipt| receipt["admission"] == admission) { + return Ok(()); + } + if tokio::time::Instant::now() > deadline { + anyhow::bail!("receipt {} never said {admission}", path.display()); + } + tokio::time::sleep(POLL).await; + } +} + +/// A canonical, empty working tree for one daemon. +fn workspace(sandbox: &Path, name: &str) -> anyhow::Result { + let root = sandbox.join(name); + std::fs::create_dir_all(&root)?; + Ok(root.canonicalize()?) +} + +/// A daemon serving `root`, with its state outside the tree it serves. fn daemon_config( root: &Path, + state_dir: &Path, connect_url: &str, key: Option<&str>, receipt: Option<&Path>, + on_closed: OnClosed, ) -> anyhow::Result { - let root = root.canonicalize()?; let mut metadata = BTreeMap::new(); metadata.insert("harbor.trialId".to_owned(), "trial-1".to_owned()); + let mut registration = RegistrationConfig::new( + connect_url.to_owned(), + environment_daemon::upgrade::resolve_discovery_url(Some(connect_url), None)?, + ); + registration.registration_key = key.map(SecretString::new); + registration.display_name = Some(DisplayName::new("registration live daemon")?); + registration.metadata = Metadata::new(metadata)?; + registration.receipt_path = receipt.map(Path::to_path_buf); + registration.on_closed = on_closed; Ok(DaemonConfig { listen: None, - cwd: root.clone(), - fs_root: root.clone(), - state_dir: root.join(".lightspeed-envd"), + cwd: root.to_path_buf(), + fs_root: root.to_path_buf(), + state_dir: state_dir.to_path_buf(), read_only_fs: false, - registration: Some({ - let mut registration = RegistrationConfig::new( - connect_url.to_owned(), - environment_daemon::upgrade::resolve_discovery_url(Some(connect_url), None)?, - ); - registration.registration_key = key.map(SecretString::new); - registration.display_name = Some("registration live daemon".to_owned()); - registration.metadata = metadata; - registration.receipt_path = receipt.map(Path::to_path_buf); - registration - }), + registration: Some(registration), scrubbed_env: Vec::new(), }) } -fn spawn_daemon(config: DaemonConfig) -> tokio::task::JoinHandle> { - tokio::spawn(async move { - let runtime = DaemonRuntime::new(config)?; - environment_daemon::server::run(runtime).await +/// One running daemon: how it ends, and the runtime whose shutdown token +/// stops it deliberately. +struct Daemon { + exit: tokio::task::JoinHandle, + runtime: DaemonRuntime, +} + +fn spawn_daemon(config: DaemonConfig) -> anyhow::Result { + let runtime = DaemonRuntime::new(config)?; + let exit = tokio::spawn(environment_daemon::server::run(runtime.clone())); + Ok(Daemon { exit, runtime }) +} + +fn now_ms() -> i64 { + i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock after the epoch") + .as_millis(), + ) + .expect("millisecond clock fits i64") +} + +/// The registered facts of an environment view. +struct RegisteredFacts { + registration_key_id: String, + daemon_id: String, + identity_mode: EnvironmentIdentityModeView, + connection_changed_at_ms: i64, + last_seen_at_ms: i64, +} + +fn registered_source(view: &EnvironmentView) -> anyhow::Result { + let EnvironmentSourceView::Registered { + registration_key_id, + daemon_id, + identity_mode, + connection_changed_at_ms, + last_seen_at_ms, + .. + } = &view.source + else { + anyhow::bail!("expected a registered source, got {:?}", view.source) + }; + Ok(RegisteredFacts { + registration_key_id: registration_key_id.clone(), + daemon_id: daemon_id.clone(), + identity_mode: *identity_mode, + connection_changed_at_ms: *connection_changed_at_ms, + last_seen_at_ms: *last_seen_at_ms, }) } +/// Whether a registered environment's daemon is connected; `None` for +/// other kinds. +fn is_connected(view: &EnvironmentView) -> Option { + match &view.source { + EnvironmentSourceView::Registered { connected, .. } => Some(*connected), + _ => None, + } +} + +/// Dial the worker route and expect the gateway to refuse the upgrade. +async fn expect_route_refusal( + gateway: &EnvironmentGatewayClientConfig, + key: &RouteKey, +) -> anyhow::Result { + match gateway + .connect(key, gateway.connect_options("registration-live")) + .await + { + Ok((client, _)) => { + client.close().await?; + anyhow::bail!("the worker route upgraded for {}", key.environment_id) + } + Err(error) => Ok(error), + } +} + +async fn read_key( + api: &GatewayAgentApi, + key_id: &str, +) -> anyhow::Result { + Ok(api + .read_environment_registration_key(EnvironmentRegistrationKeyReadParams { + registration_key_id: key_id.to_owned(), + }) + .await? + .result + .registration_key) +} + +/// Wait until exactly `count` open environments are listed under the key +/// and one of them is connected. async fn wait_for_registered( api: &GatewayAgentApi, key_id: &str, count: usize, - status: EnvironmentLifecycleStatusView, + connected: fn(&EnvironmentView) -> Option, ) -> anyhow::Result { let deadline = tokio::time::Instant::now() + SETTLE; loop { - let environments = list_by_key(api, key_id).await?; + let environments = list_by_key(api, key_id, false).await?; if environments.len() == count && let Some(environment) = environments .iter() - .find(|environment| environment.status == status) + .find(|environment| connected(environment) == Some(true)) { return Ok(environment.clone()); } if tokio::time::Instant::now() > deadline { anyhow::bail!( - "timed out waiting for {count} environment(s) under {key_id} with one {status:?}: {environments:?}" + "timed out waiting for {count} environment(s) under {key_id} with one connected: {environments:?}" ); } tokio::time::sleep(POLL).await; } } +/// Wait until exactly `count` open environments are listed under the key, +/// all connected. async fn wait_for_registered_count( api: &GatewayAgentApi, key_id: &str, @@ -833,26 +1475,30 @@ async fn wait_for_registered_count( ) -> anyhow::Result> { let deadline = tokio::time::Instant::now() + SETTLE; loop { - let environments = list_by_key(api, key_id).await?; + let environments = list_by_key(api, key_id, false).await?; if environments.len() == count && environments .iter() - .all(|environment| environment.status == EnvironmentLifecycleStatusView::Ready) + .all(|environment| is_connected(environment) == Some(true)) { return Ok(environments); } if tokio::time::Instant::now() > deadline { - anyhow::bail!("timed out waiting for {count} ready environments: {environments:?}"); + anyhow::bail!("timed out waiting for {count} connected environments: {environments:?}"); } tokio::time::sleep(POLL).await; } } -async fn list_by_key(api: &GatewayAgentApi, key_id: &str) -> anyhow::Result> { +async fn list_by_key( + api: &GatewayAgentApi, + key_id: &str, + include_closed: bool, +) -> anyhow::Result> { Ok(api .list_environments(EnvironmentListParams { - metadata: Default::default(), registration_key_id: Some(key_id.to_owned()), + include_closed, ..EnvironmentListParams::default() }) .await? @@ -860,11 +1506,12 @@ async fn list_by_key(api: &GatewayAgentApi, key_id: &str) -> anyhow::Result anyhow::Result<()> { + what: &str, + ready: impl Fn(&EnvironmentView) -> bool, +) -> anyhow::Result { let deadline = tokio::time::Instant::now() + SETTLE; loop { let environment = api @@ -874,15 +1521,25 @@ async fn wait_for_status( .await? .result .environment; - if environment.status == status { - return Ok(()); + if ready(&environment) { + return Ok(environment); } if tokio::time::Instant::now() > deadline { anyhow::bail!( - "timed out waiting for {environment_id} to be {status:?}; last {:?}", - environment.status + "timed out waiting for {environment_id} to be {what}; last {environment:?}" ); } tokio::time::sleep(POLL).await; } } + +async fn wait_for_state( + api: &GatewayAgentApi, + environment_id: &str, + state: EnvironmentStateView, +) -> anyhow::Result { + wait_for_environment(api, environment_id, &format!("{state:?}"), |view| { + view.state == state + }) + .await +} diff --git a/crates/temporal-runtime/tests/profiles_live.rs b/crates/temporal-runtime/tests/profiles_live.rs index 88bdfdd1c..5665e6ca8 100644 --- a/crates/temporal-runtime/tests/profiles_live.rs +++ b/crates/temporal-runtime/tests/profiles_live.rs @@ -13,7 +13,7 @@ use api_projection::model_to_api; use harness::SessionId; use support::live::{ LIVE_TEST_LOCK, fake_worker_activities, final_assistant_text, live_workflow_handle, - read_session_view, require_storage_live_env, run_with_live_worker, wait_for_environment_status, + read_session_view, require_storage_live_env, run_with_live_worker, wait_for_environment_state, wait_for_terminal_run, }; use temporal_runtime::{gateway::GatewayAgentApi, pg_store_from_env}; @@ -42,6 +42,88 @@ async fn temporal_live_profile_selection_leaves_environment_lifecycle_independen run_with_live_worker(activities, run_profile_environment_selection_live_client).await } +/// Registers the fake controller as a provider bound to this universe, +/// directly through the store: the deployment API is deployment-scoped and +/// these tests drive one universe's gateway. +async fn register_fake_provider( + store: &store_pg::PgStore, + controller: &temporal_runtime::environments::fake_controller::FakeController, + provider_id: &str, +) -> anyhow::Result<()> { + use environments::{ + EnvironmentEndpoint, EnvironmentProviderBindingStore, EnvironmentProviderId, + EnvironmentProviderStore, ProviderCredentialChange, PutEnvironmentProvider, + PutEnvironmentProviderBinding, + }; + + store + .put_provider(PutEnvironmentProvider { + provider_id: EnvironmentProviderId::new(provider_id), + display_name: Some("Live fake provider".to_owned()), + controller_endpoint: EnvironmentEndpoint::parse(controller.controller_url())?, + data_route_base: None, + credential: match controller.credential() { + Some(credential) => { + ProviderCredentialChange::Set(auth::SecretValue::new(credential)) + } + None => ProviderCredentialChange::Keep, + }, + updated_at_ms: 1, + }) + .await?; + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id: store.config().universe_id, + provider_id: EnvironmentProviderId::new(provider_id), + enabled: true, + updated_at_ms: 1, + }) + .await?; + Ok(()) +} + +async fn delete_fake_provider(store: &store_pg::PgStore, provider_id: &str) -> anyhow::Result<()> { + use environments::{ + EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, + }; + + let provider_id = EnvironmentProviderId::new(provider_id); + store + .delete_provider_binding(store.config().universe_id, &provider_id) + .await?; + store.delete_provider(&provider_id).await?; + Ok(()) +} + +fn profile_with_environments( + profile_id: &ProfileId, + model: &harness::ModelSelection, + environments: Vec, +) -> AgentProfileInput { + AgentProfileInput { + profile_id: profile_id.clone(), + display_name: None, + description: None, + document: ProfileDocument { + config: Some(SessionConfig { + model: Some(model_to_api(model)), + features: Some(api::FeaturesConfig { + environments: Some(api::EnvironmentsFeature { + version: api::CURRENT_FEATURE_VERSION, + selection: false, + prompts: None, + skills: None, + environments, + }), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }, + } +} + /// Closing and deleting a session never closes its selected environment. async fn run_profile_environment_selection_live_client( client: Client, @@ -50,12 +132,7 @@ async fn run_profile_environment_selection_live_client( ) -> anyhow::Result<()> { use std::collections::BTreeMap; - use environment_protocol::shared::EnvironmentTransport; - use environments::{ - EnvironmentConnectionSpec, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, - EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, - PutEnvironmentProvider, PutEnvironmentProviderBinding, - }; + use temporal_runtime::environments::fake_controller::{FAKE_TEMPLATE_ID, FakeController}; let store = pg_store_from_env().await?; let model = support::live::openai_live_model(); @@ -65,80 +142,36 @@ async fn run_profile_environment_selection_live_client( .build(); let suffix = uuid::Uuid::new_v4().simple().to_string(); let provider_id = format!("fake-profile-{suffix}"); - let binding_id = format!("binding-profile-{suffix}"); let profile_id = ProfileId::new(format!("live_selection_{suffix}")); - // Register the in-process fake provider and bind it to this universe - // directly through the store: the deployment API is deployment-scoped and - // this test drives one universe's gateway. - store - .put_provider(PutEnvironmentProvider { - provider_id: EnvironmentProviderId::new(provider_id.clone()), - display_name: Some("Live fake provider".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "in-process", - EnvironmentTransport::Provider { - provider_type: "fake".to_owned(), - }, - ), - metadata: BTreeMap::new(), - updated_at_ms: 1, - }) - .await?; - store - .put_provider_binding(PutEnvironmentProviderBinding { - universe_id: store.config().universe_id, - binding_id: EnvironmentProviderBindingId::new(binding_id.clone()), - provider_id: EnvironmentProviderId::new(provider_id.clone()), - status: EnvironmentProviderBindingStatus::Enabled, - expected_revision: None, - metadata: BTreeMap::new(), - updated_at_ms: 1, - }) - .await?; + let controller = FakeController::start().await?; + register_fake_provider(&store, &controller, &provider_id).await?; let environment = api .create_environment(api::EnvironmentCreateParams { request_id: format!("independent-{suffix}"), - binding_id: binding_id.clone(), - template_id: "rust-v1".into(), + provider_id: provider_id.clone(), + template_id: FAKE_TEMPLATE_ID.into(), display_name: None, metadata: BTreeMap::new(), - idle_policy: None, + idle: None, }) .await? .result .environment; let environment_id = environment.environment_id.clone(); api.create_profile(ProfileCreateParams { - profile: AgentProfileInput { - profile_id: profile_id.clone(), - display_name: None, - description: None, - document: ProfileDocument { - config: Some(SessionConfig { - model: Some(model_to_api(&model)), - features: Some(api::FeaturesConfig { - environments: Some(api::EnvironmentsFeature { - version: api::CURRENT_FEATURE_VERSION, - selection: false, - prompts: None, - skills: None, - environments: vec![api::EnvironmentAttachment { - environment_id: Some(environment_id.clone()), - inherit: false, - default: true, - access: api::EnvironmentAccess::Read, - working_directory: None, - }], - }), - ..Default::default() - }), - ..Default::default() - }), - ..Default::default() - }, - }, + profile: profile_with_environments( + &profile_id, + &model, + vec![api::EnvironmentAttachment { + environment_id: Some(environment_id.clone()), + inherit: false, + default: true, + access: api::EnvironmentAccess::Read, + working_directory: None, + }], + ), }) .await?; let start = || { @@ -174,12 +207,7 @@ async fn run_profile_environment_selection_live_client( }) .await?; assert!(!applied.result.applied.active_environment_changed); - wait_for_environment_status( - &api, - &environment_id, - api::EnvironmentLifecycleStatusView::Ready, - ) - .await?; + wait_for_environment_state(&api, &environment_id, api::EnvironmentStateView::Open).await?; api.close_session(api::SessionCloseParams { session_id: session_id.to_string(), force: false, @@ -187,14 +215,8 @@ async fn run_profile_environment_selection_live_client( .await?; api.reconcile_environments_once().await?; assert_eq!( - api.read_environment(api::EnvironmentReadParams { - environment_id: environment_id.clone() - }) - .await? - .result - .environment - .status, - api::EnvironmentLifecycleStatusView::Ready + read_environment_view(&api, &environment_id).await?.state, + api::EnvironmentStateView::Open ); api.delete_session(api::SessionDeleteParams { shared_only: false, @@ -205,37 +227,30 @@ async fn run_profile_environment_selection_live_client( .await?; api.reconcile_environments_once().await?; assert_eq!( - api.read_environment(api::EnvironmentReadParams { - environment_id: environment_id.clone() - }) - .await? - .result - .environment - .status, - api::EnvironmentLifecycleStatusView::Ready + read_environment_view(&api, &environment_id).await?.state, + api::EnvironmentStateView::Open ); api.close_environment(api::EnvironmentCloseParams { environment_id: environment_id.clone(), }) .await?; - wait_for_environment_status( - &api, - &environment_id, - api::EnvironmentLifecycleStatusView::Closed, - ) - .await?; + wait_for_environment_state(&api, &environment_id, api::EnvironmentStateView::Closed).await?; api.delete_profile(ProfileDeleteParams { profile_id }) .await?; - store - .delete_provider_binding( - store.config().universe_id, - &EnvironmentProviderBindingId::new(binding_id), - ) - .await?; - store - .delete_provider(&EnvironmentProviderId::new(provider_id)) - .await?; - Ok(()) + delete_fake_provider(&store, &provider_id).await +} + +async fn read_environment_view( + api: &GatewayAgentApi, + environment_id: &str, +) -> anyhow::Result { + Ok(api + .read_environment(api::EnvironmentReadParams { + environment_id: environment_id.to_owned(), + }) + .await? + .result + .environment) } async fn run_profiles_live_client( diff --git a/crates/temporal-runtime/tests/runs_live.rs b/crates/temporal-runtime/tests/runs_live.rs index b82013fef..d8fe71cc9 100644 --- a/crates/temporal-runtime/tests/runs_live.rs +++ b/crates/temporal-runtime/tests/runs_live.rs @@ -156,9 +156,10 @@ async fn temporal_live_steering_during_final_turn_adds_a_turn() -> anyhow::Resul // steering is materialized while the call is in flight, and instead of // completing on that final output the run gets one more turn whose // request carries the steering — so "steer" never silently does - // nothing. + // nothing. Zero tool rounds: every session carries the core blob tools, + // so the fake model would otherwise spend its first turn calling one. let (activities, counters) = - fake_worker_activities_for_run_control(Duration::from_secs(4), Duration::ZERO, 1).await?; + fake_worker_activities_for_run_control(Duration::from_secs(4), Duration::ZERO, 0).await?; run_with_live_worker(activities, move |client, task_queue, session_id| { run_steering_final_turn_live_client(client, task_queue, session_id, counters) }) @@ -175,9 +176,12 @@ async fn temporal_live_queued_runs_return_promptly_and_run_in_order() -> anyhow: // Queueing: runs started behind an active run return `queued` at // once, are visible in the session view, can be cancelled while queued, // and start in order after the active run ends. Cancelling the active - // run leaves the queued one untouched and it runs next. + // run leaves the queued one untouched and it runs next. Zero tool + // rounds: every session carries the core blob tools, so the fake model + // would otherwise spend the first executed run's opening turn calling + // one, making that run take two generations. let (activities, counters) = - fake_worker_activities_for_run_control(Duration::from_secs(5), Duration::ZERO, 1).await?; + fake_worker_activities_for_run_control(Duration::from_secs(5), Duration::ZERO, 0).await?; run_with_live_worker(activities, move |client, task_queue, session_id| { run_queue_live_client(client, task_queue, session_id, counters) }) @@ -509,7 +513,8 @@ async fn run_steering_final_turn_live_client( session_id: SessionId, counters: temporal_runtime::worker::FakeRuntimeCounters, ) -> anyhow::Result<()> { - // No tools: the fake model answers in one turn. + // No workspace tools, and the worker's fake model takes zero tool rounds, + // so it answers in one turn. let api = run_control_api(&client, task_queue, &session_id, false).await?; let run = start_text_run(&api, &session_id, "answer directly").await?; wait_until( @@ -683,11 +688,11 @@ async fn run_queue_live_client( wait_for_terminal_run_with_timeout(&api, &session_id, &fourth.id, 40).await?; assert_eq!(fourth_terminal.status, api::RunStatus::Completed); - // Exactly one generation per executed run (first, third, fourth); the - // cancelled first run never got a second turn. Whether the worker - // abandoned the first generation or it completed (and was discarded) - // depends on heartbeat timing with this short delay and is covered by - // the mid-generation cancel scenario. + // Exactly one generation per executed run (first, third, fourth), since + // the fake model answers directly; the cancelled first run never got a + // second turn. Whether the worker abandoned the first generation or it + // completed (and was discarded) depends on heartbeat timing with this + // short delay and is covered by the mid-generation cancel scenario. assert_eq!(counters.generations_started(), 3); assert_eq!( counters.generations_completed() + counters.generations_abandoned(), @@ -699,7 +704,7 @@ async fn run_queue_live_client( } /// `wait_for_terminal_run` with a longer budget, for scenarios whose fake -/// provider sleeps on purpose. +/// provider sleeps on purpose or that drive many sequential activities. async fn wait_for_terminal_run_with_timeout( api: &GatewayAgentApi, session_id: &SessionId, @@ -1100,7 +1105,12 @@ async fn run_unbounded_hosted_run_live_client( config: None, }) .await?; - let run = wait_for_terminal_run(&api, &session_id, &started.result.run.id).await?; + // Thirty-one generations and thirty tool batches run back to back, which + // takes about 30 s on the local dev stack, right at the default + // terminal-run budget. Wait with headroom: the test is about transition + // count, not latency. + let run = + wait_for_terminal_run_with_timeout(&api, &session_id, &started.result.run.id, 120).await?; assert!( final_assistant_text(&run).is_some_and(|text| text.contains("Fake agent completed run")) ); diff --git a/crates/temporal-runtime/tests/subagents_live.rs b/crates/temporal-runtime/tests/subagents_live.rs index 41ae3566f..3f000d1b5 100644 --- a/crates/temporal-runtime/tests/subagents_live.rs +++ b/crates/temporal-runtime/tests/subagents_live.rs @@ -1314,49 +1314,14 @@ async fn run_agent_run_inherit_environment_live_client( ) -> anyhow::Result<()> { use std::collections::BTreeMap; - use environment_protocol::shared::EnvironmentTransport; - use environments::{ - EnvironmentConnectionSpec, EnvironmentProviderBindingId, EnvironmentProviderBindingStatus, - EnvironmentProviderBindingStore, EnvironmentProviderId, EnvironmentProviderStore, - PutEnvironmentProvider, PutEnvironmentProviderBinding, - }; + use temporal_runtime::environments::fake_controller::FakeController; let store = pg_store_from_env().await?; let suffix = uuid::Uuid::new_v4().simple().to_string(); let provider_id = format!("fake-inherit-{suffix}"); - let binding_id = format!("binding-inherit-{suffix}"); - store - .put_provider(PutEnvironmentProvider { - provider_id: EnvironmentProviderId::new(provider_id.clone()), - display_name: Some("Live fake provider".to_owned()), - controller_connection: EnvironmentConnectionSpec::new( - "in-process", - EnvironmentTransport::Provider { - provider_type: "fake".to_owned(), - }, - ), - metadata: BTreeMap::new(), - updated_at_ms: 1, - }) - .await?; - store - .put_provider_binding(PutEnvironmentProviderBinding { - universe_id: store.config().universe_id, - binding_id: EnvironmentProviderBindingId::new(binding_id.clone()), - provider_id: EnvironmentProviderId::new(provider_id.clone()), - status: EnvironmentProviderBindingStatus::Enabled, - expected_revision: None, - metadata: BTreeMap::new(), - updated_at_ms: 1, - }) - .await?; - let environments_feature = |attachment: api::EnvironmentAttachment| api::EnvironmentsFeature { - version: api::CURRENT_FEATURE_VERSION, - selection: false, - prompts: None, - skills: None, - environments: vec![attachment], - }; + // Serves the controller and data routes until the test returns. + let controller = FakeController::start().await?; + register_fake_provider(&store, &controller, &provider_id).await?; // Child profile: inherits whatever environment its parent has active. let child_profile_id = ProfileId::new(format!("live_inherit_child_{suffix}")); @@ -1392,11 +1357,11 @@ async fn run_agent_run_inherit_environment_live_client( let independent_environment = api .create_environment(api::EnvironmentCreateParams { request_id: format!("inherit-env-{suffix}"), - binding_id: binding_id.clone(), + provider_id: provider_id.clone(), template_id: "rust-v1".into(), display_name: None, metadata: BTreeMap::new(), - idle_policy: None, + idle: None, }) .await? .result @@ -1507,12 +1472,11 @@ async fn run_agent_run_inherit_environment_live_client( .environment; assert!( !matches!( - environment.status, - api::EnvironmentLifecycleStatusView::Closing - | api::EnvironmentLifecycleStatusView::Closed + environment.state, + api::EnvironmentStateView::Closing | api::EnvironmentStateView::Closed ), "inherited environment must stay open after the child closes, got {:?}", - environment.status + environment.state ); let mut all = vec![session_id]; @@ -1525,6 +1489,56 @@ async fn run_agent_run_inherit_environment_live_client( Ok(()) } +fn environments_feature(attachment: api::EnvironmentAttachment) -> api::EnvironmentsFeature { + api::EnvironmentsFeature { + version: api::CURRENT_FEATURE_VERSION, + selection: false, + prompts: None, + skills: None, + environments: vec![attachment], + } +} + +/// Registers the fake controller as a provider bound to this universe, +/// directly through the store: the deployment API is deployment-scoped and +/// these tests drive one universe's gateway. +async fn register_fake_provider( + store: &store_pg::PgStore, + controller: &temporal_runtime::environments::fake_controller::FakeController, + provider_id: &str, +) -> anyhow::Result<()> { + use environments::{ + EnvironmentEndpoint, EnvironmentProviderBindingStore, EnvironmentProviderId, + EnvironmentProviderStore, ProviderCredentialChange, PutEnvironmentProvider, + PutEnvironmentProviderBinding, + }; + + store + .put_provider(PutEnvironmentProvider { + provider_id: EnvironmentProviderId::new(provider_id), + display_name: Some("Live fake provider".to_owned()), + controller_endpoint: EnvironmentEndpoint::parse(controller.controller_url())?, + data_route_base: None, + credential: match controller.credential() { + Some(credential) => { + ProviderCredentialChange::Set(auth::SecretValue::new(credential)) + } + None => ProviderCredentialChange::Keep, + }, + updated_at_ms: 1, + }) + .await?; + store + .put_provider_binding(PutEnvironmentProviderBinding { + universe_id: store.config().universe_id, + provider_id: EnvironmentProviderId::new(provider_id), + enabled: true, + updated_at_ms: 1, + }) + .await?; + Ok(()) +} + async fn run_agent_spawn_cancel_live_client( client: Client, session_id: SessionId, diff --git a/crates/temporal-runtime/tests/support/live.rs b/crates/temporal-runtime/tests/support/live.rs index 5b190bfc4..2d88060b2 100644 --- a/crates/temporal-runtime/tests/support/live.rs +++ b/crates/temporal-runtime/tests/support/live.rs @@ -472,10 +472,12 @@ pub async fn read_session_view( .session) } -pub async fn wait_for_environment_status( +/// Drive the reconciler of `api`'s universe until the environment reaches +/// `expected`. +pub async fn wait_for_environment_state( api: &temporal_runtime::gateway::GatewayAgentApi, environment_id: &str, - expected: api::EnvironmentLifecycleStatusView, + expected: api::EnvironmentStateView, ) -> anyhow::Result { let started = Instant::now(); loop { @@ -487,13 +489,14 @@ pub async fn wait_for_environment_status( .await? .result .environment; - if environment.status == expected { + if environment.state == expected { return Ok(environment); } if started.elapsed() > Duration::from_secs(20) { anyhow::bail!( - "timed out waiting for environment {environment_id} to reach {expected:?}; current status is {:?}", - environment.status + "timed out waiting for environment {environment_id} to reach {expected:?}; current state is {:?} ({:?})", + environment.state, + environment.state_message ); } tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/crates/temporal-runtime/tests/vfs_transfer_live.rs b/crates/temporal-runtime/tests/vfs_transfer_live.rs index 2faf3451a..b00fd2209 100644 --- a/crates/temporal-runtime/tests/vfs_transfer_live.rs +++ b/crates/temporal-runtime/tests/vfs_transfer_live.rs @@ -13,11 +13,13 @@ use environment_daemon::{ config::{DaemonConfig, RegistrationConfig}, }; use environment_protocol::{ + bytes::SecretString, data::{ - handshake::InitializeParams, - inventory::{ScanContent, ScanDigestAlgorithm, ScanParams}, + fs::{TreeContent, TreeFilter}, + scan::{ScanLimits, ScanParams, ScanRoot, ScanRootResult, ScanSkip, ScanSkipReason}, }, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentPath, SecretString}, + paths::EnvironmentPath, + routes::CONNECT_PATH, }; use environments::EnvironmentStore; use harness::{ @@ -112,97 +114,225 @@ async fn temporal_live_vfs_transfers_follow_profile_grants_and_publish_large_fil let mut tasks = Tasks(vec![gateway.abort_handle()]); let sandbox = tempfile::tempdir()?; let root = sandbox.path().canonicalize()?; + let daemon_state = tempfile::tempdir()?; let context = support::live::local_request_context_for(api::AccessScope::Universe { universe_id }) .await?; // Keep the large transfer scenario off the current-thread test stack. - let result = temporal_runtime::gateway::request_context::with_request_context(context, Box::pin(async { - let state = runtime.state_for(universe_id, false).await?; - let key = state.api.create_environment_registration_key(api::EnvironmentRegistrationKeyCreateParams { - display_name: "VFS transfer live".into(), - identity_mode: api::EnvironmentIdentityModeView::Ephemeral, - max_active_environments: Some(1), - ephemeral_disconnect_grace_ms: None, - expires_at_ms: None, - }).await?.result; - let connect_url = format!("ws://{address}/environment-gateway/connect"); - let mut registration = RegistrationConfig::new(connect_url.clone(), - environment_daemon::upgrade::resolve_discovery_url(Some(&connect_url), None)?); - registration.registration_key = Some(SecretString::new(key.secret.0)); - let daemon = DaemonRuntime::new(DaemonConfig { - listen: None, cwd: root.clone(), fs_root: root.clone(), state_dir: root.join(".envd"), - read_only_fs: false, registration: Some(registration), scrubbed_env: Vec::new(), - })?; - let daemon = tokio::spawn(environment_daemon::server::run(daemon)); - tasks.0.push(daemon.abort_handle()); - let environment = tokio::time::timeout(Duration::from_secs(30), async { - loop { - let environments = state.api.list_environments(api::EnvironmentListParams::default()).await?.result.environments; - if let Some(environment) = environments.into_iter().find(|e| e.status == api::EnvironmentLifecycleStatusView::Ready) { - return anyhow::Ok(environment); + let result = temporal_runtime::gateway::request_context::with_request_context( + context, + Box::pin(async { + let state = runtime.state_for(universe_id, false).await?; + let key = state + .api + .create_environment_registration_key(api::EnvironmentRegistrationKeyCreateParams { + display_name: "VFS transfer live".into(), + identity_mode: api::EnvironmentIdentityModeView::Ephemeral, + max_active_environments: Some(1), + ephemeral_disconnect_grace_ms: None, + offline_close_after_ms: None, + expires_at_ms: None, + }) + .await? + .result; + let connect_url = format!("ws://{address}{CONNECT_PATH}"); + let mut registration = RegistrationConfig::new( + connect_url.clone(), + environment_daemon::upgrade::resolve_discovery_url(Some(&connect_url), None)?, + ); + registration.registration_key = Some(SecretString::new(key.secret.0)); + let daemon = DaemonRuntime::new(DaemonConfig { + listen: None, + cwd: root.clone(), + fs_root: root.clone(), + state_dir: daemon_state.path().into(), + read_only_fs: false, + registration: Some(registration), + scrubbed_env: Vec::new(), + })?; + let daemon = tokio::spawn(environment_daemon::server::run(daemon)); + tasks.0.push(daemon.abort_handle()); + let environment = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let environments = state + .api + .list_environments(api::EnvironmentListParams::default()) + .await? + .result + .environments; + if let Some(environment) = environments + .into_iter() + .find(|e| e.state == api::EnvironmentStateView::Open) + { + return anyhow::Ok(environment); + } + tokio::time::sleep(Duration::from_millis(100)).await; } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }).await??; - let hosted = Arc::new(SessionTools::from_pg_store(state.store.clone()) - .with_environment_gateway(gateway_config.clone())); - let activities = WorkerActivities::for_universe(universe_id, - ActivityState::from_pg_store(state.store.clone(), Arc::new(TransferLlm { blobs: state.store.clone() }), hosted.clone()) - .with_hosted_tools(hosted).with_environment_gateway(gateway_config.clone())); - let gateway_config = gateway_config.clone(); - let store = state.store.clone(); - run_with_live_worker(activities, |client, queue, base_session| async move { - let api = GatewayAgentApi::builder(client.clone(), store.clone()) - .with_task_queue(queue).with_environment_gateway(gateway_config.clone()).build(); - // All provider surfaces execute both directions. Additional sessions - // test the public grant boundary using the same profile path. - for (index, (provider, mode)) in [ - (ProviderApiKind::OpenAiResponses, "edit"), - (ProviderApiKind::AnthropicMessages, "edit"), - (ProviderApiKind::OpenAiCompletions, "edit"), - (ProviderApiKind::OpenAiResponses, "readonly"), - (ProviderApiKind::OpenAiResponses, "sourcing"), - (ProviderApiKind::OpenAiResponses, "noenv"), - (ProviderApiKind::OpenAiResponses, "content"), - (ProviderApiKind::AnthropicMessages, "content"), - (ProviderApiKind::OpenAiCompletions, "content"), - (ProviderApiKind::OpenAiResponses, "contentread"), - ].into_iter().enumerate() { - let session = SessionId::new(format!("{base_session}_{index}_{mode}")); - let case = Box::pin(run_case(&api, store.as_ref(), &session, provider, mode, index, &root, &environment.environment_id)).await; - let handle = client.get_workflow_handle::( - temporal_workflow::compose_workflow_id(universe_id, &session)); - let _ = handle.terminate(WorkflowTerminateOptions::builder().reason("transfer live cleanup").build()).await; - case?; - } - // Scan negotiation and conditional observations use the same reverse route. - let record = store.read_environment(&harness::EnvironmentId::new(&environment.environment_id)).await?; - let connection = gateway_config.connection_for(universe_id, &record); - let mut remote = environment_client::EnvironmentDataClient::connect(&connection.endpoint, gateway_config.connect_options("transfer-live-scan")).await?; - let initialized = remote.initialize(&InitializeParams { - protocol_version: CURRENT_PROTOCOL_VERSION, client_name: "transfer-live-scan".into(), - scope: connection.scope, resume_connection_id: None, - }).await?; - anyhow::ensure!(initialized.capabilities.filesystem_scan && initialized.capabilities.filesystem_transfer); - let mut query = ScanParams { - follow_symlinks: false, - roots: vec![EnvironmentPath::new("./capture-source")?], include_patterns: vec!["new.bin".into()], - read_content: false, digest_algorithm: Some(ScanDigestAlgorithm::Sha256), limits: Default::default(), if_none_match: None, - }; - let scan = remote.scan(&query).await?; - anyhow::ensure!(scan.complete && scan.diagnostics.is_empty()); - assert!(scan.entries.iter().any(|entry| matches!(&entry.content, ScanContent::File { digest: Some(digest), .. } - if digest == BlobRef::from_bytes(&std::fs::read(root.join("capture-source/new.bin")).unwrap()).as_str()))); - query.if_none_match = Some(scan.fingerprint.expect("complete scan fingerprint")); - let unchanged = remote.scan(&query).await?; - anyhow::ensure!(unchanged.unchanged && unchanged.entries.is_empty()); - std::fs::write(root.join("capture-source/new.bin"), b"changed after observation")?; - let changed = remote.scan(&query).await?; - anyhow::ensure!(changed.complete && !changed.unchanged && changed.fingerprint != query.if_none_match); - remote.close().await?; - anyhow::Ok(()) - }).await - })).await; + }) + .await??; + let hosted = Arc::new( + SessionTools::from_pg_store(state.store.clone()) + .with_environment_gateway(gateway_config.clone()), + ); + let activities = WorkerActivities::for_universe( + universe_id, + ActivityState::from_pg_store( + state.store.clone(), + Arc::new(TransferLlm { + blobs: state.store.clone(), + }), + hosted.clone(), + ) + .with_hosted_tools(hosted) + .with_environment_gateway(gateway_config.clone()), + ); + let gateway_config = gateway_config.clone(); + let store = state.store.clone(); + run_with_live_worker(activities, |client, queue, base_session| async move { + let api = GatewayAgentApi::builder(client.clone(), store.clone()) + .with_task_queue(queue) + .with_environment_gateway(gateway_config.clone()) + .build(); + // All provider surfaces execute both directions. Additional sessions + // test the public grant boundary using the same profile path. + for (index, (provider, mode)) in [ + (ProviderApiKind::OpenAiResponses, "edit"), + (ProviderApiKind::AnthropicMessages, "edit"), + (ProviderApiKind::OpenAiCompletions, "edit"), + (ProviderApiKind::OpenAiResponses, "readonly"), + (ProviderApiKind::OpenAiResponses, "sourcing"), + (ProviderApiKind::OpenAiResponses, "noenv"), + (ProviderApiKind::OpenAiResponses, "content"), + (ProviderApiKind::AnthropicMessages, "content"), + (ProviderApiKind::OpenAiCompletions, "content"), + (ProviderApiKind::OpenAiResponses, "contentread"), + ] + .into_iter() + .enumerate() + { + let session = SessionId::new(format!("{base_session}_{index}_{mode}")); + let case = Box::pin(run_case( + &api, + store.as_ref(), + &session, + provider, + mode, + index, + &root, + &environment.environment_id, + )) + .await; + let handle = client + .get_workflow_handle::( + temporal_workflow::compose_workflow_id(universe_id, &session), + ); + let _ = handle + .terminate( + WorkflowTerminateOptions::builder() + .reason("transfer live cleanup") + .build(), + ) + .await; + case?; + } + // Scan negotiation and conditional observations use the same reverse route. + let record = store + .read_environment(&harness::EnvironmentId::new(&environment.environment_id)) + .await?; + let (remote, initialized) = environment_client::DataClient::connect_initialized( + &gateway_config.endpoint_for(universe_id, &record), + gateway_config.connect_options("transfer-live-scan"), + ) + .await?; + anyhow::ensure!(initialized.capabilities.filesystem.is_some()); + let ceilings = initialized.limits.filesystem.scan; + // Scans observe small files such as skills and prompts: the + // large captured file is listed as skipped, never hashed, + // while a small sibling is observed with its digest. + std::fs::write(root.join("capture-source/observed.txt"), b"observed")?; + let mut query = ScanParams { + roots: vec![ScanRoot { + path: EnvironmentPath::new(root.join("capture-source").to_string_lossy())?, + if_none_match: None, + max_depth: None, + }], + filter: TreeFilter { + include: vec!["new.bin".into(), "observed.txt".into()], + ..TreeFilter::default() + }, + read_content: false, + include_digests: true, + follow_symlinks: false, + limits: ScanLimits { + max_entries: ceilings.max_entries, + max_file_bytes: ceilings.max_file_bytes, + max_total_bytes: ceilings.max_total_bytes, + max_duration_ms: ceilings.max_duration_ms, + }, + }; + anyhow::ensure!( + std::fs::metadata(root.join("capture-source/new.bin"))?.len() + > ceilings.max_file_bytes, + "the captured file must exceed the scan's per-file ceiling" + ); + let expected = + BlobRef::from_bytes(&std::fs::read(root.join("capture-source/observed.txt"))?); + let scan = remote.scan(&query).await?; + let fingerprint = match scan.roots.as_slice() { + [ + ScanRootResult::Complete { + fingerprint, + entries, + skipped, + skipped_entry_count, + .. + }, + ] => { + anyhow::ensure!( + *skipped_entry_count == 1 + && matches!(skipped.as_slice(), [ScanSkip { + path: Some(path), + reason: ScanSkipReason::FileTooLarge, + }] if path.as_str() == "new.bin"), + "{scan:?}" + ); + anyhow::ensure!( + matches!(entries.as_slice(), [entry] + if entry.path.as_str() == "observed.txt" + && matches!(&entry.content, TreeContent::File { digest: Some(digest), .. } + if format!("sha256:{}", digest.hex()) == expected.as_str())), + "{scan:?}" + ); + fingerprint.clone() + } + other => anyhow::bail!("capture-source scan did not complete: {other:?}"), + }; + query.roots[0].if_none_match = Some(fingerprint.clone()); + let unchanged = remote.scan(&query).await?; + anyhow::ensure!( + matches!( + unchanged.roots.as_slice(), + [ScanRootResult::Unchanged { .. }] + ), + "{unchanged:?}" + ); + std::fs::write( + root.join("capture-source/observed.txt"), + b"changed after observation", + )?; + let changed = remote.scan(&query).await?; + anyhow::ensure!( + matches!(changed.roots.as_slice(), + [ScanRootResult::Complete { fingerprint: changed, .. }] if *changed != fingerprint), + "{changed:?}" + ); + remote.close().await?; + anyhow::Ok(()) + }) + .await + }), + ) + .await; drop(tasks); let store = runtime.state_for(universe_id, false).await?.store.clone(); runtime.evict(universe_id).await; diff --git a/crates/temporal-workflow/Cargo.toml b/crates/temporal-workflow/Cargo.toml index b6d5bfce0..49f5fac0b 100644 --- a/crates/temporal-workflow/Cargo.toml +++ b/crates/temporal-workflow/Cargo.toml @@ -11,7 +11,6 @@ harness = { path = "../harness", features = ["contract"] } schemars = { version = "1", features = ["uuid1"] } sha2 = "0.11" hex = "0.4" -environment-protocol = { path = "../environment-protocol" } anyhow = "1" chrono = { version = "0.4", default-features = false, features = ["std"] } futures = "0.3" diff --git a/crates/temporal-workflow/src/activities.rs b/crates/temporal-workflow/src/activities.rs index 603c7762a..f468a63d8 100644 --- a/crates/temporal-workflow/src/activities.rs +++ b/crates/temporal-workflow/src/activities.rs @@ -9,10 +9,11 @@ use crate::{ AppendEventsRequest, AwaitEnvironmentReadyActivityRequest, AwaitEnvironmentReadyActivityResult, AwaitMaterializationRequest, AwaitMaterializationResult, ContextCompactActivityRequest, CreateOrLoadSessionRequest, CreateOrLoadSessionResult, EnvironmentJobCancelActivityRequest, - EnvironmentJobPollActivityRequest, EnvironmentJobPollActivityResult, - EnvironmentJobPrepareWorkflowToolRequest, EnvironmentJobStartActivityRequest, - EnvironmentJobStartActivityResult, JoinedContextPreparationRequest, LlmGenerateActivityRequest, - PutBlobRequest, ReadBlobRequest, ReadBlobResult, RuntimeProjectionRefreshActivityRequest, + EnvironmentJobCancelActivityResult, EnvironmentJobPollActivityRequest, + EnvironmentJobPollActivityResult, EnvironmentJobPrepareWorkflowToolRequest, + EnvironmentJobStartActivityRequest, EnvironmentJobStartActivityResult, + JoinedContextPreparationRequest, LlmGenerateActivityRequest, PutBlobRequest, ReadBlobRequest, + ReadBlobResult, RuntimeProjectionRefreshActivityRequest, RuntimeProjectionRefreshActivityResult, SubagentCloseActivityRequest, SubagentPrepareActivityRequest, SubagentPrepareActivityResult, SubagentResolveActivityRequest, ToolInvokeBatchActivityRequest, ToolInvokeCallActivityRequest, ToolInvokeCallActivityResult, @@ -270,7 +271,7 @@ impl WorkflowActivities { pub async fn environment_job_cancel( _ctx: ActivityContext, _request: EnvironmentJobCancelActivityRequest, - ) -> Result, ActivityError> { + ) -> Result { unimplemented!("workflow activity definition only") } diff --git a/crates/temporal-workflow/src/environment_jobs.rs b/crates/temporal-workflow/src/environment_jobs.rs new file mode 100644 index 000000000..09ab7b321 --- /dev/null +++ b/crates/temporal-workflow/src/environment_jobs.rs @@ -0,0 +1,495 @@ +//! Records and activity contracts of `EnvironmentJobWorkflow`. +//! +//! The workflow owns every type it persists in history (arguments, +//! continue-as-new state, local-activity results, signals and the start +//! payload). None of them is an environment protocol wire type: the job +//! activities map between these records and `job/*` at their boundary, so a +//! protocol change never changes a history shape. + +use std::collections::BTreeMap; + +use harness::{BlobRef, PromiseSourceCheckResult}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::WorkflowToolStartArgs; + +/// A job id on the environment's daemon (`jobId` on the wire). The start +/// activity validates it when it maps the payload onto `job/start`. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct EnvironmentJobId(String); + +impl EnvironmentJobId { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn into_string(self) -> String { + self.0 + } +} + +impl std::fmt::Display for EnvironmentJobId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// Lifecycle of one job as the workflow records it. Every daemon status has +/// a variant; `lost` is the workflow's own outcome for a job the daemon no +/// longer knows. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentJobStatus { + /// Waiting for dependencies or its queue. + Queued, + Running, + /// Exited with code 0. + Succeeded, + /// Exited otherwise, or could not start. + Failed, + /// Its timeout expired and the daemon stopped it. + TimedOut, + /// Stopped or never started because of a cancel. + Cancelled, + /// A dependency did not finish as the dependency policy requires. + DependencyFailed, + /// The daemon stopped while the job was queued or running. + Interrupted, + /// The daemon answered `notFound` for the job: it never existed there or + /// its record was garbage-collected. + Lost, +} + +impl EnvironmentJobStatus { + /// `queued` and `running` are live; every other status is final. + pub fn is_terminal(self) -> bool { + !matches!(self, Self::Queued | Self::Running) + } + + pub fn is_success(self) -> bool { + self == Self::Succeeded + } +} + +/// What the workflow keeps about one job. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentJobRecord { + pub job_id: EnvironmentJobId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub status: EnvironmentJobStatus, + /// Resolved ids of the jobs this one depends on. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub depends_on: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, +} + +impl EnvironmentJobRecord { + /// The record of a job the daemon answered `notFound` for and the + /// workflow holds nothing else about. + pub fn lost(job_id: EnvironmentJobId) -> Self { + Self { + job_id, + name: None, + status: EnvironmentJobStatus::Lost, + depends_on: Vec::new(), + queue_key: None, + } + } +} + +/// One job as a poll or cancel activity observed it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind", deny_unknown_fields)] +pub enum EnvironmentJobObservation { + /// The daemon's summary, mapped onto the workflow's record. + Job { record: EnvironmentJobRecord }, + /// The daemon answered this entry with `notFound`. The workflow records + /// the job as `lost`, a terminal failure. + NotFound { job_id: EnvironmentJobId }, +} + +impl EnvironmentJobObservation { + pub fn job_id(&self) -> &EnvironmentJobId { + match self { + Self::Job { record } => &record.job_id, + Self::NotFound { job_id } => job_id, + } + } +} + +/// Which jobs a cancel affects. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentJobCancelScope { + /// Only the named jobs. + #[default] + Job, + /// The named jobs and every job that transitively depends on them. + Dependents, +} + +/// Start payload of a job group, stored in CAS and named by +/// [`EnvironmentJobStartActivityRequest::request_ref`]. Written once by the +/// starter, read by the start activity, which resolves credential-bound +/// secret variables and maps it onto `job/start`. It never holds secret +/// values. Its content address is the stable fingerprint of the request, so +/// a starter derives the job-group id (and with it the workflow id) from the +/// encoded payload. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentJobStartPayloadV1 { + /// [`Self::VERSION`]. + pub version: u32, + /// Idempotency key of the start (`requestId` on the wire): a retried + /// start with the same id and jobs replays the existing jobs. + pub request_id: String, + /// Never empty. + pub jobs: Vec, +} + +impl EnvironmentJobStartPayloadV1 { + pub const VERSION: u32 = 1; + + pub fn new(request_id: impl Into, jobs: Vec) -> Self { + Self { + version: Self::VERSION, + request_id: request_id.into(), + jobs, + } + } + + /// The bytes a starter stores in CAS. + pub fn encode(&self) -> Result, serde_json::Error> { + serde_json::to_vec(self) + } + + /// Decode a stored payload, checking its version before its shape. + pub fn decode(bytes: &[u8]) -> Result { + #[derive(Deserialize)] + struct VersionProbe { + version: u32, + } + let probe: VersionProbe = + serde_json::from_slice(bytes).map_err(EnvironmentJobStartPayloadError::Decode)?; + if probe.version != Self::VERSION { + return Err(EnvironmentJobStartPayloadError::UnsupportedVersion { + version: probe.version, + }); + } + serde_json::from_slice(bytes).map_err(EnvironmentJobStartPayloadError::Decode) + } + + pub fn job_ids(&self) -> Vec { + self.jobs.iter().map(|job| job.job_id.clone()).collect() + } +} + +/// Why a stored start payload could not be read. +#[derive(Debug)] +pub enum EnvironmentJobStartPayloadError { + Decode(serde_json::Error), + UnsupportedVersion { version: u32 }, +} + +impl std::fmt::Display for EnvironmentJobStartPayloadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Decode(error) => write!(f, "decode environment job start payload: {error}"), + Self::UnsupportedVersion { version } => write!( + f, + "unsupported environment job start payload version {version} (expected {})", + EnvironmentJobStartPayloadV1::VERSION + ), + } + } +} + +impl std::error::Error for EnvironmentJobStartPayloadError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Decode(error) => Some(error), + Self::UnsupportedVersion { .. } => None, + } + } +} + +/// One job of a start payload. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentJobSpec { + pub job_id: EnvironmentJobId, + /// Local name other jobs of the same start can depend on. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub command: EnvironmentJobCommand, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub depends_on: Vec, + #[serde( + default, + skip_serializing_if = "EnvironmentJobDependencyPolicy::is_default" + )] + pub dependency_policy: EnvironmentJobDependencyPolicy, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, +} + +/// What a job runs. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentJobCommand { + /// Program and arguments. Not empty. + pub argv: Vec, + /// Absolute working directory in the environment; the daemon's default + /// when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cwd: Option, + /// Plain variables. Credential-bound variables are resolved by the start + /// activity and never stored here. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, + /// Text written to standard input, then end of file; `/dev/null` when + /// absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stdin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + +/// A dependency of a job. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind", deny_unknown_fields)] +pub enum EnvironmentJobDependency { + /// An existing job on the daemon, or one in the same start. + JobId { job_id: EnvironmentJobId }, + /// The `name` of a job in the same start. + Name { name: String }, +} + +/// What a job waits for from its dependencies. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentJobDependencyPolicy { + /// Run once every dependency succeeded; otherwise `dependency_failed`. + #[default] + AllSucceeded, + /// Run once every dependency is terminal, whatever its status. + AllTerminal, +} + +impl EnvironmentJobDependencyPolicy { + pub fn is_default(&self) -> bool { + *self == Self::AllSucceeded + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobStartActivityRequest { + pub universe_id: Uuid, + pub environment_id: String, + pub job_group_id: String, + /// CAS reference of the encoded [`EnvironmentJobStartPayloadV1`]. + pub request_ref: BlobRef, +} + +/// The started jobs, one record per payload job. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobStartActivityResult { + pub jobs: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobSubscription { + pub holder_workflow_id: String, + pub promise_id: String, + pub completion_key: String, + pub job_id: EnvironmentJobId, + #[serde(default)] + pub notified: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobWorkflowArgs { + pub universe_id: Uuid, + pub start: EnvironmentJobStartActivityRequest, + pub job_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub subscriptions: Vec, + #[serde(default)] + pub started: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub jobs: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub resolutions: BTreeMap, + #[serde(default = "default_environment_job_poll_ms")] + pub poll_ms: u64, + #[serde(default)] + pub poll_attempt: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workflow_tool: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobWorkflowToolContext { + pub execution_id: String, + pub invocation_id: harness::WorkflowToolInvocationId, +} + +/// Accepted inputs for the core job workflow. Bare public starts arrive as +/// fully prepared job arguments; internally supervised starts use the fixed +/// internally supervised input and are normalized by an activity before +/// provider work begins. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum EnvironmentJobWorkflowInput { + WorkflowTool(WorkflowToolStartArgs), + Job(EnvironmentJobWorkflowArgs), +} + +impl From for EnvironmentJobWorkflowInput { + fn from(value: EnvironmentJobWorkflowArgs) -> Self { + Self::Job(value) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobPrepareWorkflowToolRequest { + pub start: WorkflowToolStartArgs, +} + +fn default_environment_job_poll_ms() -> u64 { + 2_000 +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobWorkflowSnapshot { + pub environment_id: String, + pub job_group_id: String, + #[serde(default)] + pub started: bool, + #[serde(default)] + pub jobs: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub resolutions: BTreeMap, + /// Every job of the group has a terminal record and a recorded + /// resolution. + pub terminal: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, +} + +/// Poll of the group's jobs that are not settled yet. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobPollActivityRequest { + pub universe_id: Uuid, + pub environment_id: String, + pub job_group_id: String, + pub job_ids: Vec, +} + +/// One observation per polled job. `resolutions` is keyed by job id and +/// holds the promise result of every job observed terminal, `NotFound` +/// included (a `Failed` resolution carrying the model-facing result); the +/// workflow falls back to a `Failed` resolution without a payload for a +/// `NotFound` job that has none. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobPollActivityResult { + pub jobs: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub resolutions: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobCancelSignal { + pub jobs: Vec, + #[serde(default)] + pub scope: EnvironmentJobCancelScope, + #[serde(default)] + pub force: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobCancelActivityRequest { + pub universe_id: Uuid, + pub environment_id: String, + pub jobs: Vec, + pub scope: EnvironmentJobCancelScope, + pub force: bool, +} + +/// One observation per job the cancel listed (named jobs first, then +/// dependents). Observations of jobs outside the group are ignored. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct EnvironmentJobCancelActivityResult { + pub jobs: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_payload_round_trips_and_checks_its_version() { + let payload = EnvironmentJobStartPayloadV1::new( + "jobreq:1", + vec![EnvironmentJobSpec { + job_id: EnvironmentJobId::new("build"), + name: Some("build".to_owned()), + command: EnvironmentJobCommand { + argv: vec!["make".to_owned()], + cwd: Some("/workspace".to_owned()), + env: BTreeMap::new(), + stdin: None, + timeout_ms: Some(60_000), + }, + depends_on: vec![EnvironmentJobDependency::Name { + name: "fetch".to_owned(), + }], + dependency_policy: EnvironmentJobDependencyPolicy::AllTerminal, + queue_key: None, + }], + ); + let bytes = payload.encode().expect("encode payload"); + assert_eq!( + EnvironmentJobStartPayloadV1::decode(&bytes).expect("decode payload"), + payload + ); + + let mut future: serde_json::Value = serde_json::from_slice(&bytes).expect("json"); + future["version"] = serde_json::json!(2); + future["extra"] = serde_json::json!(true); + assert!(matches!( + EnvironmentJobStartPayloadV1::decode(&serde_json::to_vec(&future).expect("encode")), + Err(EnvironmentJobStartPayloadError::UnsupportedVersion { version: 2 }) + )); + } + + #[test] + fn only_live_statuses_are_not_terminal() { + use EnvironmentJobStatus::*; + for status in [Queued, Running] { + assert!(!status.is_terminal()); + } + for status in [ + Succeeded, + Failed, + TimedOut, + Cancelled, + DependencyFailed, + Interrupted, + Lost, + ] { + assert!(status.is_terminal()); + } + assert!(Succeeded.is_success()); + assert!(!Lost.is_success()); + } +} diff --git a/crates/temporal-workflow/src/lib.rs b/crates/temporal-workflow/src/lib.rs index 57ce0f75d..cef7a3453 100644 --- a/crates/temporal-workflow/src/lib.rs +++ b/crates/temporal-workflow/src/lib.rs @@ -5,6 +5,18 @@ mod code_execution; mod code_tools; pub use code_tools::*; mod config; +mod environment_jobs; +pub use environment_jobs::{ + EnvironmentJobCancelActivityRequest, EnvironmentJobCancelActivityResult, + EnvironmentJobCancelScope, EnvironmentJobCancelSignal, EnvironmentJobCommand, + EnvironmentJobDependency, EnvironmentJobDependencyPolicy, EnvironmentJobId, + EnvironmentJobObservation, EnvironmentJobPollActivityRequest, EnvironmentJobPollActivityResult, + EnvironmentJobPrepareWorkflowToolRequest, EnvironmentJobRecord, EnvironmentJobSpec, + EnvironmentJobStartActivityRequest, EnvironmentJobStartActivityResult, + EnvironmentJobStartPayloadError, EnvironmentJobStartPayloadV1, EnvironmentJobStatus, + EnvironmentJobSubscription, EnvironmentJobWorkflowArgs, EnvironmentJobWorkflowInput, + EnvironmentJobWorkflowSnapshot, EnvironmentJobWorkflowToolContext, +}; mod rehydrate; mod session_preparation; mod temporal_helpers; @@ -62,28 +74,22 @@ pub use types::{ AwaitEnvironmentReadyActivityRequest, AwaitEnvironmentReadyActivityResult, AwaitMaterializationRequest, AwaitMaterializationResult, AwaitOutcome, AwaitPromiseResult, CancellingWatchdog, ContextCompactActivityRequest, CreateOrLoadSessionRequest, - CreateOrLoadSessionResult, EnvironmentJobCancelActivityRequest, EnvironmentJobCancelSignal, - EnvironmentJobPollActivityRequest, EnvironmentJobPollActivityResult, - EnvironmentJobPrepareWorkflowToolRequest, EnvironmentJobStartActivityRequest, - EnvironmentJobStartActivityResult, EnvironmentJobStartPayload, EnvironmentJobSubscription, - EnvironmentJobWorkflowArgs, EnvironmentJobWorkflowInput, EnvironmentJobWorkflowSnapshot, - EnvironmentJobWorkflowToolContext, JoinedContextPreparationRequest, - LLM_PROVIDER_TRANSIENT_ERROR_TYPE, LLM_TRANSIENT_FAILURE_DETAILS_VERSION, - LlmGenerateActivityRequest, LlmTransientFailureDetails, PendingEmission, - PendingPromiseCancellation, PendingSourceResolution, PendingToolBatchResume, PromiseSourcePoll, - PutBlobRequest, ReadBlobRequest, ReadBlobResult, RuntimeProjectionRefreshActivityRequest, - RuntimeProjectionRefreshActivityResult, SessionBootstrapPayloadTooLarge, SubagentChildRef, - SubagentCloseActivityRequest, SubagentExecutionPhase, SubagentExecutionSnapshot, - SubagentPrepareActivityRequest, SubagentPrepareActivityResult, SubagentResolveActivityRequest, - SubagentTerminal, ToolInvokeBatchActivityRequest, ToolInvokeCallActivityRequest, - ToolInvokeCallActivityResult, ToolPreparePromiseControlsActivityRequest, - WORKFLOW_TOOL_RECIPE_FINGERPRINT_PREFIX, WORKFLOW_TOOL_RECIPE_FORMAT_V1, - WORKFLOW_TOOL_RECOVERY_QUERY, WorkflowToolExecutionCancelRequest, - WorkflowToolExecutionCheckRequest, WorkflowToolRecipeV1, WorkflowToolRecoveryResult, - WorkflowToolReplyValidationRequest, WorkflowToolReplyValidationResult, - WorkflowToolStartActivityRequest, WorkflowToolStartActivityResult, WorkflowToolStartArgs, - compose_environment_job_workflow_id, compose_workflow_id, split_workflow_id, - workflow_tool_recipe_fingerprint, + CreateOrLoadSessionResult, JoinedContextPreparationRequest, LLM_PROVIDER_TRANSIENT_ERROR_TYPE, + LLM_TRANSIENT_FAILURE_DETAILS_VERSION, LlmGenerateActivityRequest, LlmTransientFailureDetails, + PendingEmission, PendingPromiseCancellation, PendingSourceResolution, PendingToolBatchResume, + PromiseSourcePoll, PutBlobRequest, ReadBlobRequest, ReadBlobResult, + RuntimeProjectionRefreshActivityRequest, RuntimeProjectionRefreshActivityResult, + SessionBootstrapPayloadTooLarge, SubagentChildRef, SubagentCloseActivityRequest, + SubagentExecutionPhase, SubagentExecutionSnapshot, SubagentPrepareActivityRequest, + SubagentPrepareActivityResult, SubagentResolveActivityRequest, SubagentTerminal, + ToolInvokeBatchActivityRequest, ToolInvokeCallActivityRequest, ToolInvokeCallActivityResult, + ToolPreparePromiseControlsActivityRequest, WORKFLOW_TOOL_RECIPE_FINGERPRINT_PREFIX, + WORKFLOW_TOOL_RECIPE_FORMAT_V1, WORKFLOW_TOOL_RECOVERY_QUERY, + WorkflowToolExecutionCancelRequest, WorkflowToolExecutionCheckRequest, WorkflowToolRecipeV1, + WorkflowToolRecoveryResult, WorkflowToolReplyValidationRequest, + WorkflowToolReplyValidationResult, WorkflowToolStartActivityRequest, + WorkflowToolStartActivityResult, WorkflowToolStartArgs, compose_environment_job_workflow_id, + compose_workflow_id, split_workflow_id, workflow_tool_recipe_fingerprint, }; pub use workflows::bots; pub use workflows::channels; diff --git a/crates/temporal-workflow/src/types.rs b/crates/temporal-workflow/src/types.rs index 1994cc1ee..d24f73735 100644 --- a/crates/temporal-workflow/src/types.rs +++ b/crates/temporal-workflow/src/types.rs @@ -400,79 +400,6 @@ pub struct CancellingWatchdog { pub since_ms: u64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobStartActivityRequest { - pub universe_id: Uuid, - pub environment_id: String, - pub job_group_id: String, - pub request_ref: BlobRef, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobStartPayload { - pub request: environment_protocol::data::jobs::StartJobsParams, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobStartActivityResult { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobSubscription { - pub holder_workflow_id: String, - pub promise_id: String, - pub completion_key: String, - pub job_id: environment_protocol::shared::JobId, - #[serde(default)] - pub notified: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobWorkflowArgs { - pub universe_id: Uuid, - pub start: EnvironmentJobStartActivityRequest, - pub job_ids: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub subscriptions: Vec, - #[serde(default)] - pub started: bool, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub jobs: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub resolutions: BTreeMap, - #[serde(default = "default_environment_job_poll_ms")] - pub poll_ms: u64, - #[serde(default)] - pub poll_attempt: u32, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub workflow_tool: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobWorkflowToolContext { - pub execution_id: String, - pub invocation_id: harness::WorkflowToolInvocationId, -} - -/// Accepted inputs for the core job workflow. Bare public starts arrive as -/// fully prepared job arguments; internally supervised starts use the fixed -/// internally supervised input and are normalized by an activity before -/// provider work begins. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum EnvironmentJobWorkflowInput { - WorkflowTool(WorkflowToolStartArgs), - Job(EnvironmentJobWorkflowArgs), -} - -impl From for EnvironmentJobWorkflowInput { - fn from(value: EnvironmentJobWorkflowArgs) -> Self { - Self::Job(value) - } -} - -/// Sub-agent execution: one delegation supervised by /// `SubagentExecutionWorkflow`, started on call by the parent session. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct SubagentPrepareActivityRequest { @@ -547,62 +474,6 @@ pub struct SubagentExecutionSnapshot { pub resolution: Option, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobPrepareWorkflowToolRequest { - pub start: WorkflowToolStartArgs, -} - -fn default_environment_job_poll_ms() -> u64 { - 2_000 -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobWorkflowSnapshot { - pub environment_id: String, - pub job_group_id: String, - #[serde(default)] - pub started: bool, - #[serde(default)] - pub jobs: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub resolutions: BTreeMap, - pub terminal: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub last_error: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobPollActivityRequest { - pub universe_id: Uuid, - pub environment_id: String, - pub job_group_id: String, - pub job_ids: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobPollActivityResult { - pub jobs: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub resolutions: BTreeMap, - pub terminal: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobCancelSignal { - pub jobs: Vec, - pub scope: environment_protocol::data::jobs::JobCancelScope, - pub force: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct EnvironmentJobCancelActivityRequest { - pub universe_id: Uuid, - pub environment_id: String, - pub jobs: Vec, - pub scope: environment_protocol::data::jobs::JobCancelScope, - pub force: bool, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct CreateOrLoadSessionRequest { pub session_id: SessionId, diff --git a/crates/temporal-workflow/src/workflows/environment_job.rs b/crates/temporal-workflow/src/workflows/environment_job.rs index d948e7fb5..79927389d 100644 --- a/crates/temporal-workflow/src/workflows/environment_job.rs +++ b/crates/temporal-workflow/src/workflows/environment_job.rs @@ -10,10 +10,12 @@ use temporalio_sdk::{ use crate::{ AgentSessionWorkflow, DEFAULT_CONTINUE_AS_NEW_HISTORY_THRESHOLD, - EnvironmentJobCancelActivityRequest, EnvironmentJobCancelSignal, - EnvironmentJobPollActivityRequest, EnvironmentJobSubscription, EnvironmentJobWorkflowInput, - EnvironmentJobWorkflowSnapshot, EnvironmentJobWorkflowToolContext, WorkflowActivities, - WorkflowToolRecoveryResult, compose_environment_job_workflow_id, + EnvironmentJobCancelActivityRequest, EnvironmentJobCancelScope, EnvironmentJobCancelSignal, + EnvironmentJobId, EnvironmentJobObservation, EnvironmentJobPollActivityRequest, + EnvironmentJobPollActivityResult, EnvironmentJobRecord, EnvironmentJobStatus, + EnvironmentJobSubscription, EnvironmentJobWorkflowInput, EnvironmentJobWorkflowSnapshot, + EnvironmentJobWorkflowToolContext, WorkflowActivities, WorkflowToolRecoveryResult, + compose_environment_job_workflow_id, }; #[workflow] @@ -82,6 +84,7 @@ impl EnvironmentJobWorkflow { state.snapshot.started = args.started; state.snapshot.jobs = args.jobs.clone(); state.snapshot.resolutions = args.resolutions.clone(); + refresh_terminal(&mut state.snapshot, &args.job_ids); state.workflow_tool = args.workflow_tool.clone(); for subscription in args.subscriptions.drain(..) { if !state.subscriptions.iter().any(|existing| { @@ -137,17 +140,9 @@ impl EnvironmentJobWorkflow { ) .await { - Ok(jobs) => ctx.state_mut(|state| { - for job in jobs { - if let Some(existing) = state - .snapshot - .jobs - .iter_mut() - .find(|existing| existing.job_id == job.job_id) - { - *existing = job; - } - } + Ok(result) => ctx.state_mut(|state| { + apply_observations(&mut state.snapshot, &args.job_ids, result.jobs); + refresh_terminal(&mut state.snapshot, &args.job_ids); state.snapshot.last_error = None; }), Err(error) => { @@ -156,7 +151,8 @@ impl EnvironmentJobWorkflow { } } - if !ctx.state(|state| state.snapshot.terminal) { + let unsettled = ctx.state(|state| unsettled_job_ids(&state.snapshot, &args.job_ids)); + if !unsettled.is_empty() { match ctx .execute_local_activity( WorkflowActivities::environment_job_poll, @@ -164,7 +160,7 @@ impl EnvironmentJobWorkflow { universe_id: args.universe_id, environment_id: args.start.environment_id.clone(), job_group_id: args.start.job_group_id.clone(), - job_ids: args.job_ids.clone(), + job_ids: unsettled, }, environment_job_activity_options(), ) @@ -172,9 +168,7 @@ impl EnvironmentJobWorkflow { { Ok(result) => { ctx.state_mut(|state| { - state.snapshot.jobs = result.jobs; - state.snapshot.resolutions.extend(result.resolutions); - state.snapshot.terminal = result.terminal; + apply_poll_result(&mut state.snapshot, &args.job_ids, result); state.snapshot.last_error = None; }); } @@ -298,7 +292,7 @@ async fn cancel_workflow_jobs( universe_id: args.universe_id, environment_id: args.start.environment_id.clone(), jobs, - scope: environment_protocol::data::jobs::JobCancelScope::Job, + scope: EnvironmentJobCancelScope::Job, force: false, }, environment_job_activity_options(), @@ -306,6 +300,86 @@ async fn cancel_workflow_jobs( .await; } +/// Apply a poll: record the resolutions, then the observations. A job the +/// daemon answered `notFound` for is `lost`; when the activity returned no +/// resolution for it, the job still settles as a failure without a payload, +/// so its subscribers are released. +fn apply_poll_result( + snapshot: &mut EnvironmentJobWorkflowSnapshot, + job_ids: &[EnvironmentJobId], + result: EnvironmentJobPollActivityResult, +) { + snapshot.resolutions.extend( + result + .resolutions + .into_iter() + .filter(|(job_id, _)| job_ids.iter().any(|id| id.as_str() == job_id)), + ); + for observation in &result.jobs { + if let EnvironmentJobObservation::NotFound { job_id } = observation + && job_ids.contains(job_id) + { + snapshot + .resolutions + .entry(job_id.as_str().to_owned()) + .or_insert(harness::PromiseSourceCheckResult::Failed { error_ref: None }); + } + } + apply_observations(snapshot, job_ids, result.jobs); + refresh_terminal(snapshot, job_ids); +} + +/// A summary replaces the job's record; `notFound` marks the job `lost`, +/// keeping what the workflow already knew about it. Observations of jobs +/// outside the group (such as dependents a cancel reached) are ignored. +fn apply_observations( + snapshot: &mut EnvironmentJobWorkflowSnapshot, + job_ids: &[EnvironmentJobId], + observations: Vec, +) { + for observation in observations { + if !job_ids.contains(observation.job_id()) { + continue; + } + let existing = snapshot + .jobs + .iter_mut() + .find(|existing| &existing.job_id == observation.job_id()); + match (observation, existing) { + (EnvironmentJobObservation::Job { record }, Some(existing)) => *existing = record, + (EnvironmentJobObservation::Job { record }, None) => snapshot.jobs.push(record), + (EnvironmentJobObservation::NotFound { .. }, Some(existing)) => { + existing.status = EnvironmentJobStatus::Lost; + } + (EnvironmentJobObservation::NotFound { job_id }, None) => { + snapshot.jobs.push(EnvironmentJobRecord::lost(job_id)); + } + } + } +} + +/// Jobs of the group without both a terminal record and a resolution. +fn unsettled_job_ids( + snapshot: &EnvironmentJobWorkflowSnapshot, + job_ids: &[EnvironmentJobId], +) -> Vec { + job_ids + .iter() + .filter(|job_id| { + let terminal = snapshot + .jobs + .iter() + .any(|job| &job.job_id == *job_id && job.status.is_terminal()); + !(terminal && snapshot.resolutions.contains_key(job_id.as_str())) + }) + .cloned() + .collect() +} + +fn refresh_terminal(snapshot: &mut EnvironmentJobWorkflowSnapshot, job_ids: &[EnvironmentJobId]) { + snapshot.terminal = !job_ids.is_empty() && unsettled_job_ids(snapshot, job_ids).is_empty(); +} + /// Environment jobs deliberately remain co-located with the core environment /// registry and environment adapter. Their short, idempotent adapter calls therefore /// run as local activities: Temporal still records completion and retries, but @@ -358,7 +432,7 @@ fn queue_workflow_tool_cancellation( { state.pending_cancels.push(EnvironmentJobCancelSignal { jobs: vec![subscription.job_id.clone()], - scope: environment_protocol::data::jobs::JobCancelScope::Job, + scope: EnvironmentJobCancelScope::Job, force: false, }); } @@ -456,7 +530,8 @@ fn workflow_tool_recovery(state: &EnvironmentJobWorkflow) -> WorkflowToolRecover #[cfg(test)] mod tests { - use environment_protocol::shared::JobId; + use std::collections::BTreeMap; + use harness::{BlobRef, PromiseSourceCheckResult}; use super::*; @@ -466,7 +541,7 @@ mod tests { holder_workflow_id: "universe/session_1".to_owned(), promise_id: "promise_1".to_owned(), completion_key: "job_1".to_owned(), - job_id: JobId::new("job_1"), + job_id: EnvironmentJobId::new("job_1"), notified: false, } } @@ -515,6 +590,143 @@ mod tests { assert!(second.is_empty()); } + fn record(job_id: &str, status: EnvironmentJobStatus) -> EnvironmentJobRecord { + EnvironmentJobRecord { + job_id: EnvironmentJobId::new(job_id), + name: Some(job_id.to_owned()), + status, + depends_on: Vec::new(), + queue_key: None, + } + } + + #[test] + fn not_found_settles_the_job_as_lost() { + let job_ids = vec![ + EnvironmentJobId::new("job_1"), + EnvironmentJobId::new("job_2"), + ]; + let mut snapshot = EnvironmentJobWorkflowSnapshot { + started: true, + jobs: vec![ + record("job_1", EnvironmentJobStatus::Running), + record("job_2", EnvironmentJobStatus::Queued), + ], + ..Default::default() + }; + assert_eq!(unsettled_job_ids(&snapshot, &job_ids), job_ids); + + let done = BlobRef::from_bytes(b"done"); + apply_poll_result( + &mut snapshot, + &job_ids, + EnvironmentJobPollActivityResult { + jobs: vec![ + EnvironmentJobObservation::Job { + record: record("job_1", EnvironmentJobStatus::Succeeded), + }, + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("job_2"), + }, + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("elsewhere"), + }, + ], + resolutions: BTreeMap::from([( + "job_1".to_owned(), + PromiseSourceCheckResult::Resolved { + payload_ref: Some(done.clone()), + }, + )]), + }, + ); + + assert!(snapshot.terminal); + assert!(unsettled_job_ids(&snapshot, &job_ids).is_empty()); + assert_eq!(snapshot.jobs.len(), 2); + assert_eq!(snapshot.jobs[1].status, EnvironmentJobStatus::Lost); + // The record keeps what the workflow knew before the job was lost. + assert_eq!(snapshot.jobs[1].name.as_deref(), Some("job_2")); + assert_eq!( + snapshot.resolutions.get("job_2"), + Some(&PromiseSourceCheckResult::Failed { error_ref: None }) + ); + assert!(!snapshot.resolutions.contains_key("elsewhere")); + } + + #[test] + fn activity_resolution_of_a_lost_job_wins_and_running_jobs_stay_unsettled() { + let job_ids = vec![ + EnvironmentJobId::new("job_1"), + EnvironmentJobId::new("job_2"), + ]; + let mut snapshot = EnvironmentJobWorkflowSnapshot::default(); + let lost = BlobRef::from_bytes(b"lost"); + apply_poll_result( + &mut snapshot, + &job_ids, + EnvironmentJobPollActivityResult { + jobs: vec![ + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("job_1"), + }, + EnvironmentJobObservation::Job { + record: record("job_2", EnvironmentJobStatus::Running), + }, + ], + resolutions: BTreeMap::from([( + "job_1".to_owned(), + PromiseSourceCheckResult::Failed { + error_ref: Some(lost.clone()), + }, + )]), + }, + ); + + assert!(!snapshot.terminal); + assert_eq!( + snapshot.jobs[0], + EnvironmentJobRecord::lost(EnvironmentJobId::new("job_1")) + ); + assert_eq!( + snapshot.resolutions.get("job_1"), + Some(&PromiseSourceCheckResult::Failed { + error_ref: Some(lost) + }) + ); + assert_eq!( + unsettled_job_ids(&snapshot, &job_ids), + vec![EnvironmentJobId::new("job_2")] + ); + } + + #[test] + fn cancel_not_found_marks_lost_but_waits_for_the_poll_resolution() { + let job_ids = vec![EnvironmentJobId::new("job_1")]; + let mut snapshot = EnvironmentJobWorkflowSnapshot { + jobs: vec![record("job_1", EnvironmentJobStatus::Running)], + ..Default::default() + }; + apply_observations( + &mut snapshot, + &job_ids, + vec![ + EnvironmentJobObservation::NotFound { + job_id: EnvironmentJobId::new("job_1"), + }, + EnvironmentJobObservation::Job { + record: record("dependent", EnvironmentJobStatus::Cancelled), + }, + ], + ); + refresh_terminal(&mut snapshot, &job_ids); + + assert_eq!(snapshot.jobs.len(), 1); + assert_eq!(snapshot.jobs[0].status, EnvironmentJobStatus::Lost); + assert!(!snapshot.terminal); + assert_eq!(unsettled_job_ids(&snapshot, &job_ids), job_ids); + } + #[test] fn workflow_tool_cancellation_targets_only_the_matching_job() { let invocation_id = @@ -545,7 +757,10 @@ mod tests { ); assert_eq!(workflow.pending_cancels.len(), 1); - assert_eq!(workflow.pending_cancels[0].jobs, vec![JobId::new("job_1")]); + assert_eq!( + workflow.pending_cancels[0].jobs, + vec![EnvironmentJobId::new("job_1")] + ); assert!(workflow.nudged); queue_workflow_tool_cancellation( diff --git a/crates/temporal-workflow/src/workflows/session/preparation.rs b/crates/temporal-workflow/src/workflows/session/preparation.rs index c84d39480..f9d6d6c87 100644 --- a/crates/temporal-workflow/src/workflows/session/preparation.rs +++ b/crates/temporal-workflow/src/workflows/session/preparation.rs @@ -380,8 +380,8 @@ async fn prepare_operation( "session configuration revision changed", )); } - summary.config_changed = drive.state().lifecycle.config.as_ref() != Some(&config); configure(ctx, drive, &mut candidate, config, expected_revision).await?; + summary.config_changed = config_changed(drive, &candidate); } SessionOperation::ApplyProfile { profile, @@ -418,6 +418,14 @@ async fn prepare_operation( Ok((candidate, summary)) } +/// Whether the candidate's admitted configuration differs from the live one. +/// Admission materializes defaults (such as the compaction policy) into the +/// requested document, so comparing the request itself would report a +/// change for a replacement the harness admits as a no-op. +fn config_changed(live: &CoreAgentDrive, candidate: &PreparationCandidate) -> bool { + live.state().lifecycle.config != candidate.state().lifecycle.config +} + async fn configure( ctx: &mut WorkflowContext, drive: &mut CoreAgentDrive, @@ -504,7 +512,6 @@ async fn apply_profile( workflow_time_ms(ctx), )?; if profile.config.is_some() { - summary.config_changed = drive.state().lifecycle.config.as_ref() != Some(&config); candidate.push( CoreAgentCommand::ReplaceSessionConfig { expected_revision, @@ -512,6 +519,7 @@ async fn apply_profile( }, workflow_time_ms(ctx), )?; + summary.config_changed = config_changed(drive, candidate); } // The default fills an empty pointer and never overrides a live // selection. The configuration replacement above has already cleared a @@ -690,6 +698,68 @@ pub(super) fn abandon_pending_run(state: &mut AgentSessionWorkflow) { mod tests { use super::*; + /// A live session opened with the default configuration, whose admission + /// materialized the default compaction policy. + fn opened_live() -> CoreAgentDrive { + let empty = CoreAgentDrive::from_replayed( + SessionId::new("config-changed"), + CoreAgentState::new(), + None, + ); + let mut opening = PreparationCandidate::new(&empty); + opening + .push( + CoreAgentCommand::OpenSession { + config: crate::default_session_config(harness::ModelSelection { + api_kind: harness::ProviderApiKind::OpenAiResponses, + provider_id: "openai".into(), + model: "gpt-test".into(), + }), + }, + 1, + ) + .unwrap(); + let state = opening.state().clone(); + let head = state.reduced_to.clone(); + CoreAgentDrive::from_replayed(empty.session_id().clone(), state, head) + } + + #[test] + fn config_change_compares_the_admitted_configuration() { + let live = opened_live(); + let current = live.state().lifecycle.config.clone().unwrap(); + assert!(current.context.compaction.is_some()); + // A profile document omits the defaults admission fills in: the same + // configuration is no change. + let mut same = current.clone(); + same.context.compaction = None; + let mut candidate = PreparationCandidate::new(&live); + candidate + .push( + CoreAgentCommand::ReplaceSessionConfig { + expected_revision: None, + config: same, + }, + 2, + ) + .unwrap(); + assert!(!config_changed(&live, &candidate)); + + let mut other = current; + other.model.model = "gpt-other".into(); + let mut candidate = PreparationCandidate::new(&live); + candidate + .push( + CoreAgentCommand::ReplaceSessionConfig { + expected_revision: None, + config: other, + }, + 2, + ) + .unwrap(); + assert!(config_changed(&live, &candidate)); + } + fn pending() -> PendingRunPreparation { let config = crate::default_session_config(harness::ModelSelection { api_kind: harness::ProviderApiKind::OpenAiResponses, diff --git a/crates/tools/Cargo.toml b/crates/tools/Cargo.toml index 7428e7895..0d2da8f64 100644 --- a/crates/tools/Cargo.toml +++ b/crates/tools/Cargo.toml @@ -8,9 +8,10 @@ api = { path = "../api" } async-trait = "0.1" harness = { path = "../harness", features = ["contract"] } futures-util = "0.3" -glob = "0.3" +globset = "0.4" +ignore = "0.4" environment-client = { path = "../environment-client" } -environment-protocol = { path = "../environment-protocol", features = ["schema"] } +environment-protocol = { path = "../environment-protocol" } jsonschema = { version = "0.58", default-features = false } regex = "1" schemars = "1" diff --git a/crates/tools/src/builtin/canonical.rs b/crates/tools/src/builtin/canonical.rs index c3087217d..db334315b 100644 --- a/crates/tools/src/builtin/canonical.rs +++ b/crates/tools/src/builtin/canonical.rs @@ -498,7 +498,7 @@ fn job_submit_schema() -> Value { "job_id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$", - "description": "Your name for the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read." + "description": "Your name for this run of the job (for example \"build\"). It keys the job's promise in the result and identifies the job to job_read. An id names one run: to run a command again, choose a new id." }, "argv": { "type": "array", "items": { "type": "string" } }, "cwd": { "type": ["string", "null"] }, @@ -578,9 +578,16 @@ fn job_read_schema() -> Value { "type": "object", "properties": { "jobs": { "type": "array", "items": job_handle_schema() }, - "output_bytes": { "type": ["integer", "null"], "minimum": 0 }, - "after_seq": { "type": ["integer", "null"], "minimum": 0 }, - "include_artifacts": { "type": "boolean" } + "output_bytes": { + "type": ["integer", "null"], + "minimum": 0, + "description": "Output bytes to show per job. Without `after`, half come from the start of the output and half from its end." + }, + "after": { + "type": ["integer", "null"], + "minimum": 0, + "description": "Byte offset to read forward from, such as a result's `output_next` or its omitted range's `offset`." + } }, "required": ["jobs"], "additionalProperties": false, diff --git a/crates/tools/src/builtin/output_schema_tests.rs b/crates/tools/src/builtin/output_schema_tests.rs index 776aaf8d3..520f2ac58 100644 --- a/crates/tools/src/builtin/output_schema_tests.rs +++ b/crates/tools/src/builtin/output_schema_tests.rs @@ -289,24 +289,22 @@ async fn process_schema_requires_one_typed_encoding_per_stream() { fn durable_job_and_agent_results_preserve_their_exact_serialized_names() { use crate::environment::jobs::{ JobHandle, JobSubmitResult, JobSubmitted, ModelJobOutputSegment, ModelJobResult, - ModelJobResultSet, + ModelJobResultSet, ModelJobStatus, ModelJobSummary, ModelOutputOmission, ModelOutputStream, + ModelStoppedBy, }; use crate::subagents::{SubagentResultEnvelope, SubagentResultStatus, SubagentToolKind}; - use environment_protocol::{ - data::jobs::{JobArtifact, JobOutputStream, JobStatus, JobSummary}, - shared::{EnvironmentPath, JobId}, - }; + use environment_protocol::ids::JobId; let handle = JobHandle { environment_id: "env-1".into(), - job_id: JobId::new("build"), + job_id: JobId::new("build").unwrap(), }; let submitted = JobSubmitResult { jobs: vec![JobSubmitted { name: Some("Build".into()), job_id: handle.job_id.clone(), handle: Some(handle.clone()), - status: JobStatus::Queued, + status: ModelJobStatus::Queued, dependencies: vec![], queue_key: None, promise: Some("promise_1".into()), @@ -319,35 +317,48 @@ fn durable_job_and_agent_results_preserve_their_exact_serialized_names() { .expect("submitted jobs"); let result = ModelJobResult { handle: Some(handle), - summary: Some(JobSummary { - namespace: "session".into(), - job_id: JobId::new("build"), + summary: Some(ModelJobSummary { + job_id: JobId::new("build").unwrap(), name: None, - status: JobStatus::TimedOut, - dependencies: vec![], + status: ModelJobStatus::TimedOut, + depends_on: vec![], + queue_key: None, created_at_ms: 1, - queued_at_ms: None, started_at_ms: Some(2), finished_at_ms: Some(3), + duration_ms: Some(1), exit_code: None, - orphaned_descendants: true, - failure: None, - queue_key: None, + signal: Some("SIGTERM".into()), + core_dumped: false, + stopped_by: Some(ModelStoppedBy::Timeout), + swept_processes: vec![crate::environment::process::LeftoverProcess { + pid: 7, + command: "sleep 100".into(), + }], + error: None, }), output: vec![ModelJobOutputSegment { - stream: JobOutputStream::Stdout, + stream: ModelOutputStream::Stdout, + text: Some("start".into()), + blob_ref: None, + media_type: None, + byte_len: None, + }], + omitted: Some(ModelOutputOmission { + offset: 5, + bytes: 90, + lost_bytes: 0, + }), + tail: vec![ModelJobOutputSegment { + stream: ModelOutputStream::Stderr, text: Some("done".into()), blob_ref: None, media_type: None, byte_len: Some(4), }], - output_next_seq: 9, + output_next: 99, + output_end: 99, truncated: false, - artifacts: vec![JobArtifact { - path: EnvironmentPath::new("/out").unwrap(), - kind: None, - metadata: Default::default(), - }], error: None, }; let schema = validator(BuiltinTool::environment_canonical( @@ -355,8 +366,8 @@ fn durable_job_and_agent_results_preserve_their_exact_serialized_names() { )); schema.validate(&serialized(&result)).expect("joined job"); let mut invalid = serialized(&result); - invalid["summary"]["status"] = json!("timed_out"); - assert!(!schema.is_valid(&invalid), "job statuses use camelCase"); + invalid["summary"]["status"] = json!("timedOut"); + assert!(!schema.is_valid(&invalid), "job statuses use snake_case"); validator(BuiltinTool::environment_canonical( BuiltinToolOperation::JobRead, )) diff --git a/crates/tools/src/builtin/shared.rs b/crates/tools/src/builtin/shared.rs index bb57517e1..cd4bb2b66 100644 --- a/crates/tools/src/builtin/shared.rs +++ b/crates/tools/src/builtin/shared.rs @@ -31,6 +31,9 @@ pub(super) fn visible_with_search_stop( crate::fs::FsSearchStop::TimeLimit => { "[truncated: time budget exhausted — narrow the path or add an include filter]" } + crate::fs::FsSearchStop::ResponseLimit => { + "[truncated: response size limit reached — narrow the pattern or the path]" + } }; if !visible.is_empty() { visible.push('\n'); diff --git a/crates/tools/src/environment.rs b/crates/tools/src/environment.rs index c2ca34c59..248c871bd 100644 --- a/crates/tools/src/environment.rs +++ b/crates/tools/src/environment.rs @@ -15,6 +15,7 @@ pub(crate) mod catalog_text; pub mod control; pub mod handles; pub mod jobs; +pub mod output; pub mod process; pub mod projection; pub mod sources; diff --git a/crates/tools/src/environment/control.rs b/crates/tools/src/environment/control.rs index 9145bf382..d2513d948 100644 --- a/crates/tools/src/environment/control.rs +++ b/crates/tools/src/environment/control.rs @@ -35,14 +35,12 @@ pub struct EnvironmentDeactivateArgs {} #[derive(Clone, Debug, Serialize, Deserialize, schemars::JsonSchema)] pub struct EnvironmentModelView { pub environment_id: String, - pub provider_id: Option, pub display_name: Option, pub status: Option, pub access: String, pub default: bool, pub working_directory: Option, pub active: bool, - pub observed_at_ms: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub status_message: Option, } diff --git a/crates/tools/src/environment/jobs.rs b/crates/tools/src/environment/jobs.rs index 4662f0923..4d68531d4 100644 --- a/crates/tools/src/environment/jobs.rs +++ b/crates/tools/src/environment/jobs.rs @@ -1,16 +1,26 @@ //! Durable environment job capability boundary. +//! +//! The model-facing argument and result types here are this crate's own: +//! they map to and from the environment protocol's job wire types at this +//! boundary, so a protocol change never silently reshapes what the model +//! reads or writes. use std::collections::BTreeMap; use async_trait::async_trait; use environment_protocol::{ - data::jobs::{ - CancelJobsParams, CancelJobsResponse, JobArtifact, JobCancelScope, - JobDependency as ProtocolJobDependency, JobDependencyPolicy, JobOutputChunk, JobStartSpec, - JobStatus, JobSummary, ReadJobsParams, ReadJobsResponse, StartJobsParams, - StartJobsResponse, + bytes::ByteChunk, + data::{ + exec::{CommandSpec, ExecInput, ExitStatus, StoppedBy}, + handshake::JobLimits, + job::{ + CancelJobsParams, CancelJobsResponse, JobDependency, JobDependencyPolicy, JobReadEntry, + JobReadRequest, JobSpec, JobStatus, JobSummary, ReadJobsParams, ReadJobsResponse, + StartJobsParams, StartJobsResponse, + }, }, - shared::{ByteChunk, EnvironmentPath, JobId}, + ids::{JobId, JobName, QueueKey}, + paths::EnvironmentPath, }; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -20,7 +30,13 @@ use harness::{ storage::{BlobEdge, BlobGraphStore, BlobStore, BlobStoreError}, }; -use crate::fs::FsPath; +use crate::{ + environment::{ + output::{ShownOutput, is_stderr}, + process::LeftoverProcess, + }, + fs::FsPath, +}; pub const JOB_SUBMIT_TOOL_NAME: &str = "job_submit"; pub const JOB_RUN_TOOL_NAME: &str = "job_run"; @@ -32,11 +48,19 @@ pub const JOB_RUN_WORKFLOW_SEMANTIC_TYPE: &str = "lightspeed.environment.job.run pub const JOB_RUN_DEFAULT_TIMEOUT_MS: u64 = 30 * 60 * 1_000; pub const JOB_RUN_MAX_TIMEOUT_MS: u64 = 60 * 60 * 1_000; pub const JOB_RUN_DEADLINE_AFTER_MS: u64 = 65 * 60 * 1_000; +/// Output a job result shows when the reader names no budget: the first +/// half from the start of the output, the second half from its end. +pub const DEFAULT_JOB_OUTPUT_BYTES: u64 = 16 * 1024; pub type JobExecResult = Result; #[async_trait] pub trait JobExecutor: Send + Sync { + /// The endpoint's job ceilings, when known; reads stay within them. + fn limits(&self) -> Option { + None + } + async fn start_jobs(&self, request: StartJobsParams) -> JobExecResult; async fn read_jobs(&self, request: ReadJobsParams) -> JobExecResult; @@ -52,6 +76,10 @@ pub enum JobError { #[error("invalid environment job request: {message}")] InvalidRequest { message: String }, + /// A job id is taken by another run; the model must choose a new id. + #[error("environment job id in use: {message}")] + JobIdInUse { message: String }, + #[error("environment job execution failed: {message}")] Failed { message: String }, } @@ -69,6 +97,7 @@ pub struct JobHandleArg { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] pub struct JobHandle { pub environment_id: String, + #[schemars(with = "String")] pub job_id: JobId, } @@ -97,7 +126,13 @@ pub struct JobRunArgs { } impl JobRunArgs { - pub fn into_protocol_spec(self, job_id: JobId) -> JobExecResult { + /// The wire spec of this single job. A relative `cwd` resolves against + /// `working_directory`, which is also the default `cwd`. + pub fn into_job_spec( + self, + job_id: JobId, + working_directory: Option<&str>, + ) -> JobExecResult { let timeout_ms = self.timeout_ms.unwrap_or(JOB_RUN_DEFAULT_TIMEOUT_MS); if timeout_ms > JOB_RUN_MAX_TIMEOUT_MS { return Err(JobError::InvalidRequest { @@ -113,10 +148,10 @@ impl JobRunArgs { stdin: self.stdin, timeout_ms: Some(timeout_ms), depends_on: Vec::new(), - dependency_policy: JobDependencyPolicy::AllSucceeded, + dependency_policy: JobDependencyPolicyArg::AllSucceeded, queue_key: self.queue_key, } - .into_protocol_spec(job_id) + .into_job_spec(job_id, working_directory) } } @@ -150,22 +185,18 @@ impl JobSubmitExecutionContextV1 { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct JobDependencyArg { - #[serde(default, alias = "jobId", skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none")] pub job_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, } impl JobDependencyArg { - fn into_protocol_dependency(self) -> JobExecResult { + fn into_job_dependency(self) -> JobExecResult { match (self.job_id, self.name) { - (Some(job_id), None) => Ok(ProtocolJobDependency { - job_id: Some(job_id), - name: None, - }), - (None, Some(name)) if !name.is_empty() => Ok(ProtocolJobDependency { - job_id: None, - name: Some(name), + (Some(job_id), None) => Ok(JobDependency::JobId { job_id }), + (None, Some(name)) if !name.is_empty() => Ok(JobDependency::Name { + name: job_name(name)?, }), (Some(_), Some(_)) => Err(JobError::InvalidRequest { message: "job dependency must specify job_id or name, not both".to_owned(), @@ -177,6 +208,26 @@ impl JobDependencyArg { } } +/// What a job waits for from its `depends_on` jobs. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum JobDependencyPolicyArg { + /// Run once every dependency succeeded. + #[default] + AllSucceeded, + /// Run once every dependency finished, whatever its outcome. + AllTerminal, +} + +impl From for JobDependencyPolicy { + fn from(policy: JobDependencyPolicyArg) -> Self { + match policy { + JobDependencyPolicyArg::AllSucceeded => Self::AllSucceeded, + JobDependencyPolicyArg::AllTerminal => Self::AllTerminal, + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct JobSubmitSpecArgs { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -194,13 +245,21 @@ pub struct JobSubmitSpecArgs { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub depends_on: Vec, #[serde(default)] - pub dependency_policy: JobDependencyPolicy, + pub dependency_policy: JobDependencyPolicyArg, #[serde(default, skip_serializing_if = "Option::is_none")] pub queue_key: Option, } impl JobSubmitSpecArgs { - pub fn into_protocol_spec(self, job_id: JobId) -> JobExecResult { + /// The wire spec of this job. A relative `cwd` resolves against + /// `working_directory`, which is also the default `cwd`; without one a + /// relative `cwd` is refused and an absent one means the environment's + /// default. + pub fn into_job_spec( + self, + job_id: JobId, + working_directory: Option<&str>, + ) -> JobExecResult { if self.argv.is_empty() { return Err(JobError::InvalidRequest { message: "job argv must not be empty".to_owned(), @@ -217,42 +276,50 @@ impl JobSubmitSpecArgs { let depends_on = self .depends_on .into_iter() - .map(JobDependencyArg::into_protocol_dependency) + .map(JobDependencyArg::into_job_dependency) .collect::>>()?; - Ok(JobStartSpec { + Ok(JobSpec { job_id, - name: self.name, - argv: self.argv, - cwd: self.cwd.as_ref().map(environment_path).transpose()?, - env: self.env, - secret_env: BTreeMap::new(), - stdin: self.stdin.map(|value| ByteChunk::from(value.into_bytes())), - timeout_ms: self.timeout_ms, + name: self.name.map(job_name).transpose()?, + command: CommandSpec { + argv: self.argv, + cwd: job_cwd(self.cwd.as_ref(), working_directory)?, + env: self.env, + secret_env: BTreeMap::new(), + input: match self.stdin { + Some(stdin) => ExecInput::Bytes { + data: ByteChunk::from(stdin.into_bytes()), + }, + None => ExecInput::None {}, + }, + timeout_ms: self.timeout_ms, + }, depends_on, - dependency_policy: self.dependency_policy, - queue_key: self.queue_key, + dependency_policy: self.dependency_policy.into(), + queue_key: self + .queue_key + .map(|key| { + QueueKey::new(key).map_err(|error| JobError::InvalidRequest { + message: format!("queue_key is invalid: {error}"), + }) + }) + .transpose()?, }) } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct JobReadArgs { pub jobs: Vec, + /// Output bytes to show per job. Without `after`, half come from the + /// start of the output and half from its end. #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_bytes: Option, + pub output_bytes: Option, + /// Byte offset to page forward from, such as a previous result's + /// `output_next` or its omitted range's `offset`. #[serde(default, skip_serializing_if = "Option::is_none")] - pub after_seq: Option, - #[serde(default)] - pub include_artifacts: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct JobCancelArgs { - pub jobs: Vec, - #[serde(default)] - pub scope: JobCancelScope, - #[serde(default)] - pub force: bool, + pub after: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] @@ -264,10 +331,12 @@ pub struct JobSubmitResult { pub struct JobSubmitted { #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, + #[schemars(with = "String")] pub job_id: JobId, #[serde(default, skip_serializing_if = "Option::is_none")] pub handle: Option, - pub status: JobStatus, + pub status: ModelJobStatus, + #[schemars(with = "Vec")] pub dependencies: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub queue_key: Option, @@ -276,33 +345,216 @@ pub struct JobSubmitted { pub promise: Option, } +impl JobSubmitted { + pub fn from_summary(summary: &JobSummary, handle: Option) -> Self { + Self { + name: summary.name.as_ref().map(|name| name.as_str().to_owned()), + job_id: summary.job_id.clone(), + handle, + status: summary.status.into(), + dependencies: summary.depends_on.clone(), + queue_key: summary + .queue_key + .as_ref() + .map(|key| key.as_str().to_owned()), + promise: None, + } + } +} + +/// Where a job stands. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ModelJobStatus { + Queued, + Running, + Succeeded, + Failed, + TimedOut, + Cancelled, + DependencyFailed, + Interrupted, +} + +impl ModelJobStatus { + pub fn is_terminal(self) -> bool { + !matches!(self, Self::Queued | Self::Running) + } +} + +impl From for ModelJobStatus { + fn from(status: JobStatus) -> Self { + match status { + JobStatus::Queued => Self::Queued, + JobStatus::Running => Self::Running, + JobStatus::Succeeded => Self::Succeeded, + JobStatus::Failed => Self::Failed, + JobStatus::TimedOut => Self::TimedOut, + JobStatus::Cancelled => Self::Cancelled, + JobStatus::DependencyFailed => Self::DependencyFailed, + JobStatus::Interrupted => Self::Interrupted, + } + } +} + +/// Why the environment stopped a job that did not end on its own. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ModelStoppedBy { + Timeout, + Terminate, + Kill, +} + +impl From for ModelStoppedBy { + fn from(stopped_by: StoppedBy) -> Self { + match stopped_by { + StoppedBy::Timeout => Self::Timeout, + StoppedBy::Terminate => Self::Terminate, + StoppedBy::Kill => Self::Kill, + } + } +} + +/// What a job is and how it ended. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct ModelJobSummary { + #[schemars(with = "String")] + pub job_id: JobId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub status: ModelJobStatus, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[schemars(with = "Vec")] + pub depends_on: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_key: Option, + pub created_at_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finished_at_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exit_code: Option, + /// The signal that killed the job's command, such as `SIGSEGV`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signal: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub core_dumped: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stopped_by: Option, + /// Processes the job left running, which the environment stopped when + /// the job finished; the job's script should shut them down itself. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub swept_processes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl From for ModelJobSummary { + fn from(summary: JobSummary) -> Self { + let (exit_code, signal, core_dumped) = match summary.exit { + Some(ExitStatus::Code { code }) => (Some(code), None, false), + Some(ExitStatus::Signal { + signal, + core_dumped, + }) => (None, Some(signal.as_str().to_owned()), core_dumped), + None => (None, None, false), + }; + Self { + job_id: summary.job_id, + name: summary.name.map(JobName::into_string), + status: summary.status.into(), + depends_on: summary.depends_on, + queue_key: summary.queue_key.map(QueueKey::into_string), + created_at_ms: summary.created_at_ms, + started_at_ms: summary.started_at_ms, + finished_at_ms: summary.finished_at_ms, + duration_ms: summary.duration_ms, + exit_code, + signal, + core_dumped, + stopped_by: summary.stopped_by.map(Into::into), + swept_processes: summary + .leftovers + .map(|report| { + report + .processes + .into_iter() + .map(|process| LeftoverProcess { + pid: process.pid, + command: process.command.unwrap_or_default(), + }) + .collect() + }) + .unwrap_or_default(), + error: summary.error.map(|error| error.to_string()), + } + } +} + +/// A range of output the result leaves out. Read it with `job_read` +/// `after: offset`; the `lost_bytes` of it that the environment dropped to +/// bound retained output read back as a gap. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +pub struct ModelOutputOmission { + /// Byte offset of the first omitted byte. + pub offset: u64, + pub bytes: u64, + /// Omitted bytes that are gone for good; reading them again does not + /// return them. + #[serde(default, skip_serializing_if = "is_zero")] + pub lost_bytes: u64, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -#[serde(rename_all = "camelCase")] pub struct ModelJobResult { #[serde(default, skip_serializing_if = "Option::is_none")] pub handle: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary: Option, + pub summary: Option, + /// Output from the requested position, in order. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub output: Vec, - pub output_next_seq: u64, - pub truncated: bool, + /// Output between `output` and `tail` that is not shown. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub omitted: Option, + /// The last output bytes, after the omitted range. #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub artifacts: Vec, + pub tail: Vec, + /// Byte offset after the last output shown; pass it as `after` to read + /// on. + pub output_next: u64, + /// Output bytes the job produced so far. + pub output_end: u64, + /// More output exists after `output_next`. + pub truncated: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub error: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -#[serde(rename_all = "camelCase")] pub struct ModelJobResultSet { pub jobs: Vec, } +/// Which stream produced an output segment. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ModelOutputStream { + Stdout, + Stderr, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -#[serde(rename_all = "camelCase")] pub struct ModelJobOutputSegment { - pub stream: environment_protocol::data::jobs::JobOutputStream, + pub stream: ModelOutputStream, #[serde(default, skip_serializing_if = "Option::is_none")] pub text: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -313,21 +565,6 @@ pub struct ModelJobOutputSegment { pub byte_len: Option, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct JobCancelResultSet { - pub jobs: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct JobCancelResultEntry { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub handle: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub summary: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, -} - pub fn visible_job_read_output(jobs: &[ModelJobResult]) -> String { serde_json::to_string_pretty(&ModelJobResultSet { jobs: jobs.to_vec(), @@ -335,15 +572,107 @@ pub fn visible_job_read_output(jobs: &[ModelJobResult]) -> String { .unwrap_or_else(|error| format!("failed to encode job results: {error}")) } +/// The `job/read` entries that show one job: a page forward from `after`, +/// or, without it, the head and the tail of the output, each half of +/// `output_bytes`. [`NormalizeJobResultInput::from_entries`] reassembles the +/// answer. +pub fn job_output_requests( + job_id: &JobId, + after: Option, + output_bytes: u64, +) -> Vec { + match after { + Some(after) => vec![JobReadRequest { + job_id: job_id.clone(), + after: Some(after), + tail_bytes: None, + max_bytes: Some(output_bytes), + }], + None => { + let head = output_bytes / 2; + vec![ + JobReadRequest { + job_id: job_id.clone(), + after: Some(0), + tail_bytes: None, + max_bytes: Some(head), + }, + JobReadRequest { + job_id: job_id.clone(), + after: None, + tail_bytes: Some(output_bytes - head), + max_bytes: Some(output_bytes - head), + }, + ] + } + } +} + +/// A complete `job/read` for `jobs`, each shown as [`job_output_requests`] +/// describes, waiting for nothing. +pub fn job_output_read(jobs: &[JobId], after: Option, output_bytes: u64) -> ReadJobsParams { + let entries = jobs + .iter() + .flat_map(|job_id| job_output_requests(job_id, after, output_bytes)) + .collect::>(); + ReadJobsParams { + max_bytes: output_bytes.saturating_mul(jobs.len() as u64), + jobs: entries, + wait_ms: 0, + return_on: environment_protocol::data::exec::ReturnOn::AnyOutput, + } +} + #[derive(Clone, Debug, Default)] pub struct NormalizeJobResultInput { pub handle: Option, pub summary: Option, - pub output_chunks: Vec, - pub output_next_seq: u64, - pub artifacts: Vec, + /// Output shown from the requested position: `output`, then `omitted` + /// and `tail`. + pub output: ShownOutput, pub error: Option, - pub output_bytes: Option, +} + +impl NormalizeJobResultInput { + /// Reassemble the entries [`job_output_requests`] asked for one job: a + /// summary and its pages, or the entry's error (`notFound` for a job the + /// environment does not know). + pub fn from_entries( + handle: Option, + entries: impl IntoIterator, + ) -> Self { + let mut summary = None; + let mut pages = Vec::new(); + let mut error = None; + for entry in entries { + match entry { + JobReadEntry::Job { + summary: entry_summary, + output, + } => { + summary = Some(*entry_summary); + pages.push(output); + } + JobReadEntry::Error { + job_id, + error: cause, + } => { + error.get_or_insert_with(|| format!("job {job_id}: {cause}")); + } + } + } + let mut pages = pages.into_iter(); + let output = pages + .next() + .map(|head| ShownOutput::from_pages(head, pages.next())) + .unwrap_or_default(); + Self { + handle, + summary, + output, + error, + } + } } pub async fn normalize_job_result( @@ -353,46 +682,76 @@ pub async fn normalize_job_result( let NormalizeJobResultInput { handle, summary, - output_chunks, - output_next_seq, - artifacts, + output, error, - output_bytes, } = input; - let observed_bytes = output_chunks - .iter() - .map(|chunk| chunk.chunk.as_slice().len()) - .sum::(); - let truncated = output_bytes.is_some_and(|limit| observed_bytes >= limit); + let truncated = output.next < output.end; + let omitted = output.omitted.map(|omission| ModelOutputOmission { + offset: omission.offset, + bytes: omission.bytes, + lost_bytes: omission.lost_bytes, + }); + let head = segments(blobs, output.head).await?; + let tail = segments(blobs, output.tail).await?; + Ok(ModelJobResult { + handle: handle.map(|mut handle| { + handle.environment_id = super::handles::environment_handle(&handle.environment_id); + handle + }), + summary: summary.map(Into::into), + output: head, + omitted, + tail, + output_next: output.next, + output_end: output.end, + truncated, + error, + }) +} + +/// A result that carries only an error, such as an unknown job. +pub fn model_job_error(handle: Option, error: String) -> ModelJobResult { + ModelJobResult { + handle: handle.map(|mut handle| { + handle.environment_id = super::handles::environment_handle(&handle.environment_id); + handle + }), + summary: None, + output: Vec::new(), + omitted: None, + tail: Vec::new(), + output_next: 0, + output_end: 0, + truncated: false, + error: Some(error), + } +} + +/// Merge consecutive chunks of one stream; text stays inline, bytes that +/// are not UTF-8 go to CAS and are referenced. +async fn segments( + blobs: &dyn BlobStore, + chunks: Vec, +) -> Result, BlobStoreError> { let mut output = Vec::new(); let mut index = 0; - while index < output_chunks.len() { - let stream = output_chunks[index].stream; + while index < chunks.len() { + let stream = model_stream(chunks[index].stream); let mut bytes = Vec::new(); - while index < output_chunks.len() && output_chunks[index].stream == stream { - bytes.extend_from_slice(output_chunks[index].chunk.as_slice()); + while index < chunks.len() && model_stream(chunks[index].stream) == stream { + bytes.extend_from_slice(chunks[index].data.as_slice()); index += 1; } match String::from_utf8(bytes) { Ok(text) => { if !text.is_empty() { - if let Some(ModelJobOutputSegment { - stream: previous_stream, - text: Some(previous), - .. - }) = output.last_mut() - && *previous_stream == stream - { - previous.push_str(&text); - } else { - output.push(ModelJobOutputSegment { - stream, - text: Some(text), - blob_ref: None, - media_type: None, - byte_len: None, - }); - } + output.push(ModelJobOutputSegment { + stream, + text: Some(text), + blob_ref: None, + media_type: None, + byte_len: None, + }); } } Err(error) => { @@ -409,18 +768,15 @@ pub async fn normalize_job_result( } } } - Ok(ModelJobResult { - handle: handle.map(|mut handle| { - handle.environment_id = super::handles::environment_handle(&handle.environment_id); - handle - }), - summary, - output, - output_next_seq, - truncated, - artifacts, - error, - }) + Ok(output) +} + +fn model_stream(stream: environment_protocol::data::exec::OutputStream) -> ModelOutputStream { + if is_stderr(stream) { + ModelOutputStream::Stderr + } else { + ModelOutputStream::Stdout + } } pub async fn store_model_job_result( @@ -438,6 +794,7 @@ pub async fn store_model_job_result( let edges = result .output .iter() + .chain(&result.tail) .filter_map(|segment| { segment .blob_ref @@ -453,18 +810,44 @@ pub async fn store_model_job_result( Ok(root) } -fn environment_path(path: &FsPath) -> JobExecResult { - EnvironmentPath::new(path.as_str()).map_err(|error| JobError::InvalidRequest { - message: error.to_string(), +fn job_name(name: String) -> JobExecResult { + JobName::new(name).map_err(|error| JobError::InvalidRequest { + message: format!("job name is invalid: {error}"), }) } +fn job_cwd( + cwd: Option<&FsPath>, + working_directory: Option<&str>, +) -> JobExecResult> { + let invalid = |message: String| JobError::InvalidRequest { message }; + let resolved = match (cwd, working_directory) { + (Some(cwd), _) if cwd.is_absolute() => cwd.as_str().to_owned(), + (cwd, Some(base)) => crate::environment::sources::absolute( + std::path::Path::new(base), + cwd.map_or(".", FsPath::as_str), + ) + .map_err(invalid)? + .to_string_lossy() + .into_owned(), + (Some(cwd), None) => { + return Err(invalid(format!( + "job cwd {cwd} is relative and the environment has no working directory" + ))); + } + (None, None) => return Ok(None), + }; + EnvironmentPath::new(resolved) + .map(Some) + .map_err(|error| invalid(error.to_string())) +} + #[cfg(test)] mod tests { use std::sync::{Arc, Mutex}; use async_trait::async_trait; - use environment_protocol::data::jobs::{JobOutputStream, JobStatus}; + use environment_protocol::data::exec::{OutputChunk, OutputPage, OutputStream, SignalName}; use harness::storage::{ BlobEdge, BlobGraphStore, BlobStore, BlobStoreError, InMemoryBlobStore, }; @@ -472,6 +855,10 @@ mod tests { use super::*; + fn job_id(value: &str) -> JobId { + JobId::new(value).unwrap() + } + #[test] fn job_submit_arguments_do_not_accept_an_environment_override() { let error = serde_json::from_value::(json!({ @@ -491,15 +878,65 @@ mod tests { })) .expect("decode job_run"); let spec = args - .into_protocol_spec(JobId::new("job-derived")) + .into_job_spec(job_id("job-derived"), None) .expect("materialize job_run"); - assert_eq!(spec.job_id, JobId::new("job-derived")); - assert_eq!(spec.argv, vec!["cargo", "test"]); - assert_eq!(spec.timeout_ms, Some(JOB_RUN_DEFAULT_TIMEOUT_MS)); + assert_eq!(spec.job_id, job_id("job-derived")); + assert_eq!(spec.command.argv, vec!["cargo", "test"]); + assert_eq!(spec.command.timeout_ms, Some(JOB_RUN_DEFAULT_TIMEOUT_MS)); assert!(spec.depends_on.is_empty()); } + #[test] + fn job_cwd_resolves_against_the_working_directory() { + let spec = |cwd: Option<&str>, base: Option<&str>| { + JobSubmitSpecArgs { + name: None, + job_id: job_id("build"), + argv: vec!["true".to_owned()], + cwd: cwd.map(|cwd| FsPath::new(cwd).unwrap()), + env: BTreeMap::new(), + stdin: Some("input".to_owned()), + timeout_ms: None, + depends_on: Vec::new(), + dependency_policy: JobDependencyPolicyArg::AllSucceeded, + queue_key: None, + } + .into_job_spec(job_id("build"), base) + }; + let resolved = spec(Some("sub"), Some("/srv")).unwrap(); + assert_eq!(resolved.command.cwd.unwrap().as_str(), "/srv/sub"); + assert_eq!( + resolved.command.input, + ExecInput::Bytes { + data: b"input".to_vec().into() + } + ); + assert_eq!( + spec(None, Some("/srv")) + .unwrap() + .command + .cwd + .unwrap() + .as_str(), + "/srv" + ); + assert_eq!( + spec(Some("/abs"), Some("/srv")) + .unwrap() + .command + .cwd + .unwrap() + .as_str(), + "/abs" + ); + assert!(spec(None, None).unwrap().command.cwd.is_none()); + assert!(matches!( + spec(Some("sub"), None), + Err(JobError::InvalidRequest { .. }) + )); + } + #[test] fn job_submit_accepts_snake_case_job_id_dependencies() { let args = serde_json::from_value::(json!({ @@ -509,7 +946,8 @@ mod tests { }, { "job_id": "test", "argv": ["true"], - "depends_on": [{"job_id": "build"}] + "depends_on": [{"job_id": "build"}, {"name": "lint"}], + "dependency_policy": "allTerminal" }] })) .expect("decode job_submit with snake_case dependency id"); @@ -519,41 +957,23 @@ mod tests { .into_iter() .map(|spec| { let job_id = spec.job_id.clone(); - spec.into_protocol_spec(job_id) + spec.into_job_spec(job_id, None) }) .collect::>>() .expect("materialize protocol specs"); assert_eq!( specs[1].depends_on, - vec![ProtocolJobDependency::job_id("build")] - ); - } - - #[test] - fn job_submit_accepts_camel_case_dependency_alias_for_compatibility() { - let args = serde_json::from_value::(json!({ - "jobs": [{ - "job_id": "build", - "argv": ["true"] - }, { - "job_id": "test", - "argv": ["true"], - "depends_on": [{"jobId": "build"}] - }] - })) - .expect("decode job_submit with camelCase dependency id alias"); - - let spec = args.jobs.into_iter().nth(1).expect("dependent job"); - let job_id = spec.job_id.clone(); - let spec = spec - .into_protocol_spec(job_id) - .expect("materialize protocol spec"); - - assert_eq!( - spec.depends_on, - vec![ProtocolJobDependency::job_id("build")] + vec![ + JobDependency::JobId { + job_id: job_id("build") + }, + JobDependency::Name { + name: JobName::new("lint").unwrap() + } + ] ); + assert_eq!(specs[1].dependency_policy, JobDependencyPolicy::AllTerminal); } #[test] @@ -570,7 +990,7 @@ mod tests { let spec = args.jobs.into_iter().next().expect("job"); let job_id = spec.job_id.clone(); let error = spec - .into_protocol_spec(job_id) + .into_job_spec(job_id, None) .expect_err("empty dependency must be rejected locally"); assert!(matches!(error, JobError::InvalidRequest { .. })); @@ -593,43 +1013,34 @@ mod tests { })) .expect("timeout range is a domain validation"); let error = args - .into_protocol_spec(JobId::new("job-derived")) + .into_job_spec(job_id("job-derived"), None) .expect_err("timeout above maximum must fail"); assert!(matches!(error, JobError::InvalidRequest { .. })); } #[test] fn job_ids_must_fit_the_completion_key_shape() { - for job_id in ["build:1", "", "-lead", &"x".repeat(65)] { + for value in ["build:1", "-lead", &"x".repeat(65)] { + let Ok(id) = JobId::new(value) else { + continue; + }; let error = JobSubmitSpecArgs { name: None, - job_id: JobId::new(if job_id.is_empty() { - "placeholder" - } else { - job_id - }), + job_id: id.clone(), argv: vec!["true".to_owned()], cwd: None, env: BTreeMap::new(), stdin: None, timeout_ms: None, depends_on: Vec::new(), - dependency_policy: JobDependencyPolicy::AllSucceeded, + dependency_policy: JobDependencyPolicyArg::AllSucceeded, queue_key: None, } - .into_protocol_spec(JobId::new(if job_id.is_empty() { - "placeholder" - } else { - job_id - })); - if job_id.is_empty() { - error.expect("placeholder id is valid"); - } else { - assert!( - matches!(error, Err(JobError::InvalidRequest { .. })), - "{job_id:?} must be rejected" - ); - } + .into_job_spec(id, None); + assert!( + matches!(error, Err(JobError::InvalidRequest { .. })), + "{value:?} must be rejected" + ); } } @@ -643,6 +1054,88 @@ mod tests { assert_eq!(context.working_directory.as_deref(), Some("/srv")); } + #[test] + fn head_and_tail_requests_split_the_budget() { + let requests = job_output_requests(&job_id("build"), None, 16); + assert_eq!(requests.len(), 2); + assert_eq!( + (requests[0].after, requests[0].max_bytes), + (Some(0), Some(8)) + ); + assert_eq!( + (requests[1].tail_bytes, requests[1].max_bytes), + (Some(8), Some(8)) + ); + let paged = job_output_requests(&job_id("build"), Some(40), 16); + assert_eq!((paged[0].after, paged[0].max_bytes), (Some(40), Some(16))); + let read = job_output_read(&[job_id("a"), job_id("b")], None, 16); + assert_eq!(read.jobs.len(), 4); + assert_eq!(read.max_bytes, 32); + } + + #[tokio::test(flavor = "current_thread")] + async fn job_results_show_head_and_tail_with_the_skipped_count() { + let blobs = InMemoryBlobStore::new(); + let entries = vec![ + JobReadEntry::Job { + summary: Box::new(test_summary()), + output: page( + vec![chunk(0, OutputStream::Stdout, b"compiling\n")], + 10, + 1000, + ), + }, + JobReadEntry::Job { + summary: Box::new(test_summary()), + output: page( + vec![chunk(990, OutputStream::Stderr, b"FAILED: 3\n")], + 1000, + 1000, + ), + }, + ]; + let result = + normalize_job_result(&blobs, NormalizeJobResultInput::from_entries(None, entries)) + .await + .expect("normalize"); + + assert_eq!(result.output[0].text.as_deref(), Some("compiling\n")); + assert_eq!(result.tail[0].text.as_deref(), Some("FAILED: 3\n")); + assert_eq!(result.tail[0].stream, ModelOutputStream::Stderr); + assert_eq!( + result.omitted, + Some(ModelOutputOmission { + offset: 10, + bytes: 980, + lost_bytes: 0, + }) + ); + assert!(!result.truncated); + let value = serde_json::to_value(&result).expect("semantic JSON"); + assert_eq!(value["output_end"], 1000); + assert_eq!(value["summary"]["status"], "failed"); + assert_eq!(value["summary"]["signal"], "SIGSEGV"); + } + + #[tokio::test(flavor = "current_thread")] + async fn unknown_jobs_read_as_errors() { + let blobs = InMemoryBlobStore::new(); + let result = normalize_job_result( + &blobs, + NormalizeJobResultInput::from_entries( + None, + vec![JobReadEntry::Error { + job_id: job_id("gone"), + error: environment_protocol::error::ProtocolError::not_found("unknown job"), + }], + ), + ) + .await + .expect("normalize"); + assert!(result.summary.is_none()); + assert!(result.error.as_deref().is_some_and(|e| e.contains("gone"))); + } + #[tokio::test(flavor = "current_thread")] async fn normalizer_reassembles_utf8_and_preserves_stream_order() { let blobs = InMemoryBlobStore::new(); @@ -650,14 +1143,19 @@ mod tests { &blobs, NormalizeJobResultInput { summary: Some(test_summary()), - output_chunks: vec![ - output_chunk(0, JobOutputStream::Stdout, vec![b'a', 0xe2]), - output_chunk(1, JobOutputStream::Stdout, vec![0x82, 0xac, b'\n']), - output_chunk(2, JobOutputStream::Stderr, b"warning\n".to_vec()), - output_chunk(3, JobOutputStream::Stdout, b"done\n".to_vec()), - ], - output_next_seq: 4, - output_bytes: Some(1024), + output: ShownOutput::from_pages( + page( + vec![ + chunk(0, OutputStream::Stdout, &[b'a', 0xe2]), + chunk(2, OutputStream::Stdout, &[0x82, 0xac, b'\n']), + chunk(5, OutputStream::Stderr, b"warning\n"), + chunk(13, OutputStream::Stdout, b"done\n"), + ], + 18, + 18, + ), + None, + ), ..Default::default() }, ) @@ -666,12 +1164,10 @@ mod tests { assert_eq!(result.output.len(), 3); assert_eq!(result.output[0].text.as_deref(), Some("a€\n")); - assert_eq!(result.output[1].stream, JobOutputStream::Stderr); + assert_eq!(result.output[1].stream, ModelOutputStream::Stderr); assert_eq!(result.output[2].text.as_deref(), Some("done\n")); assert!(!result.truncated); - let value = serde_json::to_value(&result).expect("semantic JSON"); - assert!(value.get("outputChunks").is_none()); - assert_eq!(value["outputNextSeq"], 4); + assert_eq!(result.output_next, 18); } #[tokio::test(flavor = "current_thread")] @@ -681,16 +1177,17 @@ mod tests { &blobs, NormalizeJobResultInput { summary: Some(test_summary()), - output_chunks: vec![output_chunk(0, JobOutputStream::Stdout, vec![0xff, 0x00])], - output_next_seq: 1, - output_bytes: Some(2), + output: ShownOutput::from_pages( + page(vec![chunk(0, OutputStream::Stdout, &[0xff, 0x00])], 2, 4), + None, + ), ..Default::default() }, ) .await .expect("normalize binary"); - assert!(result.truncated); + assert!(result.truncated, "output continues after output_next"); let segment = &result.output[0]; assert!(segment.text.is_none()); assert_eq!( @@ -715,8 +1212,10 @@ mod tests { &blobs, NormalizeJobResultInput { summary: Some(test_summary()), - output_chunks: vec![output_chunk(0, JobOutputStream::Stdout, vec![0xff])], - output_next_seq: 1, + output: ShownOutput::from_pages( + page(vec![chunk(0, OutputStream::Stdout, &[0xff])], 1, 1), + None, + ), ..Default::default() }, ) @@ -746,29 +1245,44 @@ mod tests { } } - fn output_chunk(seq: u64, stream: JobOutputStream, bytes: Vec) -> JobOutputChunk { - JobOutputChunk { - seq, + fn chunk(offset: u64, stream: OutputStream, bytes: &[u8]) -> OutputChunk { + OutputChunk { + offset, stream, - chunk: ByteChunk::from(bytes), + data: bytes.to_vec().into(), + } + } + + fn page(chunks: Vec, next: u64, end: u64) -> OutputPage { + OutputPage { + chunks, + next, + end, + gaps: Vec::new(), } } fn test_summary() -> JobSummary { JobSummary { - namespace: "environment-a".to_owned(), - job_id: JobId::new("job-a"), + job_id: job_id("job-a"), + request_id: environment_protocol::ids::JobRequestId::new("request-a").unwrap(), name: None, - status: JobStatus::Succeeded, - dependencies: Vec::new(), + status: JobStatus::Failed, + depends_on: Vec::new(), + queue_key: None, created_at_ms: 1, queued_at_ms: Some(1), started_at_ms: Some(2), finished_at_ms: Some(3), - exit_code: Some(0), - orphaned_descendants: false, - failure: None, - queue_key: None, + cancel_requested_at_ms: None, + duration_ms: Some(1), + exit: Some(ExitStatus::Signal { + signal: SignalName::new("SIGSEGV").unwrap(), + core_dumped: false, + }), + stopped_by: None, + leftovers: None, + error: None, } } } diff --git a/crates/tools/src/environment/output.rs b/crates/tools/src/environment/output.rs new file mode 100644 index 000000000..e970f4aeb --- /dev/null +++ b/crates/tools/src/environment/output.rs @@ -0,0 +1,411 @@ +//! Shaping command output pages into what the model sees: a head, the +//! bytes omitted after it, and a tail. +//! +//! Environment reads return byte-offset pages with typed gaps. A finished +//! command whose output exceeds the tool budget is shown as its first bytes +//! and its last bytes, with the omitted count in between, because failures +//! are reported at the end. + +use environment_protocol::data::exec::{OutputChunk, OutputPage, OutputStream}; + +/// Output selected for the model, in offset order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ShownOutput { + /// Chunks before the first omitted range. + pub head: Vec, + /// The omitted ranges, merged: where the first starts, how many bytes + /// all of them hold together, and how many of those the environment no + /// longer holds. + pub omitted: Option, + /// Chunks after the first omitted range. + pub tail: Vec, + /// Offset after the last byte or gap shown. + pub next: u64, + /// Bytes the command produced so far. + pub end: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Omission { + pub offset: u64, + pub bytes: u64, + /// Of `bytes`, those the environment's typed gaps report as gone + /// (`overflow`, `expired`): reading them again returns the same gap. + /// The rest was skipped here and can still be read. + pub lost_bytes: u64, +} + +impl ShownOutput { + /// The first page of a read, with an optional second page that starts + /// later (a tail read). Bytes of `tail` that `head` already returned + /// are dropped; the range between the two pages and every gap inside + /// them count as omitted. + pub fn from_pages(head: OutputPage, tail: Option) -> Self { + let mut shown = Self::default(); + let mut items = Vec::new(); + let mut next = head.next; + let mut end = head.end; + push_page(&mut items, head, 0); + if let Some(tail) = tail { + let tail_start = page_start(&tail); + if tail_start > next { + items.push(Item::Gap { + offset: next, + bytes: tail_start - next, + lost: false, + }); + } + let skip_before = next; + next = next.max(tail.next); + end = end.max(tail.end); + push_page(&mut items, tail, skip_before); + } + shown.next = next; + shown.end = end; + for item in items { + match item { + Item::Chunk(chunk) if shown.omitted.is_none() => shown.head.push(chunk), + Item::Chunk(chunk) => shown.tail.push(chunk), + Item::Gap { + offset, + bytes, + lost, + } => { + let lost_bytes = if lost { bytes } else { 0 }; + match &mut shown.omitted { + Some(omission) => { + omission.bytes += bytes; + omission.lost_bytes += lost_bytes; + } + None => { + shown.omitted = Some(Omission { + offset, + bytes, + lost_bytes, + }) + } + } + } + } + } + shown + } + + /// Keep at most `budget` bytes: the first half of the budget from the + /// head and the rest from the end of what was read, omitting the middle. + /// Used when a single page returned more than the model may see. + pub fn trim_to(mut self, budget: u64) -> Self { + let total = self.shown_bytes(); + if total <= budget { + return self; + } + let head_budget = budget / 2; + let tail_budget = budget - head_budget; + let mut chunks: Vec = self.head.drain(..).chain(self.tail.drain(..)).collect(); + let first = chunks + .first() + .map(|chunk| chunk.offset) + .unwrap_or(self.next); + let mut head = Vec::new(); + let mut taken = 0u64; + let mut cut_at = first; + let mut index = 0; + while index < chunks.len() && taken < head_budget { + let chunk = &mut chunks[index]; + let room = head_budget - taken; + let length = chunk.data.len() as u64; + if length <= room { + taken += length; + cut_at = chunk.offset + length; + head.push(chunk.clone()); + index += 1; + } else { + head.push(slice_chunk(chunk, 0, room)); + cut_at = chunk.offset + room; + taken = head_budget; + } + } + let mut tail = Vec::new(); + let mut kept = 0u64; + for chunk in chunks.iter().rev() { + if kept >= tail_budget { + break; + } + let length = chunk.data.len() as u64; + let chunk_end = chunk.offset + length; + let start = chunk.offset.max(cut_at); + if chunk_end <= start { + break; + } + let available = chunk_end - start; + let keep = available.min(tail_budget - kept); + let from = chunk_end - keep - chunk.offset; + tail.push(slice_chunk(chunk, from, keep)); + kept += keep; + } + tail.reverse(); + let dropped = total - taken - kept; + let omitted = self.omitted_bytes() + dropped; + let offset = self + .omitted + .map_or(cut_at, |omission| omission.offset.min(cut_at)); + self.head = head; + self.tail = tail; + let lost_bytes = self.lost_bytes(); + self.omitted = (omitted > 0).then_some(Omission { + offset, + bytes: omitted, + lost_bytes, + }); + self + } + + pub fn shown_bytes(&self) -> u64 { + self.head + .iter() + .chain(&self.tail) + .map(|chunk| chunk.data.len() as u64) + .sum() + } + + pub fn omitted_bytes(&self) -> u64 { + self.omitted.map_or(0, |omission| omission.bytes) + } + + /// Omitted bytes the environment no longer holds. + pub fn lost_bytes(&self) -> u64 { + self.omitted.map_or(0, |omission| omission.lost_bytes) + } +} + +enum Item { + Chunk(OutputChunk), + /// `lost`: a gap the environment reported; otherwise a readable range + /// between two pages. + Gap { + offset: u64, + bytes: u64, + lost: bool, + }, +} + +/// Where a page begins: its first chunk or gap, or `next` when empty. +fn page_start(page: &OutputPage) -> u64 { + let first_chunk = page.chunks.first().map(|chunk| chunk.offset); + let first_gap = page.gaps.first().map(|gap| gap.offset); + match (first_chunk, first_gap) { + (Some(chunk), Some(gap)) => chunk.min(gap), + (Some(offset), None) | (None, Some(offset)) => offset, + (None, None) => page.next, + } +} + +/// Append a page's chunks and gaps in offset order, dropping bytes before +/// `skip_before`. +fn push_page(items: &mut Vec, page: OutputPage, skip_before: u64) { + let mut entries: Vec = page + .chunks + .into_iter() + .map(Item::Chunk) + .chain(page.gaps.into_iter().map(|gap| Item::Gap { + offset: gap.offset, + bytes: gap.omitted_bytes, + lost: true, + })) + .collect(); + entries.sort_by_key(|item| match item { + Item::Chunk(chunk) => chunk.offset, + Item::Gap { offset, .. } => *offset, + }); + for item in entries { + match item { + Item::Chunk(chunk) => { + let length = chunk.data.len() as u64; + if chunk.offset + length <= skip_before { + continue; + } + if chunk.offset < skip_before { + let from = skip_before - chunk.offset; + items.push(Item::Chunk(slice_chunk(&chunk, from, length - from))); + } else { + items.push(Item::Chunk(chunk)); + } + } + Item::Gap { + offset, + bytes, + lost, + } => { + let gap_end = offset + bytes; + if gap_end <= skip_before { + continue; + } + let start = offset.max(skip_before); + items.push(Item::Gap { + offset: start, + bytes: gap_end - start, + lost, + }); + } + } + } +} + +fn slice_chunk(chunk: &OutputChunk, from: u64, length: u64) -> OutputChunk { + let from = from as usize; + let to = from + length as usize; + OutputChunk { + offset: chunk.offset + from as u64, + stream: chunk.stream, + data: chunk.data.as_slice()[from..to].to_vec().into(), + } +} + +/// Whether a chunk belongs to standard error rather than standard output +/// (a PTY merges both into standard output). +pub fn is_stderr(stream: OutputStream) -> bool { + matches!(stream, OutputStream::Stderr) +} + +#[cfg(test)] +mod tests { + use environment_protocol::data::exec::{GapReason, OutputGap}; + + use super::*; + + fn chunk(offset: u64, text: &str) -> OutputChunk { + OutputChunk { + offset, + stream: OutputStream::Stdout, + data: text.as_bytes().to_vec().into(), + } + } + + fn text(chunks: &[OutputChunk]) -> String { + chunks + .iter() + .map(|chunk| String::from_utf8_lossy(chunk.data.as_slice()).into_owned()) + .collect() + } + + #[test] + fn head_and_tail_pages_show_the_skipped_range() { + let head = OutputPage { + chunks: vec![chunk(0, "head")], + next: 4, + end: 100, + gaps: vec![], + }; + let tail = OutputPage { + chunks: vec![chunk(96, "tail")], + next: 100, + end: 100, + gaps: vec![], + }; + let shown = ShownOutput::from_pages(head, Some(tail)); + assert_eq!(text(&shown.head), "head"); + assert_eq!(text(&shown.tail), "tail"); + assert_eq!( + shown.omitted, + Some(Omission { + offset: 4, + bytes: 92, + lost_bytes: 0, + }) + ); + assert_eq!((shown.next, shown.end), (100, 100)); + } + + #[test] + fn overlapping_pages_show_every_byte_once() { + let head = OutputPage { + chunks: vec![chunk(0, "abcdef")], + next: 6, + end: 8, + gaps: vec![], + }; + let tail = OutputPage { + chunks: vec![chunk(4, "efgh")], + next: 8, + end: 8, + gaps: vec![], + }; + let shown = ShownOutput::from_pages(head, Some(tail)); + assert_eq!(text(&shown.head), "abcdefgh"); + assert!(shown.omitted.is_none()); + } + + #[test] + fn daemon_gaps_count_as_omitted() { + let page = OutputPage { + chunks: vec![chunk(0, "ab"), chunk(10, "yz")], + next: 12, + end: 12, + gaps: vec![OutputGap { + offset: 2, + omitted_bytes: 8, + reason: GapReason::Overflow, + }], + }; + let shown = ShownOutput::from_pages(page, None); + assert_eq!(text(&shown.head), "ab"); + assert_eq!(text(&shown.tail), "yz"); + assert_eq!(shown.omitted_bytes(), 8); + assert_eq!( + shown.lost_bytes(), + 8, + "an overflow gap cannot be read again" + ); + } + + #[test] + fn retention_gaps_and_skipped_ranges_are_told_apart() { + let head = OutputPage { + chunks: vec![chunk(0, "head")], + next: 14, + end: 100, + gaps: vec![OutputGap { + offset: 4, + omitted_bytes: 10, + reason: GapReason::Overflow, + }], + }; + let tail = OutputPage { + chunks: vec![chunk(96, "tail")], + next: 100, + end: 100, + gaps: vec![], + }; + let shown = ShownOutput::from_pages(head, Some(tail)); + assert_eq!( + shown.omitted, + Some(Omission { + offset: 4, + bytes: 92, + lost_bytes: 10, + }) + ); + } + + #[test] + fn trimming_keeps_the_first_and_last_bytes() { + let page = OutputPage { + chunks: vec![chunk(0, "0123456789"), chunk(10, "abcdefghij")], + next: 20, + end: 20, + gaps: vec![], + }; + let shown = ShownOutput::from_pages(page, None).trim_to(8); + assert_eq!(text(&shown.head), "0123"); + assert_eq!(text(&shown.tail), "ghij"); + assert_eq!( + shown.omitted, + Some(Omission { + offset: 4, + bytes: 12, + lost_bytes: 0, + }) + ); + assert_eq!(shown.shown_bytes(), 8); + } +} diff --git a/crates/tools/src/environment/process/mod.rs b/crates/tools/src/environment/process/mod.rs index 60a6ca374..7008d6be7 100644 --- a/crates/tools/src/environment/process/mod.rs +++ b/crates/tools/src/environment/process/mod.rs @@ -11,7 +11,7 @@ use std::collections::BTreeMap; use async_trait::async_trait; -use environment_protocol::shared::SecretString; +use environment_protocol::bytes::SecretString; use serde::{Deserialize, Serialize}; use thiserror::Error; diff --git a/crates/tools/src/environment/sources.rs b/crates/tools/src/environment/sources.rs index 22a508cf3..e4cb070dc 100644 --- a/crates/tools/src/environment/sources.rs +++ b/crates/tools/src/environment/sources.rs @@ -1,7 +1,10 @@ //! Shared machine source scope, resolved from the session's effective directory. use environment_protocol::{ - data::inventory::{InventoryLimits, ScanParams}, - shared::EnvironmentPath, + data::{ + fs::TreeFilter, + scan::{ScanLimits, ScanParams, ScanRoot}, + }, + paths::EnvironmentPath, }; use std::{ collections::BTreeSet, @@ -61,30 +64,77 @@ pub fn scan_query( } } } - if roots.len() > 32 { - return Err("discovery exceeds 32 roots".into()); + // One root slot of the protocol's 32 stays free for the working + // directory probe. + if roots.len() > 31 { + return Err("discovery exceeds 31 roots".into()); } + // Discovery roots are often hidden (`.agents`) or ignored by a project's + // ignore files, so both defaults are opted out of below each root. Ok(ScanParams { roots: roots .into_iter() - .map(|p| EnvironmentPath::new(p.to_string_lossy()).map_err(|e| e.to_string())) - .collect::>()?, - include_patterns: if source == "prompts" { - vec!["*.md".into(), "*.txt".into()] - } else { - vec!["SKILL.md".into(), "**/SKILL.md".into()] + .map(|path| { + Ok(ScanRoot { + path: EnvironmentPath::new(path.to_string_lossy()) + .map_err(|e| e.to_string())?, + if_none_match: None, + max_depth: None, + }) + }) + .collect::>()?, + filter: TreeFilter { + include: if source == "prompts" { + vec!["*.md".into(), "*.txt".into()] + } else { + vec!["SKILL.md".into()] + }, + include_hidden: true, + include_ignored: true, + max_depth: Some(if source == "prompts" { 1 } else { 8 }), + ..TreeFilter::default() }, read_content: true, + include_digests: false, follow_symlinks: true, - digest_algorithm: None, - if_none_match: None, - limits: InventoryLimits { + limits: ScanLimits { max_entries: 4096, - max_depth: if source == "prompts" { 1 } else { 8 }, max_file_bytes: 64 * 1024, max_total_bytes: 2 * 1024 * 1024, - max_manifest_bytes: 4 * 1024 * 1024, max_duration_ms: 2000, }, }) } + +/// Add the working-directory probe to a discovery scan: a root examined at +/// depth 0, so the same observation proves `cwd` is an existing directory +/// (`complete` without entries) instead of a separate `fs/getMetadata`. +/// The probe is the last root of the response. +pub fn with_working_directory_probe(params: &mut ScanParams, cwd: &EnvironmentPath) { + params.roots.push(ScanRoot { + path: cwd.clone(), + if_none_match: None, + max_depth: Some(0), + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovery_scans_select_by_name_and_probe_the_working_directory() { + let mut params = scan_query(None, "/project", Some("/home/dev"), "skills").unwrap(); + assert_eq!(params.filter.include, vec!["SKILL.md"]); + assert!(params.filter.include_hidden && params.filter.include_ignored); + let cwd = EnvironmentPath::new("/project").unwrap(); + with_working_directory_probe(&mut params, &cwd); + let probe = params.roots.last().unwrap(); + assert_eq!( + (probe.path.as_str(), probe.max_depth), + ("/project", Some(0)) + ); + let prompts = scan_query(None, "/project", Some("/home/dev"), "prompts").unwrap(); + assert_eq!(prompts.filter.max_depth, Some(1)); + } +} diff --git a/crates/tools/src/environment/tools/jobs.rs b/crates/tools/src/environment/tools/jobs.rs index bb919b948..bb9d31cc0 100644 --- a/crates/tools/src/environment/tools/jobs.rs +++ b/crates/tools/src/environment/tools/jobs.rs @@ -1,14 +1,17 @@ //! Canonical durable-job operations. -use environment_protocol::data::jobs::{ReadJobsParams, StartJobsParams, StartJobsResponse}; +use environment_protocol::{ + data::job::{JobReadEntry, StartJobsParams, StartJobsResponse}, + ids::JobRequestId, +}; use crate::{ environment::{ EnvironmentToolContext, jobs::{ - JobError, JobHandle, JobReadArgs, JobSubmitArgs, JobSubmitResult, JobSubmitted, - ModelJobResultSet, NormalizeJobResultInput, normalize_job_result, - visible_job_read_output, + DEFAULT_JOB_OUTPUT_BYTES, JobHandle, JobReadArgs, JobSubmitArgs, JobSubmitResult, + JobSubmitted, ModelJobResultSet, NormalizeJobResultInput, job_output_read, + normalize_job_result, visible_job_read_output, }, }, error::ToolResult, @@ -37,32 +40,55 @@ pub async fn invoke_job_read( return Err(invalid_request("job_read requires at least one job")); } let jobs = ctx.jobs.as_ref().ok_or_else(unsupported_job_capability)?; - let response = jobs - .read_jobs(ReadJobsParams { - namespace: job_namespace(ctx, None)?, - jobs: args.jobs.into_iter().map(|handle| handle.job_id).collect(), - after_seq: args.after_seq, - max_bytes: args.output_bytes, - include_artifacts: args.include_artifacts, - wait_ms: None, - }) - .await?; - let mut normalized = Vec::with_capacity(response.jobs.len()); - for job in response.jobs { - normalized.push( - normalize_job_result( - ctx.blobs.as_ref(), - NormalizeJobResultInput { - summary: Some(job.summary), - output_chunks: job.output_chunks, - output_next_seq: job.output_next_seq, - artifacts: job.artifacts, - output_bytes: args.output_bytes, - ..Default::default() - }, - ) - .await?, - ); + let job_ids = args + .jobs + .iter() + .map(|handle| handle.job_id.clone()) + .collect::>(); + let mut output_bytes = args + .output_bytes + .unwrap_or(DEFAULT_JOB_OUTPUT_BYTES) + .min(ctx.limits.max_model_visible_output_bytes); + // Without `after` every job takes two `job/read` entries (head and + // tail); the environment's `maxRequestJobs` bounds entries, not jobs, + // so the jobs go out in as many requests as that bound needs. + let mut after = args.after; + let mut jobs_per_request = job_ids.len(); + if let Some(limits) = jobs.limits() { + let max_entries = limits.max_request_jobs as usize; + if job_ids.len() > max_entries { + return Err(invalid_request(format!( + "job_read names {} jobs; the environment reads at most {} at once", + job_ids.len(), + limits.max_request_jobs + ))); + } + if after.is_none() && max_entries < 2 { + // No room for a tail entry: show the head only. + after = Some(0); + } + let per_job = if after.is_some() { 1 } else { 2 }; + jobs_per_request = (max_entries / per_job).max(1); + let in_one_request = jobs_per_request.min(job_ids.len()) as u64; + output_bytes = output_bytes.min(limits.max_read_bytes / in_one_request); + } + let per_job = if after.is_some() { 1 } else { 2 }; + let mut normalized = Vec::with_capacity(job_ids.len()); + for chunk in job_ids.chunks(jobs_per_request.max(1)) { + let response = jobs + .read_jobs(job_output_read(chunk, after, output_bytes)) + .await?; + let mut entries = response.jobs.into_iter(); + for _ in chunk { + let job_entries: Vec = entries.by_ref().take(per_job).collect(); + normalized.push( + normalize_job_result( + ctx.blobs.as_ref(), + NormalizeJobResultInput::from_entries(None, job_entries), + ) + .await?, + ); + } } Ok(ModelJobResultSet { jobs: normalized }) } @@ -75,29 +101,29 @@ fn submit_params_from_args( ctx: &EnvironmentToolContext, args: JobSubmitArgs, ) -> ToolResult { + let working_directory = ctx.process_cwd.as_ref().map(|cwd| cwd.as_str()); let mut specs = Vec::with_capacity(args.jobs.len()); for spec in args.jobs { let job_id = spec.job_id.clone(); - specs.push(spec.into_protocol_spec(job_id)?); + specs.push(spec.into_job_spec(job_id, working_directory)?); } Ok(StartJobsParams { - namespace: job_namespace(ctx, None)?, - request_id: "default".to_owned(), + request_id: request_id(ctx), jobs: specs, }) } -fn job_namespace( - ctx: &EnvironmentToolContext, - explicit_session_id: Option<&str>, -) -> ToolResult { - explicit_session_id - .map(ToOwned::to_owned) - .or_else(|| ctx.session_id.clone()) - .ok_or_else(|| JobError::InvalidRequest { - message: "environment job namespace requires a session_id".to_owned(), +/// One request id per call: the invocation's stable operation identity, so +/// a retried call replays its jobs while a new call that reuses a job id is +/// refused instead of answered with an earlier run. +fn request_id(ctx: &EnvironmentToolContext) -> JobRequestId { + ctx.operation_id + .as_deref() + .and_then(|operation_id| JobRequestId::new(operation_id).ok()) + .unwrap_or_else(|| { + JobRequestId::new(format!("jobreq-{}", uuid::Uuid::new_v4().simple())) + .expect("uuid request id fits the id grammar") }) - .map_err(Into::into) } fn submit_result_from_response( @@ -107,21 +133,162 @@ fn submit_result_from_response( JobSubmitResult { jobs: response .jobs - .into_iter() - .map(|summary| JobSubmitted { - name: summary.name, - handle: ctx.environment_id.clone().map(|environment_id| JobHandle { + .iter() + .map(|summary| { + let handle = ctx.environment_id.clone().map(|environment_id| JobHandle { environment_id: crate::environment::handles::environment_handle( &environment_id, ), job_id: summary.job_id.clone(), - }), - job_id: summary.job_id, - status: summary.status, - dependencies: summary.dependencies, - queue_key: summary.queue_key, - promise: None, + }); + JobSubmitted::from_summary(summary, handle) }) .collect(), } } + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use environment_protocol::{ + data::job::{ + CancelJobsParams, CancelJobsResponse, JobStatus, JobSummary, ReadJobsParams, + ReadJobsResponse, + }, + ids::JobId, + }; + use harness::storage::InMemoryBlobStore; + use serde_json::json; + + use super::*; + use crate::environment::jobs::{JobExecResult, JobExecutor}; + + #[derive(Default)] + struct RecordingJobs { + starts: Mutex>, + reads: Mutex>, + limits: Option, + } + + #[async_trait] + impl JobExecutor for RecordingJobs { + fn limits(&self) -> Option { + self.limits.clone() + } + + async fn start_jobs(&self, request: StartJobsParams) -> JobExecResult { + let jobs = request + .jobs + .iter() + .map(|job| JobSummary { + job_id: job.job_id.clone(), + request_id: request.request_id.clone(), + name: None, + status: JobStatus::Queued, + depends_on: Vec::new(), + queue_key: None, + created_at_ms: 1, + queued_at_ms: None, + started_at_ms: None, + finished_at_ms: None, + cancel_requested_at_ms: None, + duration_ms: None, + exit: None, + stopped_by: None, + leftovers: None, + error: None, + }) + .collect(); + self.starts.lock().unwrap().push(request); + Ok(StartJobsResponse { jobs }) + } + + async fn read_jobs(&self, request: ReadJobsParams) -> JobExecResult { + let jobs = request + .jobs + .iter() + .map(|entry| JobReadEntry::Error { + job_id: entry.job_id.clone(), + error: environment_protocol::error::ProtocolError::not_found("unknown job"), + }) + .collect(); + self.reads.lock().unwrap().push(request); + Ok(ReadJobsResponse { jobs }) + } + + async fn cancel_jobs( + &self, + _request: CancelJobsParams, + ) -> JobExecResult { + unreachable!("not cancelled") + } + } + + #[tokio::test(flavor = "current_thread")] + async fn inline_job_submits_send_a_request_id_per_call() { + let jobs = Arc::new(RecordingJobs::default()); + let base = EnvironmentToolContext::new(None, Arc::new(InMemoryBlobStore::new())) + .with_jobs(jobs.clone()); + let args = || { + serde_json::from_value::(json!({ + "jobs": [{"job_id": "build", "argv": ["make"]}] + })) + .unwrap() + }; + let first = base.clone().with_operation_id("tool-first"); + invoke_job_submit(&first, args()).await.unwrap(); + invoke_job_submit(&first, args()).await.unwrap(); + let second = base.with_operation_id("tool-second"); + let submitted = invoke_job_submit(&second, args()).await.unwrap(); + + let starts = jobs.starts.lock().unwrap(); + assert_eq!(starts[0].request_id.as_str(), "tool-first"); + assert_eq!( + starts[0].request_id, starts[1].request_id, + "a retried call replays" + ); + assert_ne!( + starts[0].request_id, starts[2].request_id, + "a new call is a new run" + ); + assert_eq!(starts[0].jobs[0].job_id, JobId::new("build").unwrap()); + assert_eq!( + serde_json::to_value(&submitted).unwrap()["jobs"][0]["status"], + "queued" + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn job_reads_split_head_and_tail_entries_across_requests() { + let jobs = Arc::new(RecordingJobs { + limits: Some(environment_protocol::data::handshake::JobLimits { + max_read_bytes: 4096, + max_request_jobs: 4, + max_retained_output_bytes: 1 << 20, + record_retention_ms: 1, + cancel_grace_ms: 1, + }), + ..RecordingJobs::default() + }); + let ctx = EnvironmentToolContext::new(None, Arc::new(InMemoryBlobStore::new())) + .with_jobs(jobs.clone()); + let args = serde_json::from_value::(json!({ + "jobs": [{"job_id": "a"}, {"job_id": "b"}, {"job_id": "c"}] + })) + .unwrap(); + let result = invoke_job_read(&ctx, args).await.unwrap(); + + assert_eq!(result.jobs.len(), 3); + let reads = jobs.reads.lock().unwrap(); + assert_eq!(reads.len(), 2, "two jobs of head and tail fill one request"); + assert!(reads.iter().all(|read| read.jobs.len() <= 4)); + assert!(reads.iter().all(|read| read.max_bytes <= 4096)); + let named = reads + .iter() + .flat_map(|read| read.jobs.iter().map(|entry| entry.job_id.as_str())) + .collect::>(); + assert_eq!(named, ["a", "a", "b", "b", "c", "c"]); + } +} diff --git a/crates/tools/src/environment_protocol/conformance.rs b/crates/tools/src/environment_protocol/conformance.rs index 94f17e17e..9a22c42a3 100644 --- a/crates/tools/src/environment_protocol/conformance.rs +++ b/crates/tools/src/environment_protocol/conformance.rs @@ -1,263 +1,1312 @@ -//! Reusable conformance checks for environment data-plane implementations. +//! Data-plane conformance: drives any environment data server through +//! [`DataClient`] and checks the contract envd implements. +//! +//! The suite checks, against the wire and not through tool adapters: +//! +//! - the `initialize` response: plane, version, ceilings and capability +//! families; +//! - `fs/*`: staged writes with `create` and `ifMatch` preconditions and a +//! returned version, ranged reads with `fileSize` and `truncated`, +//! ceilings refused as `invalidParams`, `fs/rename` without overwrite, +//! paged `fs/readDirectory`, `fs/remove` preconditions, typed `notFound` +//! and confinement errors, the shared pattern grammar of +//! `fs/globFiles` and `fs/searchText` (and that the tools' generic glob +//! fallback selects the same files), and per-root `fs/scan` results; +//! - confinement of symlinks (when the `process` family can create them): +//! an in-root link is followed, a link that leaves the root is refused +//! with `permissionDenied {outsideRoot}`; +//! - `process/*` (when the family is present): settled exits with codes and +//! named signals, start-time input larger than a pipe buffer, tail reads, +//! duplicate ids refused before anything runs, unknown handles as +//! `notFound`, and graceful `terminate`; +//! - `job/*` (when the family is present): `requestId` replay versus +//! `jobIdInUse`, per-entry `notFound`, and byte-exact paging. +//! +//! The suite works in a fresh directory below `work_directory` and removes +//! it at the end. Commands run through `/bin/sh`. +//! +//! ```no_run +//! # async fn example(client: environment_client::DataClient) { +//! use tools::environment_protocol::conformance::{DataConformance, run}; +//! +//! let work = environment_protocol::paths::EnvironmentPath::new("/workspace").unwrap(); +//! run(&DataConformance::new(client, work)).await.into_result().expect("data plane conforms"); +//! # } +//! ``` -use std::sync::Arc; +use std::{ + collections::BTreeMap, + fmt, + future::Future, + time::{Duration, Instant}, +}; -use environment_client::{EnvironmentClientError, EnvironmentDataClient, JsonRpcTransport}; +use environment_client::{ClientError, DataClient}; use environment_protocol::{ - data::handshake::{InitializeParams, InitializedParams}, - shared::{CURRENT_PROTOCOL_VERSION, EnvironmentCapabilities}, + ENVD_PROTOCOL_VERSION, + bytes::ByteChunk, + data::{ + exec::{CommandSpec, ExecInput, ExecState, ExitStatus, ReturnOn, StoppedBy}, + fs::{ + CreateDirectoryParams, FileKind, GetMetadataParams, GlobFilesLimits, GlobFilesParams, + ReadDirectoryParams, ReadFileParams, RemoveParams, RenameParams, SearchTextLimits, + SearchTextParams, TreeFilter, WriteCondition, WriteFileParams, WriteFileResponse, + }, + handshake::InitializeResponse, + idle::GetIdleParams, + job::{JobReadEntry, JobReadRequest, JobSpec, ReadJobsParams, StartJobsParams}, + process::{ + ProcessSignal, ReadProcessParams, ReadProcessResponse, StartProcessParams, + TerminateProcessParams, WriteProcessParams, + }, + scan::{ScanLimits, ScanParams, ScanRoot, ScanRootResult}, + }, + error::{ErrorData, PermissionReason, PreconditionReason, ProtocolErrorKind}, + handshake::Plane, + ids::{JobId, JobRequestId, ProcessId}, + limits::MIN_CONCURRENT_REQUESTS, + paths::{EnvironmentPath, TreePath}, }; -use harness::storage::InMemoryBlobStore; use thiserror::Error; -use crate::{ - environment_protocol::RemoteEnvironmentConnection, - error::ToolError, - fs::{ - CopyOptions, CreateDirectoryOptions, FileAccessPolicy, FsError, FsPath, FsPathError, - RemoveOptions, - tools::{ReadFileArgs, WriteFileArgs, invoke_read_file, invoke_write_file}, - }, -}; +/// What to run the suite against. +#[derive(Clone)] +pub struct DataConformance { + /// An initialized connection to the server under test. + pub client: DataClient, + /// An existing, writable directory inside the filesystem root. + pub work_directory: EnvironmentPath, + /// A path outside the server's filesystem root, when the root is + /// narrowed; checks that it is refused with `permissionDenied`. + pub outside_root: Option, + /// Bound on waiting for a command to exit. Default 30 s. + pub command_timeout: Duration, +} +impl DataConformance { + pub fn new(client: DataClient, work_directory: EnvironmentPath) -> Self { + Self { + client, + work_directory, + outside_root: None, + command_timeout: Duration::from_secs(30), + } + } +} + +impl fmt::Debug for DataConformance { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DataConformance") + .field("work_directory", &self.work_directory) + .field("outside_root", &self.outside_root) + .finish_non_exhaustive() + } +} + +/// The outcome of one check. #[derive(Clone, Debug)] -pub struct EnvironmentDataConformanceOptions { - pub initialize: InitializeParams, - pub expected_capabilities: EnvironmentCapabilities, - pub expected_default_cwd: FsPath, - /// A relative directory that the suite may create and remove. - pub test_directory: FsPath, - /// An absolute path outside the provider's exposed filesystem route. - pub forbidden_path: FsPath, +pub struct CheckResult { + /// Stable check name, such as `fs.writeFile.preconditions`. + pub name: &'static str, + /// Why the check failed. + pub failure: Option, } -#[derive(Debug, Error)] -pub enum EnvironmentDataConformanceError { - #[error(transparent)] - Client(#[from] EnvironmentClientError), +/// Every check that ran, in order. +#[derive(Clone, Debug, Default)] +pub struct DataConformanceReport { + pub checks: Vec, +} - #[error(transparent)] - Filesystem(#[from] FsError), +impl DataConformanceReport { + pub fn passed(&self) -> bool { + self.checks.iter().all(|check| check.failure.is_none()) + } - #[error(transparent)] - Path(#[from] FsPathError), + pub fn failures(&self) -> impl Iterator { + self.checks.iter().filter(|check| check.failure.is_some()) + } - #[error(transparent)] - Tool(#[from] ToolError), + /// `Ok` when every check passed. + pub fn into_result(self) -> Result { + if self.passed() { + Ok(self) + } else { + Err(DataConformanceFailure(self)) + } + } +} + +/// A report with at least one failed check. +#[derive(Clone, Debug, Error)] +pub struct DataConformanceFailure(pub DataConformanceReport); + +impl fmt::Display for DataConformanceFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "data conformance failed:")?; + for check in self.0.failures() { + write!( + formatter, + "\n- {}: {}", + check.name, + check.failure.as_deref().unwrap_or_default() + )?; + } + Ok(()) + } +} + +type Check = Result<(), String>; + +/// Run every check that the server's capability families allow. +pub async fn run(suite: &DataConformance) -> DataConformanceReport { + let mut report = DataConformanceReport::default(); + let initialized = suite.client.initialize_response().clone(); + let handshake = check_initialize(&initialized); + let handshake_failed = handshake.is_err(); + record(&mut report, "initialize", handshake); + if handshake_failed { + return report; + } + record( + &mut report, + "env.getIdle", + call(suite.client.get_idle(&GetIdleParams {})) + .await + .map(drop), + ); - #[error("environment data conformance check failed: {0}")] - Check(String), + let work = match prepare(suite).await { + Ok(work) => work, + Err(failure) => { + record(&mut report, "fs.prepare", Err(failure)); + return report; + } + }; + let context = Context { + client: &suite.client, + initialized: &initialized, + work: &work, + timeout: suite.command_timeout, + }; + record( + &mut report, + "fs.writeFile.preconditions", + context.write_preconditions().await, + ); + record( + &mut report, + "fs.readFile.range", + context.ranged_reads().await, + ); + record( + &mut report, + "fs.readFile.notRegular", + context.reads_refuse_directories().await, + ); + record(&mut report, "fs.rename", context.rename().await); + record( + &mut report, + "fs.readDirectory.paging", + context.directory_paging().await, + ); + record(&mut report, "fs.remove", context.remove().await); + record( + &mut report, + "fs.errors", + context.errors(suite.outside_root.as_ref()).await, + ); + record( + &mut report, + "fs.globFiles.grammar", + context.glob_grammar().await, + ); + record( + &mut report, + "fs.glob.genericParity", + context.glob_parity().await, + ); + record(&mut report, "fs.searchText", context.search().await); + record(&mut report, "fs.scan.perRoot", context.scan().await); + if initialized.capabilities.process.is_some() { + record( + &mut report, + "fs.confinement.symlinks", + context + .symlink_confinement(suite.outside_root.as_ref()) + .await, + ); + record(&mut report, "process.exit", context.process_exit().await); + record( + &mut report, + "process.signal", + context.process_signal().await, + ); + record( + &mut report, + "process.largeInput", + context.process_large_input().await, + ); + record( + &mut report, + "process.duplicateId", + context.process_duplicate_id().await, + ); + record( + &mut report, + "process.unknown", + context.process_unknown().await, + ); + record( + &mut report, + "process.terminate", + context.process_terminate().await, + ); + } + if initialized.capabilities.job.is_some() { + record(&mut report, "job.requestId", context.job_request_id().await); + record(&mut report, "job.read.unknown", context.job_unknown().await); + record(&mut report, "job.read.paging", context.job_paging().await); + } + record( + &mut report, + "fs.cleanup", + call(suite.client.remove(&RemoveParams { + path: work, + recursive: true, + force: true, + })) + .await + .map(drop), + ); + report +} + +fn record(report: &mut DataConformanceReport, name: &'static str, outcome: Check) { + report.checks.push(CheckResult { + name, + failure: outcome.err(), + }); } -/// Exercises the transport handshake and the generic filesystem adapter. -/// -/// Provider implementations can invoke this from their own integration tests. -/// The supplied test directory is removed after a successful check. -pub async fn assert_environment_data_conformance( - mut client: EnvironmentDataClient, - options: EnvironmentDataConformanceOptions, -) -> Result<(), EnvironmentDataConformanceError> -where - T: JsonRpcTransport + Send + 'static, -{ - if options.test_directory.is_absolute() - || options.test_directory.is_root() - || options.test_directory.has_unresolved_parent() - || options.test_directory.segments().count() != 1 - { - return Err(check( - "test_directory must be a single relative path segment", - )); - } - if !options.forbidden_path.is_absolute() { - return Err(check("forbidden_path must be absolute")); - } - if !options.expected_capabilities.filesystem_read - || !options.expected_capabilities.filesystem_write - { - return Err(check( - "filesystem conformance requires read and write capabilities", - )); - } - - let initialized = client.initialize(&options.initialize).await?; - if initialized.protocol_version != CURRENT_PROTOCOL_VERSION { - return Err(check(format!( - "protocol version was {}, expected {CURRENT_PROTOCOL_VERSION}", +fn check_initialize(initialized: &InitializeResponse) -> Check { + ensure( + initialized.plane == Plane::Data, + "initialize did not name the data plane", + )?; + ensure( + initialized.protocol_version == ENVD_PROTOCOL_VERSION, + format!( + "protocolVersion {} differs from {ENVD_PROTOCOL_VERSION}", initialized.protocol_version - ))); - } - if initialized.capabilities != options.expected_capabilities { - return Err(check(format!( - "initialize capabilities were {:?}, expected {:?}", - initialized.capabilities, options.expected_capabilities - ))); - } - if initialized.default_cwd.as_deref() != Some(options.expected_default_cwd.as_str()) { - return Err(check(format!( - "default cwd was {:?}, expected {}", - initialized.default_cwd, options.expected_default_cwd - ))); - } - client.initialized(&InitializedParams {}).await?; - - let connection = RemoteEnvironmentConnection::new(client, initialized.capabilities) - .with_cwd(options.expected_default_cwd.clone()); - if connection.process_executor().is_some() != options.expected_capabilities.process_start { - return Err(check( - "process executor did not follow process_start capability", - )); - } - let has_any_job_capability = options.expected_capabilities.job_start - || options.expected_capabilities.job_list - || options.expected_capabilities.job_read - || options.expected_capabilities.job_cancel; - if connection.job_executor().is_some() != has_any_job_capability { - return Err(check("job executor did not follow advertised capabilities")); - } - - let (fs_context, _) = connection.into_contexts(Arc::new(InMemoryBlobStore::new())); - if fs_context.fs_cwd.as_ref() != Some(&options.expected_default_cwd) { - return Err(check("filesystem tool context did not retain default cwd")); - } - if fs_context.fs.access_policy() != FileAccessPolicy::FullReadWrite { - return Err(check( - "writable environment filesystem was not exposed as read-write", - )); - } - - let nested = options.test_directory.join("nested")?; - let source = nested.join("source.txt")?; - let copy = nested.join("copy.txt")?; - let copied_tree = FsPath::new(format!("{}-copy", options.test_directory))?; - let contents = "environment data conformance\n"; - - let write = invoke_write_file( - &fs_context, - WriteFileArgs { - path: source.clone(), - content: Some(contents.to_owned()), - content_ref: None, - }, - ) - .await?; - let expected_source = options.expected_default_cwd.join_path(&source)?; - if write.resolved_path != expected_source { - return Err(check(format!( - "relative tool path resolved to {}, expected {expected_source}", - write.resolved_path - ))); - } - - let read = invoke_read_file( - &fs_context, - ReadFileArgs { - path: source.clone(), - offset: None, - limit: None, - }, - ) + ), + )?; + ensure( + initialized.limits.max_concurrent_requests >= MIN_CONCURRENT_REQUESTS, + "maxConcurrentRequests is below the protocol minimum", + )?; + let filesystem = initialized + .capabilities + .filesystem + .as_ref() + .ok_or("the suite needs the filesystem family")?; + ensure(filesystem.write, "the suite needs a writable filesystem") +} + +async fn prepare(suite: &DataConformance) -> Result { + let work = join( + &suite.work_directory, + &format!("conformance-{}", uuid::Uuid::new_v4().simple()), + )?; + call(suite.client.create_directory(&CreateDirectoryParams { + path: work.clone(), + recursive: false, + })) .await?; - if read.text != contents.trim_end() { - return Err(check( - "read_file did not return bytes written by write_file", - )); - } - - let source_metadata = fs_context.fs.get_metadata(&expected_source).await?; - if !source_metadata.is_file || source_metadata.is_directory { - return Err(check("file metadata did not identify a regular file")); - } - let expected_nested = options.expected_default_cwd.join_path(&nested)?; - let directory_metadata = fs_context.fs.get_metadata(&expected_nested).await?; - if !directory_metadata.is_directory || directory_metadata.is_file { - return Err(check("directory metadata did not identify a directory")); - } - if !matches!( - fs_context - .fs - .create_directory(&expected_nested, CreateDirectoryOptions::single()) - .await, - Err(FsError::AlreadyExists { .. }) - ) { - return Err(check( - "provider conflict did not map to FsError::AlreadyExists", - )); - } - let entries = fs_context.fs.read_directory(&expected_nested).await?; - if !entries - .iter() - .any(|entry| entry.file_name == "source.txt" && entry.is_file) - { - return Err(check("read_directory omitted the written file")); - } - - let expected_copy = options.expected_default_cwd.join_path(©)?; - fs_context - .fs - .copy(&expected_source, &expected_copy, CopyOptions::file()) + Ok(work) +} + +struct Context<'a> { + client: &'a DataClient, + initialized: &'a InitializeResponse, + work: &'a EnvironmentPath, + timeout: Duration, +} + +impl Context<'_> { + fn path(&self, relative: &str) -> Result { + let tree = TreePath::new(relative).map_err(|error| error.to_string())?; + self.work.join(&tree).map_err(|error| error.to_string()) + } + + async fn write(&self, relative: &str, data: &[u8]) -> Result { + call(self.client.write_file(&WriteFileParams { + path: self.path(relative)?, + data: ByteChunk::from(data), + create: WriteCondition::Always, + if_match: None, + executable: false, + })) + .await + } + + async fn mkdir(&self, relative: &str) -> Check { + call(self.client.create_directory(&CreateDirectoryParams { + path: self.path(relative)?, + recursive: true, + })) + .await + .map(drop) + } + + async fn write_preconditions(&self) -> Check { + let path = self.path("precondition.txt")?; + let written = self.write("precondition.txt", b"one").await?; + let read = call(self.client.read_file(&ReadFileParams { + path: path.clone(), + offset: 0, + max_bytes: 16, + })) + .await?; + ensure( + read.version == written.version, + "a read after a write reports a different version", + )?; + let absent = self + .client + .write_file(&WriteFileParams { + path: path.clone(), + data: ByteChunk::from(&b"two"[..]), + create: WriteCondition::IfAbsent, + if_match: None, + executable: false, + }) + .await; + expect_precondition(absent, PreconditionReason::CreateCondition)?; + let rewritten = call(self.client.write_file(&WriteFileParams { + path: path.clone(), + data: ByteChunk::from(&b"two"[..]), + create: WriteCondition::IfPresent, + if_match: Some(written.version.clone()), + executable: false, + })) + .await?; + ensure( + rewritten.version != written.version, + "a changed file kept its version", + )?; + let stale = self + .client + .write_file(&WriteFileParams { + path, + data: ByteChunk::from(&b"three"[..]), + create: WriteCondition::Always, + if_match: Some(written.version), + executable: false, + }) + .await; + expect_precondition(stale, PreconditionReason::IfMatchFailed) + } + + async fn ranged_reads(&self) -> Check { + self.write("range.txt", b"0123456789").await?; + let path = self.path("range.txt")?; + let middle = call(self.client.read_file(&ReadFileParams { + path: path.clone(), + offset: 2, + max_bytes: 3, + })) .await?; - if fs_context.fs.read_file(&expected_copy).await? != contents.as_bytes() { - return Err(check("file copy did not preserve contents")); - } - - let expected_test_directory = options - .expected_default_cwd - .join_path(&options.test_directory)?; - let expected_copied_tree = options.expected_default_cwd.join_path(&copied_tree)?; - fs_context - .fs - .copy( - &expected_test_directory, - &expected_copied_tree, - CopyOptions::recursive(), + ensure( + middle.data.as_slice() == b"234", + "offset and maxBytes select the range", + )?; + ensure(middle.file_size == 10, "fileSize is the whole file's size")?; + ensure( + middle.truncated, + "truncated is set when bytes follow the range", + )?; + let end = call(self.client.read_file(&ReadFileParams { + path: path.clone(), + offset: 7, + max_bytes: 10, + })) + .await?; + ensure( + end.data.as_slice() == b"789" && !end.truncated, + "a range reaching the end is not truncated", + )?; + let ceiling = self.initialized.limits.filesystem.max_read_file_bytes; + let above = self + .client + .read_file(&ReadFileParams { + path, + offset: 0, + max_bytes: ceiling + 1, + }) + .await; + match above { + Err(error) => match error.protocol_error().map(|error| &error.data) { + Some(ErrorData::InvalidParams { + ceiling: Some(reported), + .. + }) if *reported == ceiling => Ok(()), + _ => Err(format!("maxBytes above the ceiling answered {error}")), + }, + Ok(_) => Err("maxBytes above the ceiling was accepted, not refused".into()), + } + } + + async fn reads_refuse_directories(&self) -> Check { + self.mkdir("directory").await?; + let read = self + .client + .read_file(&ReadFileParams { + path: self.path("directory")?, + offset: 0, + max_bytes: 16, + }) + .await; + match read { + Err(error) + if matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::FailedPrecondition { + reason: PreconditionReason::IsDirectory + | PreconditionReason::NotRegularFile + }) + ) => + { + Ok(()) + } + other => Err(format!("reading a directory answered {other:?}")), + } + } + + async fn rename(&self) -> Check { + self.write("rename-a.txt", b"a").await?; + self.write("rename-b.txt", b"b").await?; + let rename = |source: &str, destination: &str, overwrite| { + Ok::<_, String>(RenameParams { + source: self.path(source)?, + destination: self.path(destination)?, + overwrite, + if_match: None, + }) + }; + expect_kind( + self.client + .rename(&rename("rename-a.txt", "rename-b.txt", false)?) + .await, + ProtocolErrorKind::AlreadyExists, + )?; + call( + self.client + .rename(&rename("rename-a.txt", "rename-c.txt", false)?), ) .await?; - let copied_source = expected_copied_tree.join("nested/source.txt")?; - if fs_context.fs.read_file(&copied_source).await? != contents.as_bytes() { - return Err(check("recursive directory copy did not preserve contents")); - } - - let missing = options - .expected_default_cwd - .join("missing-conformance-file")?; - if !matches!( - fs_context.fs.read_file(&missing).await, - Err(FsError::NotFound { .. }) - ) { - return Err(check("provider notFound did not map to FsError::NotFound")); - } - if !matches!( - fs_context.fs.read_file(&options.forbidden_path).await, - Err(FsError::PermissionDenied { .. }) - ) { - return Err(check( - "route escape did not map to FsError::PermissionDenied", - )); - } - - fs_context - .fs - .remove(&expected_copy, RemoveOptions::file()) + expect_kind( + self.client + .rename(&rename("rename-a.txt", "rename-d.txt", false)?) + .await, + ProtocolErrorKind::NotFound, + )?; + let moved = call(self.client.read_file(&ReadFileParams { + path: self.path("rename-c.txt")?, + offset: 0, + max_bytes: 16, + })) .await?; - fs_context - .fs - .remove(&expected_copied_tree, RemoveOptions::recursive()) + ensure( + moved.data.as_slice() == b"a", + "rename did not move the content", + ) + } + + async fn directory_paging(&self) -> Check { + self.mkdir("listing").await?; + for name in ["c", "a", "b"] { + self.write(&format!("listing/{name}"), b"").await?; + } + let mut names = Vec::new(); + let mut cursor = None; + for _ in 0..5 { + let page = call(self.client.read_directory(&ReadDirectoryParams { + path: self.path("listing")?, + limit: 1, + cursor, + })) + .await?; + ensure(page.entries.len() <= 1, "a page exceeded its limit")?; + names.extend( + page.entries + .into_iter() + .map(|entry| entry.name.as_str().to_owned()), + ); + cursor = page.next_cursor; + if cursor.is_none() { + break; + } + } + ensure( + names == ["a", "b", "c"], + format!("paged listing returned {names:?}, not the names in order"), + ) + } + + async fn remove(&self) -> Check { + self.mkdir("removal").await?; + self.write("removal/file", b"x").await?; + let remove = |recursive, force, relative: &str| { + Ok::<_, String>(RemoveParams { + path: self.path(relative)?, + recursive, + force, + }) + }; + expect_precondition( + self.client.remove(&remove(false, false, "removal")?).await, + PreconditionReason::NotEmpty, + )?; + call(self.client.remove(&remove(true, false, "removal")?)).await?; + call(self.client.remove(&remove(false, true, "removal")?)).await?; + expect_kind( + self.client.remove(&remove(false, false, "removal")?).await, + ProtocolErrorKind::NotFound, + ) + } + + async fn errors(&self, outside_root: Option<&EnvironmentPath>) -> Check { + expect_kind( + self.client + .read_file(&ReadFileParams { + path: self.path("missing")?, + offset: 0, + max_bytes: 1, + }) + .await, + ProtocolErrorKind::NotFound, + )?; + if let Some(outside) = outside_root { + let read = self + .client + .read_file(&ReadFileParams { + path: outside.clone(), + offset: 0, + max_bytes: 1, + }) + .await; + match read { + Err(error) + if matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::PermissionDenied { + reason: Some(PermissionReason::OutsideRoot) + }) + ) => {} + other => return Err(format!("a path outside the root answered {other:?}")), + } + } + Ok(()) + } + + async fn glob_grammar(&self) -> Check { + self.mkdir("glob/sub").await?; + self.write("glob/a.rs", b"").await?; + self.write("glob/sub/b.rs", b"").await?; + let ceilings = self.initialized.limits.filesystem.glob; + let glob = |include: &str, max_depth| GlobFilesParams { + root: self.path("glob").expect("glob root"), + filter: TreeFilter { + include: vec![include.to_owned()], + max_depth, + ..TreeFilter::default() + }, + limits: GlobFilesLimits { + max_matches: ceilings.max_matches.min(100), + max_files: ceilings.max_files.min(1000), + max_duration_ms: ceilings.max_duration_ms.min(10_000), + }, + }; + let names = |matches: Vec| { + matches + .iter() + .filter_map(|path| path.file_name().map(str::to_owned)) + .collect::>() + }; + let segment = call(self.client.glob_files(&glob("*.rs", None))).await?; + ensure( + names(segment.matches) == ["a.rs", "b.rs"], + "a pattern without '/' matches names at any depth", + )?; + let anchored = call(self.client.glob_files(&glob("/*.rs", None))).await?; + ensure( + names(anchored.matches) == ["a.rs"], + "'*' stays within one segment and a leading '/' anchors", + )?; + let shallow = call(self.client.glob_files(&glob("**/*.rs", Some(1)))).await?; + ensure( + names(shallow.matches) == ["a.rs"], + "maxDepth 1 selects the root's direct children", + ) + } + + /// The tools' `glob` over this server and over an in-memory filesystem + /// holding the same tree select the same files: the generic fallback + /// follows the grammar and the walk defaults of `fs/globFiles`. + async fn glob_parity(&self) -> Check { + use crate::fs::{ + CreateDirectoryOptions, FileSystem as _, FsPath, FsToolContext, InMemoryFileSystem, + tools::{GlobArgs, invoke_glob}, + }; + + const FILES: [&str; 10] = [ + "parity/a.rs", + "parity/notes.md", + "parity/sub/b.rs", + "parity/sub/deep/c.rs", + "parity/.hidden/d.rs", + "parity/.e.rs", + "parity/build/out.rs", + "parity/skipped.rs", + "parity/sub/.gitignore", + "parity/.ignore", + ]; + const PATTERNS: [&str; 8] = [ + "*.rs", + "/*.rs", + "sub/*.rs", + "**/*.rs", + "sub/**", + "build/", + "*.{rs,md}", + "deep/*.rs", + ]; + let memory = InMemoryFileSystem::full_access(); + let local_root = FsPath::new("/parity").map_err(|error| error.to_string())?; + for file in FILES { + let contents: &[u8] = match file { + "parity/sub/.gitignore" => b"deep/\n", + "parity/.ignore" => b"skipped.rs\n", + _ => b"", + }; + if let Some((directory, _)) = file.rsplit_once('/') { + self.mkdir(directory).await?; + memory + .create_directory( + &FsPath::new(format!("/{directory}")).map_err(|error| error.to_string())?, + CreateDirectoryOptions::recursive(), + ) + .await + .map_err(|error| error.to_string())?; + } + self.write(file, contents).await?; + memory + .write_file( + &FsPath::new(format!("/{file}")).map_err(|error| error.to_string())?, + contents.to_vec(), + ) + .await + .map_err(|error| error.to_string())?; + } + let blobs: std::sync::Arc = + std::sync::Arc::new(harness::storage::InMemoryBlobStore::new()); + let remote_root = + FsPath::new(self.path("parity")?.as_str()).map_err(|error| error.to_string())?; + let remote = FsToolContext::new( + std::sync::Arc::new( + super::RemoteEnvironmentConnection::new(self.client.clone()).file_system(), + ), + blobs.clone(), + ); + let local = FsToolContext::new(std::sync::Arc::new(memory), blobs); + let relative = |root: &FsPath, matches: Vec| { + matches + .iter() + .map(|path| { + path.as_str() + .strip_prefix(root.as_str()) + .unwrap_or(path.as_str()) + .to_owned() + }) + .collect::>() + }; + for pattern in PATTERNS { + let args = |root: &FsPath| GlobArgs { + pattern: pattern.to_owned(), + path: Some(root.clone()), + max_depth: None, + limit: None, + }; + let native = invoke_glob(&remote, args(&remote_root)) + .await + .map_err(|error| format!("glob {pattern:?} on the server: {error}"))?; + let generic = invoke_glob(&local, args(&local_root)) + .await + .map_err(|error| format!("generic glob {pattern:?}: {error}"))?; + let native = relative(&remote_root, native.matches); + let generic = relative(&local_root, generic.matches); + ensure( + native == generic, + format!( + "glob {pattern:?} selected {native:?} on the server but {generic:?} in the generic fallback" + ), + )?; + } + Ok(()) + } + + async fn search(&self) -> Check { + self.mkdir("search").await?; + self.write("search/notes.txt", b"alpha\nneedle here\n") + .await?; + let ceilings = self.initialized.limits.filesystem.search; + let response = call(self.client.search_text(&SearchTextParams { + root: self.path("search")?, + pattern: "NEEDLE".into(), + pattern_ignore_case: true, + filter: TreeFilter { + include: vec!["*.txt".into()], + ..TreeFilter::default() + }, + limits: SearchTextLimits { + max_matches: ceilings.max_matches.min(10), + max_files: ceilings.max_files.min(100), + max_bytes: ceilings.max_bytes.min(1 << 20), + max_duration_ms: ceilings.max_duration_ms.min(10_000), + max_line_bytes: ceilings.max_line_bytes, + }, + })) .await?; - fs_context - .fs - .remove(&expected_test_directory, RemoveOptions::recursive()) + ensure( + matches!(response.matches.as_slice(), [found] if found.line_number == 2 && found.line == "needle here"), + format!("search answered {:?}", response.matches), + ) + } + + async fn scan(&self) -> Check { + self.mkdir("scan").await?; + self.write("scan/one.md", b"# one").await?; + let ceilings = self.initialized.limits.filesystem.scan; + let response = call(self.client.scan(&ScanParams { + roots: vec![ + ScanRoot { + path: self.path("scan")?, + if_none_match: None, + max_depth: None, + }, + ScanRoot { + path: self.path("scan-missing")?, + if_none_match: None, + max_depth: None, + }, + ScanRoot { + path: self.work.clone(), + if_none_match: None, + max_depth: Some(0), + }, + ], + filter: TreeFilter { + include: vec!["*.md".into()], + ..TreeFilter::default() + }, + read_content: true, + include_digests: false, + follow_symlinks: false, + limits: ScanLimits { + max_entries: ceilings.max_entries.min(100), + max_file_bytes: ceilings.max_file_bytes.min(4096), + max_total_bytes: ceilings.max_total_bytes.min(65536), + max_duration_ms: ceilings.max_duration_ms.min(10_000), + }, + })) .await?; - if !matches!( - fs_context.fs.get_metadata(&expected_test_directory).await, - Err(FsError::NotFound { .. }) - ) { - return Err(check("recursive remove left the test directory behind")); + match response.roots.as_slice() { + [ + ScanRootResult::Complete { entries, .. }, + ScanRootResult::Missing { .. }, + ScanRootResult::Complete { entries: probe, .. }, + ] if entries.len() == 1 + && entries[0].data.as_ref().map(ByteChunk::as_slice) == Some(&b"# one"[..]) + && probe.is_empty() => + { + Ok(()) + } + other => Err(format!("scan answered {other:?}")), + } + } + + /// An in-root symlink is followed; one whose target leaves the root is + /// refused with `permissionDenied {outsideRoot}` by `fs/readFile` and a + /// following `fs/getMetadata`, while the link itself stays describable. + async fn symlink_confinement(&self, outside_root: Option<&EnvironmentPath>) -> Check { + self.write("link-target.txt", b"inside").await?; + let mut script = "ln -s link-target.txt link-inside".to_owned(); + if let Some(outside) = outside_root { + script.push_str(&format!(" && ln -s '{outside}' link-outside")); + } + let process_id = self.start(&script, ExecInput::None {}).await?; + let (_, exited) = self.read_to_exit(&process_id).await?; + ensure( + exited.exit == Some(ExitStatus::Code { code: 0 }), + format!("creating the links exited {:?}", exited.exit), + )?; + let inside = call(self.client.read_file(&ReadFileParams { + path: self.path("link-inside")?, + offset: 0, + max_bytes: 16, + })) + .await?; + ensure( + inside.data.as_slice() == b"inside", + "reading an in-root link did not read its target", + )?; + let followed = call(self.client.get_metadata(&GetMetadataParams { + path: self.path("link-inside")?, + follow_symlinks: true, + })) + .await?; + ensure( + followed.metadata.kind == FileKind::File, + format!( + "a followed in-root link described a {:?}", + followed.metadata.kind + ), + )?; + if outside_root.is_none() { + return Ok(()); + } + let outside = self.path("link-outside")?; + let link = call(self.client.get_metadata(&GetMetadataParams { + path: outside.clone(), + follow_symlinks: false, + })) + .await?; + ensure( + link.metadata.kind == FileKind::Symlink, + "an escaping link is still describable as a symlink", + )?; + let outside_root = |result: Result<_, ClientError>, what: &str| match result { + Err(error) + if matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::PermissionDenied { + reason: Some(PermissionReason::OutsideRoot) + }) + ) => + { + Ok(()) + } + Err(error) => Err(format!("{what} through an escaping link answered {error}")), + Ok(_) => Err(format!("{what} followed a link that leaves the root")), + }; + outside_root( + self.client + .read_file(&ReadFileParams { + path: outside.clone(), + offset: 0, + max_bytes: 16, + }) + .await + .map(drop), + "fs/readFile", + )?; + outside_root( + self.client + .get_metadata(&GetMetadataParams { + path: outside, + follow_symlinks: true, + }) + .await + .map(drop), + "fs/getMetadata {followSymlinks}", + ) + } + + fn process_id(&self) -> ProcessId { + ProcessId::new(format!("conformance-{}", uuid::Uuid::new_v4().simple())) + .expect("generated ids fit the grammar") + } + + async fn start(&self, script: &str, input: ExecInput) -> Result { + let process_id = self.process_id(); + call(self.client.start_process(&StartProcessParams { + process_id: process_id.clone(), + command: shell(script, input, self.work), + pty: None, + })) + .await?; + Ok(process_id) + } + + /// Read from the cursor until the exit has settled and all output is in. + async fn read_to_exit( + &self, + process_id: &ProcessId, + ) -> Result<(Vec, ReadProcessResponse), String> { + let deadline = Instant::now() + self.timeout; + let mut output = Vec::new(); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err("the command did not exit in time".into()); + } + let response = call(self.client.read_process(&ReadProcessParams { + process_id: process_id.clone(), + after: None, + tail_bytes: None, + ack: None, + max_bytes: self.initialized.limits.process.max_read_bytes, + wait_ms: (remaining.as_millis() as u64).min(self.initialized.limits.max_wait_ms), + return_on: ReturnOn::Exit, + })) + .await?; + ensure( + response.output.gaps.is_empty(), + "output gaps in a short command", + )?; + for chunk in &response.output.chunks { + output.extend_from_slice(chunk.data.as_slice()); + } + if response.state == ExecState::Exited && response.output.next == response.output.end { + return Ok((output, response)); + } + } + } + + async fn process_exit(&self) -> Check { + let process_id = self + .start("printf out; printf err >&2; exit 3", ExecInput::None {}) + .await?; + let (output, response) = self.read_to_exit(&process_id).await?; + ensure( + response.exit == Some(ExitStatus::Code { code: 3 }), + format!("exit was {:?}, not code 3", response.exit), + )?; + let text = String::from_utf8_lossy(&output); + ensure( + text.contains("out") && text.contains("err"), + "stdout and stderr both reach the output", + ) + } + + async fn process_signal(&self) -> Check { + let process_id = self.start("kill -TERM $$", ExecInput::None {}).await?; + let (_, response) = self.read_to_exit(&process_id).await?; + match &response.exit { + Some(ExitStatus::Signal { signal, .. }) if signal.as_str() == "SIGTERM" => Ok(()), + other => Err(format!("a signal death was reported as {other:?}")), + } + } + + async fn process_large_input(&self) -> Check { + let input = vec![b'x'; 1 << 20]; + let process_id = self + .start( + "cat", + ExecInput::Bytes { + data: ByteChunk::from(input.clone()), + }, + ) + .await?; + let (output, response) = self.read_to_exit(&process_id).await?; + ensure( + response.output.end == input.len() as u64 && output == input, + format!( + "cat echoed {} of {} input bytes", + response.output.end, + input.len() + ), + )?; + let tail = self + .client + .read_process(&ReadProcessParams { + process_id, + after: None, + tail_bytes: Some(10), + ack: None, + max_bytes: 10, + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + }) + .await; + match tail { + // A fully read process may already be released. + Err(error) if error.protocol_kind() == Some(ProtocolErrorKind::NotFound) => Ok(()), + Err(error) => Err(error.to_string()), + Ok(tail) => ensure( + tail.output.next == tail.output.end + && tail + .output + .chunks + .first() + .is_none_or(|chunk| chunk.offset >= tail.output.end - 10), + "a tail read starts at end - tailBytes", + ), + } + } + + async fn process_duplicate_id(&self) -> Check { + let marker = self.path("duplicate-marker")?; + let script = format!("echo ran >> '{marker}'"); + let process_id = self.start("sleep 30", ExecInput::None {}).await?; + let duplicate = self + .client + .start_process(&StartProcessParams { + process_id: process_id.clone(), + command: shell(&script, ExecInput::None {}, self.work), + pty: None, + }) + .await; + let _ = self + .client + .terminate_process(&TerminateProcessParams { + process_id, + signal: ProcessSignal::Kill, + }) + .await; + expect_kind(duplicate, ProtocolErrorKind::AlreadyExists)?; + expect_kind( + self.client + .read_file(&ReadFileParams { + path: marker, + offset: 0, + max_bytes: 16, + }) + .await, + ProtocolErrorKind::NotFound, + ) + .map_err(|_| "a refused duplicate start ran its command".to_owned()) + } + + async fn process_unknown(&self) -> Check { + let unknown = self.process_id(); + expect_kind( + self.client + .read_process(&ReadProcessParams { + process_id: unknown.clone(), + after: None, + tail_bytes: None, + ack: None, + max_bytes: 1, + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + }) + .await, + ProtocolErrorKind::NotFound, + )?; + expect_kind( + self.client + .write_process(&WriteProcessParams { + process_id: unknown.clone(), + data: ByteChunk::from(&b"x"[..]), + close_input: false, + }) + .await, + ProtocolErrorKind::NotFound, + )?; + expect_kind( + self.client + .terminate_process(&TerminateProcessParams { + process_id: unknown, + signal: ProcessSignal::Kill, + }) + .await, + ProtocolErrorKind::NotFound, + ) } - Ok(()) + async fn process_terminate(&self) -> Check { + let process_id = self.start("sleep 60", ExecInput::None {}).await?; + let response = call(self.client.terminate_process(&TerminateProcessParams { + process_id: process_id.clone(), + signal: ProcessSignal::Terminate, + })) + .await?; + ensure( + response.running, + "terminate of a running process reported it not running", + )?; + let (_, exited) = self.read_to_exit(&process_id).await?; + ensure( + exited.stopped_by == Some(StoppedBy::Terminate), + format!("stoppedBy was {:?}", exited.stopped_by), + ) + } + + fn job(&self, job_id: &JobId, script: &str) -> JobSpec { + JobSpec { + job_id: job_id.clone(), + name: None, + command: shell(script, ExecInput::None {}, self.work), + depends_on: Vec::new(), + dependency_policy: Default::default(), + queue_key: None, + } + } + + fn job_ids(&self) -> (JobId, JobRequestId, JobRequestId) { + let suffix = uuid::Uuid::new_v4().simple(); + ( + JobId::new(format!("conformance-{suffix}")).expect("job id"), + JobRequestId::new(format!("request-a-{suffix}")).expect("request id"), + JobRequestId::new(format!("request-b-{suffix}")).expect("request id"), + ) + } + + async fn job_request_id(&self) -> Check { + let (job_id, first, second) = self.job_ids(); + let start = |request_id: &JobRequestId| StartJobsParams { + request_id: request_id.clone(), + jobs: vec![self.job(&job_id, "true")], + }; + let started = call(self.client.start_jobs(&start(&first))).await?; + let replayed = call(self.client.start_jobs(&start(&first))).await?; + ensure( + started.jobs[0].created_at_ms == replayed.jobs[0].created_at_ms, + "a retried start with the same requestId created another run", + )?; + expect_precondition( + self.client.start_jobs(&start(&second)).await, + PreconditionReason::JobIdInUse, + ) + } + + async fn job_unknown(&self) -> Check { + let (job_id, _, _) = self.job_ids(); + let response = call(self.client.read_jobs(&ReadJobsParams { + jobs: vec![JobReadRequest { + job_id, + after: None, + tail_bytes: None, + max_bytes: None, + }], + max_bytes: 0, + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + })) + .await?; + match response.jobs.as_slice() { + [JobReadEntry::Error { error, .. }] if error.kind() == ProtocolErrorKind::NotFound => { + Ok(()) + } + other => Err(format!("an unknown job read answered {other:?}")), + } + } + + async fn job_paging(&self) -> Check { + let (job_id, request_id, _) = self.job_ids(); + // 20 000 bytes: 2 000 numbered lines of 10 bytes each. + let script = "i=0; while [ $i -lt 2000 ]; do printf '%09d\\n' $i; i=$((i+1)); done"; + call(self.client.start_jobs(&StartJobsParams { + request_id, + jobs: vec![self.job(&job_id, script)], + })) + .await?; + let deadline = Instant::now() + self.timeout; + let mut output = Vec::new(); + let mut after = 0; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err("the job did not finish in time".into()); + } + let response = call(self.client.read_jobs(&ReadJobsParams { + jobs: vec![JobReadRequest { + job_id: job_id.clone(), + after: Some(after), + tail_bytes: None, + max_bytes: None, + }], + max_bytes: 4096, + wait_ms: (remaining.as_millis() as u64).min(1000), + return_on: ReturnOn::Window, + })) + .await?; + let [ + JobReadEntry::Job { + summary, + output: page, + }, + ] = response.jobs.as_slice() + else { + return Err(format!("job read answered {:?}", response.jobs)); + }; + ensure(page.gaps.is_empty(), "a 20 000-byte job reported gaps")?; + for chunk in &page.chunks { + ensure( + chunk.offset == output.len() as u64, + "pages skipped or repeated bytes", + )?; + output.extend_from_slice(chunk.data.as_slice()); + } + after = page.next; + if summary.status.is_terminal() && page.next == page.end { + break; + } + } + let expected = (0..2000).map(|i| format!("{i:09}\n")).collect::(); + ensure( + output == expected.as_bytes(), + format!( + "paged output was {} bytes, not 20 000 exact bytes", + output.len() + ), + ) + } +} + +fn shell(script: &str, input: ExecInput, cwd: &EnvironmentPath) -> CommandSpec { + CommandSpec { + argv: vec!["/bin/sh".into(), "-c".into(), script.into()], + cwd: Some(cwd.clone()), + env: BTreeMap::new(), + secret_env: BTreeMap::new(), + input, + timeout_ms: Some(120_000), + } +} + +fn join(base: &EnvironmentPath, name: &str) -> Result { + let tree = TreePath::new(name).map_err(|error| error.to_string())?; + base.join(&tree).map_err(|error| error.to_string()) +} + +fn ensure(condition: bool, message: impl Into) -> Check { + if condition { + Ok(()) + } else { + Err(message.into()) + } +} + +async fn call(future: impl Future>) -> Result { + future.await.map_err(|error| error.to_string()) } -fn check(message: impl Into) -> EnvironmentDataConformanceError { - EnvironmentDataConformanceError::Check(message.into()) +fn expect_kind(result: Result, kind: ProtocolErrorKind) -> Check { + match result { + Err(error) if error.protocol_kind() == Some(kind) => Ok(()), + other => Err(format!("expected {kind}, got {other:?}")), + } +} + +fn expect_precondition( + result: Result, + reason: PreconditionReason, +) -> Check { + match &result { + Err(error) + if matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::FailedPrecondition { reason: actual }) if *actual == reason + ) => + { + Ok(()) + } + other => Err(format!( + "expected failedPrecondition ({reason:?}), got {other:?}" + )), + } } diff --git a/crates/tools/src/environment_protocol/mod.rs b/crates/tools/src/environment_protocol/mod.rs index 72eda9b9a..127ef5a0b 100644 --- a/crates/tools/src/environment_protocol/mod.rs +++ b/crates/tools/src/environment_protocol/mod.rs @@ -3,8 +3,8 @@ pub mod conformance; pub mod remote; -pub use conformance::{ - EnvironmentDataConformanceError, EnvironmentDataConformanceOptions, - assert_environment_data_conformance, +pub use conformance::{DataConformance, DataConformanceFailure, DataConformanceReport}; +pub use remote::{ + RemoteEnvironmentConnection, RemoteEnvironmentFileSystem, RemoteJobExecutor, + RemoteProcessExecutor, }; -pub use remote::{RemoteEnvironmentConnection, RemoteEnvironmentFileSystem, RemoteProcessExecutor}; diff --git a/crates/tools/src/environment_protocol/remote.rs b/crates/tools/src/environment_protocol/remote.rs index 9ac7a60f9..85872ffb7 100644 --- a/crates/tools/src/environment_protocol/remote.rs +++ b/crates/tools/src/environment_protocol/remote.rs @@ -1,33 +1,59 @@ //! Remote environment adapters backed by `environment-client`. +//! +//! One [`DataClient`] serves every adapter of a connection: calls are +//! multiplexed, so a long process read never blocks a file read. The +//! daemon's `initialize` response decides which adapters exist (capability +//! families) and bounds every request (advertised ceilings). use std::{ sync::{ Arc, LazyLock, atomic::{AtomicU64, Ordering}, }, - time::{SystemTime, UNIX_EPOCH}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use async_trait::async_trait; -use environment_client::{EnvironmentClientError, EnvironmentDataClient, JsonRpcTransport}; +use environment_client::{ClientError, DataClient}; use environment_protocol::{ + bytes::ByteChunk, data::{ - fs as remote_fs, jobs as remote_jobs, + exec::{ + CommandSpec, ExecInput, ExecState, ExitStatus, OutputPage, PtySize, ReturnOn, StoppedBy, + }, + fs::{ + CreateDirectoryParams, FileKind, GetMetadataParams, GlobFilesLimits, GlobFilesParams, + GlobFilesStop, ReadDirectoryParams, ReadFileParams, RemoveParams, RenameParams, + SearchTextLimits, SearchTextParams, SearchTextStop, TreeFilter, WriteCondition, + WriteFileParams, + }, + handshake::{FilesystemLimits, InitializeResponse, JobLimits, TransferCeilings}, + job::{ + CancelJobsParams, CancelJobsResponse, ReadJobsParams, ReadJobsResponse, + StartJobsParams, StartJobsResponse, + }, process::{ - self as remote_process, ProcessOutputStream, ProcessTermination, ReadProcessResponse, - WriteProcessStatus, + ReadProcessParams, ReadProcessResponse, StartProcessParams, TerminateProcessParams, + WriteProcessParams, + }, + transfer::{ + AppendTransferInventoryParams, BeginTransferParams, ListTransferMissingParams, + ListTransferMissingResponse, ReadTransferContentParams, ReadTransferContentResponse, + ReadTransferInventoryParams, ReadTransferInventoryResponse, TransferOperationParams, + TransferStatus, WriteTransferContentParams, WriteTransferContentResponse, }, }, - error::EnvironmentProtocolErrorCode, - shared::{ByteChunk, EnvironmentCapabilities, EnvironmentPath, ProcessId}, + error::{Capability, ErrorData, PreconditionReason, ProtocolErrorKind}, + ids::ProcessId, + paths::EnvironmentPath, }; use harness::storage::BlobStore; -use tokio::sync::Mutex as AsyncMutex; use crate::{ environment::{ EnvironmentToolContext, jobs::{JobError, JobExecResult, JobExecutor}, + output::{ShownOutput, is_stderr}, process::{ ContinueProcessRequest, LeftoverProcess, ProcessError, ProcessExecResult, ProcessExecutor, ProcessHandle, ProcessOutput, ProcessRequest, ProcessSignal, @@ -35,13 +61,22 @@ use crate::{ }, }, fs::{ - CopyOptions, CreateDirectoryOptions, FileAccessPolicy, FileMetadata, FileSystem, FsError, - FsGlobRequest, FsGlobResponse, FsPath, FsRangedRead, FsResult, FsSearchStop, - FsTextSearchMatch, FsTextSearchRequest, FsTextSearchResponse, FsToolContext, - ReadDirectoryEntry, RemoveOptions, ranged_read_from_full, + CreateDirectoryOptions, FileAccessPolicy, FileMetadata, FileSystem, FsError, FsGlobRequest, + FsGlobResponse, FsPath, FsRangedRead, FsResult, FsSearchStop, FsTextSearchMatch, + FsTextSearchRequest, FsTextSearchResponse, FsToolContext, FsVersion, ReadDirectoryEntry, + RemoveOptions, RenameOptions, VersionedRead, WriteOptions, }, + transfer::EnvironmentTransfer, }; +/// Output a process read shows when the caller names no budget. +const DEFAULT_PROCESS_OUTPUT_BYTES: u64 = 512 * 1024; +/// Terminal size of a PTY the model asks for with `tty: true`. +const DEFAULT_PTY_SIZE: PtySize = PtySize { rows: 24, cols: 80 }; +/// How long input that the process does not read yet is retried before the +/// write gives up. +const INPUT_RETRY_WINDOW: Duration = Duration::from_secs(10); + static NEXT_PROCESS_EXECUTOR_ID: AtomicU64 = AtomicU64::new(1); static PROCESS_ID_NONCE: LazyLock = LazyLock::new(|| { let started_at_nanos = SystemTime::now() @@ -51,103 +86,73 @@ static PROCESS_ID_NONCE: LazyLock = LazyLock::new(|| { format!("{}-{started_at_nanos}", std::process::id()) }); -pub struct RemoteEnvironmentConnection { - client: Arc>>, - capabilities: EnvironmentCapabilities, +/// A data-plane connection and the tool adapters it supports. +#[derive(Clone)] +pub struct RemoteEnvironmentConnection { + client: DataClient, cwd: Option, } -impl Clone for RemoteEnvironmentConnection { - fn clone(&self) -> Self { - Self { - client: self.client.clone(), - capabilities: self.capabilities.clone(), - cwd: self.cwd.clone(), - } +impl RemoteEnvironmentConnection { + /// Adapters over an initialized client; capabilities and ceilings come + /// from its `initialize` response. + pub fn new(client: DataClient) -> Self { + Self { client, cwd: None } } -} -#[async_trait] -impl crate::transfer::EnvironmentTransfer - for RemoteEnvironmentConnection -{ - async fn request( - &self, - request: environment_protocol::data::transfer_session::TransferRequest, - ) -> crate::error::ToolResult - { - if !self.capabilities.filesystem_transfer { - return Err(crate::error::ToolError::UnsupportedCapability { - message: "environment transfer unavailable".into(), - }); - } - self.client - .lock() - .await - .transfer(&request) - .await - .map_err(|e| crate::error::ToolError::InvalidRequest { - message: e.to_string(), - }) + pub fn with_cwd(mut self, cwd: FsPath) -> Self { + self.cwd = Some(cwd); + self } -} -impl RemoteEnvironmentConnection -where - T: JsonRpcTransport + Send + 'static, -{ - pub fn new(client: EnvironmentDataClient, capabilities: EnvironmentCapabilities) -> Self { - Self { - client: Arc::new(AsyncMutex::new(client)), - capabilities, - cwd: None, - } + pub fn client(&self) -> &DataClient { + &self.client } - pub fn with_cwd(mut self, cwd: FsPath) -> Self { - self.cwd = Some(cwd); - self + pub fn initialize_response(&self) -> &InitializeResponse { + self.client.initialize_response() } - /// Gracefully shut down the shared data-plane transport after all tool - /// adapters derived from this connection have finished using it. - pub async fn close(&self) -> Result<(), EnvironmentClientError> { - self.client.lock().await.close().await + /// Close the shared connection once every adapter derived from it has + /// finished. + pub async fn close(&self) -> Result<(), ClientError> { + self.client.close().await } - pub fn file_system(&self) -> RemoteEnvironmentFileSystem { + pub fn file_system(&self) -> RemoteEnvironmentFileSystem { + let initialized = self.initialize_response(); RemoteEnvironmentFileSystem { client: self.client.clone(), - access_policy: access_policy_for_capabilities(&self.capabilities), - filesystem_search: self.capabilities.filesystem_search, - filesystem_glob: self.capabilities.filesystem_glob, - filesystem_ranged_read: self.capabilities.filesystem_ranged_read, + writable: initialized + .capabilities + .filesystem + .as_ref() + .is_some_and(|filesystem| filesystem.write), + limits: initialized.limits.filesystem.clone(), } } - pub fn process_executor(&self) -> Option> { - if !self.capabilities.process_start { - return None; - } - let process_id_prefix = NEXT_PROCESS_EXECUTOR_ID.fetch_add(1, Ordering::Relaxed); + /// Present when the daemon serves the `process` family. + pub fn process_executor(&self) -> Option { + let initialized = self.initialize_response(); + let process = initialized.capabilities.process.as_ref()?; Some(RemoteProcessExecutor { client: self.client.clone(), - process_id_prefix, + pty: process.pty, + max_read_bytes: initialized.limits.process.max_read_bytes, + max_wait_ms: initialized.limits.max_wait_ms, + process_id_prefix: NEXT_PROCESS_EXECUTOR_ID.fetch_add(1, Ordering::Relaxed), next_process_id: Arc::new(AtomicU64::new(1)), }) } - pub fn job_executor(&self) -> Option> { - if !self.capabilities.job_start - && !self.capabilities.job_list - && !self.capabilities.job_read - && !self.capabilities.job_cancel - { - return None; - } + /// Present when the daemon serves the `job` family. + pub fn job_executor(&self) -> Option { + let initialized = self.initialize_response(); + initialized.capabilities.job.as_ref()?; Some(RemoteJobExecutor { client: self.client.clone(), - capabilities: self.capabilities.clone(), + limits: initialized.limits.job.clone(), }) } @@ -155,6 +160,7 @@ where self, blobs: Arc, ) -> (FsToolContext, EnvironmentToolContext) { + let has_filesystem = self.initialize_response().capabilities.filesystem.is_some(); let fs = Arc::new(self.file_system()); let process = self .process_executor() @@ -164,8 +170,8 @@ where .map(|executor| Arc::new(executor) as Arc); let mut fs_ctx = FsToolContext::new(fs, blobs.clone()); let mut env_ctx = EnvironmentToolContext::new(process, blobs); - if self.capabilities.filesystem_transfer { - env_ctx.transfer = Some(Arc::new(self.clone())); + if has_filesystem { + env_ctx.transfer = Some(Arc::new(self.client.clone())); } if let Some(jobs) = jobs { env_ctx = env_ctx.with_jobs(jobs); @@ -174,46 +180,166 @@ where fs_ctx = fs_ctx.with_cwd(cwd.clone()); env_ctx = env_ctx.with_process_cwd(cwd); } - // Filesystem tools are exposed only when the daemon negotiated read - // access; write access is enforced by the file system's access policy. - if self.capabilities.filesystem_read { + // Filesystem tools exist only when the daemon serves the filesystem + // family; write access follows its `write` fact through the file + // system's access policy. + if has_filesystem { env_ctx = env_ctx.with_filesystem(fs_ctx.clone()); } (fs_ctx, env_ctx) } } -#[derive(Clone)] -pub struct RemoteEnvironmentFileSystem { - client: Arc>>, - access_policy: FileAccessPolicy, - filesystem_search: bool, - filesystem_glob: bool, - filesystem_ranged_read: bool, -} - #[async_trait] -impl FileSystem for RemoteEnvironmentFileSystem -where - T: JsonRpcTransport + Send + 'static, -{ - fn access_policy(&self) -> FileAccessPolicy { - self.access_policy.clone() +impl EnvironmentTransfer for DataClient { + fn transfer_ceilings(&self) -> TransferCeilings { + self.initialize_response().limits.filesystem.transfer } - async fn read_file(&self, path: &FsPath) -> FsResult> { + async fn begin(&self, params: &BeginTransferParams) -> Result { + self.transfer_begin(params).await + } + + async fn advance( + &self, + params: &TransferOperationParams, + ) -> Result { + self.transfer_advance(params).await + } + + async fn get_status( + &self, + params: &TransferOperationParams, + ) -> Result { + self.transfer_get_status(params).await + } + + async fn read_inventory( + &self, + params: &ReadTransferInventoryParams, + ) -> Result { + self.transfer_read_inventory(params).await + } + + async fn append( + &self, + params: &AppendTransferInventoryParams, + ) -> Result { + self.transfer_append(params).await + } + + async fn list_missing( + &self, + params: &ListTransferMissingParams, + ) -> Result { + self.transfer_list_missing(params).await + } + + async fn read( + &self, + params: &ReadTransferContentParams, + ) -> Result { + self.transfer_read(params).await + } + + async fn write( + &self, + params: &WriteTransferContentParams, + ) -> Result { + self.transfer_write(params).await + } + + async fn commit( + &self, + params: &TransferOperationParams, + ) -> Result { + self.transfer_commit(params).await + } + + async fn abort(&self, params: &TransferOperationParams) -> Result { + self.transfer_abort(params).await + } +} + +/// The environment filesystem over `fs/*`. Every bound comes from the +/// daemon's advertised ceilings: a read longer than one `fs/readFile` is +/// refused rather than returned short, and writes are staged and atomic. +#[derive(Clone)] +pub struct RemoteEnvironmentFileSystem { + client: DataClient, + writable: bool, + limits: FilesystemLimits, +} + +impl RemoteEnvironmentFileSystem { + async fn read_whole(&self, path: &FsPath) -> FsResult { let response = self .client - .lock() - .await - .read_file(&remote_fs::ReadFileParams { + .read_file(&ReadFileParams { path: environment_path(path)?, - offset: None, - max_bytes: None, + offset: 0, + max_bytes: self.limits.max_read_file_bytes, }) .await .map_err(|error| map_fs_error(error, path))?; - Ok(response.data.into_inner()) + if response.truncated { + return Err(FsError::InvalidInput { + message: format!( + "{path} is {} bytes; the environment returns at most {} bytes per read", + response.file_size, self.limits.max_read_file_bytes + ), + }); + } + Ok(VersionedRead { + bytes: response.data.into_inner(), + version: Some(FsVersion::new(response.version.as_str())), + }) + } + + async fn metadata(&self, path: &FsPath, follow_symlinks: bool) -> FsResult { + let metadata = self + .client + .get_metadata(&GetMetadataParams { + path: environment_path(path)?, + follow_symlinks, + }) + .await + .map_err(|error| map_fs_error(error, path))? + .metadata; + Ok(FileKindFacts { + kind: metadata.kind, + created_at_ms: metadata.created_at_ms.unwrap_or(0), + modified_at_ms: metadata.modified_at_ms, + }) + } +} + +struct FileKindFacts { + kind: FileKind, + created_at_ms: i64, + modified_at_ms: i64, +} + +#[async_trait] +impl FileSystem for RemoteEnvironmentFileSystem { + fn access_policy(&self) -> FileAccessPolicy { + if self.writable { + FileAccessPolicy::FullReadWrite + } else { + FileAccessPolicy::FullReadOnly + } + } + + async fn read_file(&self, path: &FsPath) -> FsResult> { + Ok(self.read_whole(path).await?.bytes) + } + + async fn read_file_versioned(&self, path: &FsPath) -> FsResult { + self.read_whole(path).await + } + + fn max_read_bytes(&self) -> Option { + Some(self.limits.max_read_file_bytes) } async fn read_file_range( @@ -222,39 +348,60 @@ where offset: u64, max_bytes: Option, ) -> FsResult { - if !self.filesystem_ranged_read { - let bytes = self.read_file(path).await?; - return Ok(ranged_read_from_full(bytes, offset, max_bytes)); - } + let ceiling = self.limits.max_read_file_bytes; let response = self .client - .lock() - .await - .read_file(&remote_fs::ReadFileParams { + .read_file(&ReadFileParams { path: environment_path(path)?, - offset: Some(offset), - max_bytes, + offset, + max_bytes: max_bytes.map_or(ceiling, |max| max.min(ceiling)), }) .await .map_err(|error| map_fs_error(error, path))?; - let bytes = response.data.into_inner(); - let file_size = response - .file_size - .unwrap_or_else(|| offset.saturating_add(bytes.len() as u64)); Ok(FsRangedRead { - file_size, + file_size: response.file_size, truncated: response.truncated, - bytes, + bytes: response.data.into_inner(), }) } async fn write_file(&self, path: &FsPath, contents: Vec) -> FsResult<()> { - self.client - .lock() + self.write_file_with(path, contents, WriteOptions::default()) .await - .write_file(&remote_fs::WriteFileParams { + } + + async fn write_file_with( + &self, + path: &FsPath, + contents: Vec, + options: WriteOptions, + ) -> FsResult<()> { + if contents.len() as u64 > self.limits.max_write_file_bytes { + return Err(FsError::InvalidInput { + message: format!( + "{} bytes exceed the environment's {}-byte inline write limit; write larger files from a content_ref", + contents.len(), + self.limits.max_write_file_bytes + ), + }); + } + let if_match = options + .if_match + .map(|version| { + environment_protocol::data::fs::FileVersion::new(version.as_str()).map_err( + |error| FsError::InvalidInput { + message: error.to_string(), + }, + ) + }) + .transpose()?; + self.client + .write_file(&WriteFileParams { path: environment_path(path)?, data: ByteChunk::from(contents), + create: WriteCondition::Always, + if_match, + executable: false, }) .await .map_err(|error| map_fs_error(error, path))?; @@ -267,11 +414,9 @@ where options: CreateDirectoryOptions, ) -> FsResult<()> { self.client - .lock() - .await - .create_directory(&remote_fs::CreateDirectoryParams { + .create_directory(&CreateDirectoryParams { path: environment_path(path)?, - recursive: Some(options.recursive), + recursive: options.recursive, }) .await .map_err(|error| map_fs_error(error, path))?; @@ -279,75 +424,81 @@ where } async fn get_metadata(&self, path: &FsPath) -> FsResult { - let response = self - .client - .lock() - .await - .get_metadata(&remote_fs::GetMetadataParams { - path: environment_path(path)?, - }) - .await - .map_err(|error| map_fs_error(error, path))?; + let own = self.metadata(path, false).await?; + let is_symlink = own.kind == FileKind::Symlink; + let target = if is_symlink { + self.metadata(path, true).await? + } else { + own + }; Ok(FileMetadata { - is_directory: response.is_directory, - is_file: response.is_file, - is_symlink: response.is_symlink, - created_at_ms: response.created_at_ms, - modified_at_ms: response.modified_at_ms, + is_directory: target.kind == FileKind::Directory, + is_file: target.kind == FileKind::File, + is_symlink, + created_at_ms: target.created_at_ms, + modified_at_ms: target.modified_at_ms, }) } async fn read_directory(&self, path: &FsPath) -> FsResult> { - let response = self - .client - .lock() - .await - .read_directory(&remote_fs::ReadDirectoryParams { - path: environment_path(path)?, - }) - .await - .map_err(|error| map_fs_error(error, path))?; - Ok(response - .entries - .into_iter() - .map(|entry| ReadDirectoryEntry { - file_name: entry.file_name, - is_directory: entry.is_directory, - is_file: entry.is_file, - }) - .collect()) + let directory = environment_path(path)?; + let mut entries = Vec::new(); + let mut cursor = None; + loop { + let page = self + .client + .read_directory(&ReadDirectoryParams { + path: directory.clone(), + limit: self.limits.max_directory_entries, + cursor, + }) + .await + .map_err(|error| map_fs_error(error, path))?; + entries.extend(page.entries.into_iter().map(|entry| ReadDirectoryEntry { + file_name: entry.name.as_str().to_owned(), + is_directory: entry.metadata.kind == FileKind::Directory, + is_file: entry.metadata.kind == FileKind::File, + })); + match page.next_cursor { + Some(next) => cursor = Some(next), + None => break, + } + } + Ok(entries) } async fn remove(&self, path: &FsPath, options: RemoveOptions) -> FsResult<()> { self.client - .lock() - .await - .remove(&remote_fs::RemoveParams { + .remove(&RemoveParams { path: environment_path(path)?, - recursive: Some(options.recursive), - force: Some(options.force), + recursive: options.recursive, + force: options.force, }) .await .map_err(|error| map_fs_error(error, path))?; Ok(()) } - async fn copy( + async fn rename( &self, source_path: &FsPath, destination_path: &FsPath, - options: CopyOptions, + options: RenameOptions, ) -> FsResult<()> { self.client - .lock() - .await - .copy(&remote_fs::CopyParams { - source_path: environment_path(source_path)?, - destination_path: environment_path(destination_path)?, - recursive: options.recursive, + .rename(&RenameParams { + source: environment_path(source_path)?, + destination: environment_path(destination_path)?, + overwrite: options.overwrite, + if_match: None, }) .await - .map_err(|error| map_fs_error(error, destination_path))?; + .map_err(|error| match error.protocol_kind() { + Some(ProtocolErrorKind::AlreadyExists) => FsError::AlreadyExists { + path: destination_path.clone(), + }, + _ => map_fs_error(error, source_path), + })?; Ok(()) } @@ -355,75 +506,92 @@ where &self, request: &FsTextSearchRequest, ) -> FsResult> { - if !self.filesystem_search { - return Ok(None); - } - let params = remote_fs::SearchTextParams { + let ceilings = self.limits.search; + let mut params = SearchTextParams { root: environment_path(&request.root)?, pattern: request.pattern.clone(), - include: request.include.clone(), - case_sensitive: request.case_sensitive, - max_depth: request - .max_depth - .map(|depth| u32::try_from(depth).unwrap_or(u32::MAX)), - limits: remote_fs::SearchTextLimits { - max_matches: request.limits.max_matches, - max_files: request.limits.max_files, - max_bytes: request.limits.max_bytes, - max_duration_ms: request.limits.max_duration_ms, + pattern_ignore_case: !request.case_sensitive, + filter: TreeFilter { + include: request.include.iter().cloned().collect(), + max_depth: request.max_depth.map(depth), + ..TreeFilter::default() + }, + limits: SearchTextLimits { + max_matches: count(request.limits.max_matches, ceilings.max_matches), + max_files: count(request.limits.max_files, ceilings.max_files), + max_bytes: request.limits.max_bytes.min(ceilings.max_bytes), + max_duration_ms: request.limits.max_duration_ms.min(ceilings.max_duration_ms), + max_line_bytes: ceilings.max_line_bytes, }, }; - let response = self - .client - .lock() - .await - .search_text(¶ms) - .await - .map_err(|error| map_fs_error(error, &request.root))?; + let response = match self.client.search_text(¶ms).await { + Ok(response) => response, + // `fs/searchText` walks directories; a file root searches that + // one file, as the generic fallback does: its parent, limited to + // the file's own name. + Err(error) if is_not_directory(&error) => { + let Some(file_root) = file_root_search(request)? else { + return Ok(Some(FsTextSearchResponse { + matches: Vec::new(), + files_searched: 0, + bytes_searched: 0, + skipped_files: 0, + stopped: None, + })); + }; + params.root = file_root.parent; + params.filter = file_root.filter; + self.client + .search_text(¶ms) + .await + .map_err(|error| map_fs_error(error, &request.root))? + } + Err(error) => return Err(map_fs_error(error, &request.root)), + }; let matches = response .matches .into_iter() .map(|item| { Ok(FsTextSearchMatch { path: FsPath::new(item.path.as_str()).map_err(FsError::from)?, - line_number: item.line_number, + line_number: u64::from(item.line_number), line: item.line, + line_truncated: item.line_truncated, }) }) .collect::>>()?; Ok(Some(FsTextSearchResponse { matches, - files_searched: response.files_searched, - bytes_searched: response.bytes_searched, + files_searched: u64::from(response.searched_file_count), + bytes_searched: response.searched_bytes, + skipped_files: u64::from(response.skipped_file_count), stopped: response.stopped.map(|stop| match stop { - remote_fs::SearchTextStop::MatchLimit => FsSearchStop::MatchLimit, - remote_fs::SearchTextStop::FileLimit => FsSearchStop::FileLimit, - remote_fs::SearchTextStop::ByteLimit => FsSearchStop::ByteLimit, - remote_fs::SearchTextStop::TimeLimit => FsSearchStop::TimeLimit, + SearchTextStop::MatchLimit => FsSearchStop::MatchLimit, + SearchTextStop::FileLimit => FsSearchStop::FileLimit, + SearchTextStop::ByteLimit => FsSearchStop::ByteLimit, + SearchTextStop::TimeLimit => FsSearchStop::TimeLimit, + SearchTextStop::ResponseLimit => FsSearchStop::ResponseLimit, }), })) } async fn glob_files(&self, request: &FsGlobRequest) -> FsResult> { - if !self.filesystem_glob { - return Ok(None); - } - let params = remote_fs::GlobFilesParams { + let ceilings = self.limits.glob; + let params = GlobFilesParams { root: environment_path(&request.root)?, - pattern: request.pattern.clone(), - max_depth: request - .max_depth - .map(|depth| u32::try_from(depth).unwrap_or(u32::MAX)), - limits: remote_fs::GlobFilesLimits { - max_matches: request.limits.max_matches, - max_entries: request.limits.max_entries, - max_duration_ms: request.limits.max_duration_ms, + filter: TreeFilter { + include: vec![request.pattern.clone()], + max_depth: request.max_depth.map(depth), + ..TreeFilter::default() + }, + limits: GlobFilesLimits { + max_matches: count(request.limits.max_matches, ceilings.max_matches), + max_files: count(request.limits.max_entries, ceilings.max_files), + max_duration_ms: request.limits.max_duration_ms.min(ceilings.max_duration_ms), }, }; let response = self .client - .lock() - .await .glob_files(¶ms) .await .map_err(|error| map_fs_error(error, &request.root))?; @@ -434,38 +602,96 @@ where .collect::>>()?; Ok(Some(FsGlobResponse { matches, - entries_visited: response.entries_visited, + entries_visited: u64::from(response.visited_file_count), stopped: response.stopped.map(|stop| match stop { - remote_fs::GlobFilesStop::MatchLimit => FsSearchStop::MatchLimit, - remote_fs::GlobFilesStop::EntryLimit => FsSearchStop::FileLimit, - remote_fs::GlobFilesStop::TimeLimit => FsSearchStop::TimeLimit, + GlobFilesStop::MatchLimit => FsSearchStop::MatchLimit, + GlobFilesStop::FileLimit => FsSearchStop::FileLimit, + GlobFilesStop::TimeLimit => FsSearchStop::TimeLimit, + GlobFilesStop::ResponseLimit => FsSearchStop::ResponseLimit, }), })) } } +fn is_not_directory(error: &ClientError) -> bool { + matches!( + error.protocol_error().map(|error| &error.data), + Some(ErrorData::FailedPrecondition { + reason: PreconditionReason::NotDirectory + }) + ) +} + +/// A text search rooted at a file, as a search of its parent. +struct FileRootSearch { + parent: EnvironmentPath, + filter: TreeFilter, +} + +/// The parent walk that searches only `request.root`, or `None` when the +/// request's include does not select the file (nothing to search). +fn file_root_search(request: &FsTextSearchRequest) -> FsResult> { + let invalid = || FsError::InvalidInput { + message: format!("{} is neither a directory nor a file", request.root), + }; + let name = request.root.file_name().ok_or_else(invalid)?; + let parent = request.root.parent().ok_or_else(invalid)?; + if let Some(include) = request.include.as_deref() { + let pattern = crate::fs::pattern::PathPattern::new(include, false).map_err(|error| { + FsError::InvalidInput { + message: error.to_string(), + } + })?; + if !pattern.matches(name, false) { + return Ok(None); + } + } + Ok(Some(FileRootSearch { + parent: environment_path(&parent)?, + filter: TreeFilter { + include: vec![format!("/{}", globset::escape(name))], + max_depth: Some(1), + include_hidden: true, + include_ignored: true, + ..TreeFilter::default() + }, + })) +} + +fn count(requested: u64, ceiling: u32) -> u32 { + u32::try_from(requested).unwrap_or(u32::MAX).min(ceiling) +} + +fn depth(depth: usize) -> u32 { + u32::try_from(depth).unwrap_or(u32::MAX) +} + /// Process executor over the data plane. It keeps no per-handle state: the /// daemon owns the read cursor, so a handle continued from a later batch or /// connection reads only what was not delivered yet. #[derive(Clone)] -pub struct RemoteProcessExecutor { - client: Arc>>, +pub struct RemoteProcessExecutor { + client: DataClient, + pty: bool, + max_read_bytes: u64, + max_wait_ms: u64, process_id_prefix: u64, next_process_id: Arc, } #[async_trait] -impl ProcessExecutor for RemoteProcessExecutor -where - T: JsonRpcTransport + Send + 'static, -{ +impl ProcessExecutor for RemoteProcessExecutor { async fn run_process(&self, request: ProcessRequest) -> ProcessExecResult { + if request.tty && !self.pty { + return Err(ProcessError::Unsupported { + message: "this environment cannot run commands under a PTY (tty: true)".to_owned(), + }); + } let process_id = self.next_process_id(); - let response = { - let mut client = self.client.lock().await; - client - .start_process(&remote_process::StartProcessParams { - process_id: process_id.clone(), + self.client + .start_process(&StartProcessParams { + process_id: process_id.clone(), + command: CommandSpec { argv: request.argv, cwd: request .cwd @@ -474,23 +700,26 @@ where .transpose()?, env: request.env, secret_env: request.secret_env, - stdin: request.stdin.map(ByteChunk::from), + input: match request.stdin { + Some(stdin) => ExecInput::Bytes { + data: ByteChunk::from(stdin), + }, + None => ExecInput::None {}, + }, timeout_ms: request.timeout_ms, - tty: request.tty, - }) - .await - .map_err(map_process_error)?; - client - .read_process(&remote_process::ReadProcessParams { - process_id: process_id.clone(), - after_seq: None, - max_bytes: request.max_output_bytes.map(|value| value as usize), - wait_ms: request.yield_ms, - }) - .await - .map_err(map_process_error)? - }; - Ok(output_from_read(process_id, response)) + }, + pty: request.tty.then_some(DEFAULT_PTY_SIZE), + }) + .await + .map_err(map_process_error)?; + let handle = ProcessHandle::new(process_id.as_str()); + self.read_output( + &handle, + &process_id, + request.yield_ms, + request.max_output_bytes, + ) + .await } async fn continue_process( @@ -498,64 +727,43 @@ where request: ContinueProcessRequest, ) -> ProcessExecResult { let handle = request.handle; - let process_id = ProcessId::new(handle.as_str().to_owned()); - let input = request.input.filter(|input| !input.is_empty()); - let response = { - let mut client = self.client.lock().await; - if let Some(signal) = request.signal { - client - .terminate_process(&remote_process::TerminateProcessParams { - process_id: process_id.clone(), - signal: match signal { - ProcessSignal::Interrupt => remote_process::ProcessSignal::Interrupt, - ProcessSignal::Kill => remote_process::ProcessSignal::Kill, - }, - }) - .await - .map_err(map_process_error)?; - } else if input.is_some() || request.close_stdin { - let write = client - .write_process(&remote_process::WriteProcessParams { - process_id: process_id.clone(), - chunk: input.map(ByteChunk::from), - close_stdin: request.close_stdin, - }) - .await - .map_err(map_process_error)?; - match write.status { - WriteProcessStatus::Accepted | WriteProcessStatus::Starting => {} - WriteProcessStatus::UnknownProcess => { - return Err(ProcessError::UnknownHandle { handle }); - } - WriteProcessStatus::StdinClosed => { - return Err(ProcessError::StdinClosed { handle }); - } - } - } - client - .read_process(&remote_process::ReadProcessParams { + let process_id = + ProcessId::new(handle.as_str()).map_err(|_| ProcessError::UnknownHandle { + handle: handle.clone(), + })?; + if let Some(signal) = request.signal { + self.client + .terminate_process(&TerminateProcessParams { process_id: process_id.clone(), - after_seq: None, - max_bytes: request.max_output_bytes.map(|value| value as usize), - wait_ms: request.wait_ms, + signal: match signal { + ProcessSignal::Interrupt => { + environment_protocol::data::process::ProcessSignal::Interrupt + } + ProcessSignal::Kill => { + environment_protocol::data::process::ProcessSignal::Kill + } + }, }) .await - .map_err(|error| match error { - EnvironmentClientError::Protocol(error) - if error.code == EnvironmentProtocolErrorCode::NotFound => - { - ProcessError::UnknownHandle { - handle: handle.clone(), - } - } - other => map_process_error(other), - })? - }; - Ok(output_from_read(process_id, response)) + .map_err(|error| map_handle_error(error, &handle))?; + } else { + let input = request.input.unwrap_or_default(); + if !input.is_empty() || request.close_stdin { + self.write_input(&handle, &process_id, input, request.close_stdin) + .await?; + } + } + self.read_output( + &handle, + &process_id, + request.wait_ms, + request.max_output_bytes, + ) + .await } } -impl RemoteProcessExecutor { +impl RemoteProcessExecutor { fn next_process_id(&self) -> ProcessId { let id = self.next_process_id.fetch_add(1, Ordering::Relaxed); ProcessId::new(format!( @@ -563,117 +771,311 @@ impl RemoteProcessExecutor { PROCESS_ID_NONCE.as_str(), self.process_id_prefix )) + .expect("generated process ids fit the id grammar") + } + + /// Queue `input` for the process, resending what a full input queue did + /// not accept, then close the input when asked. + async fn write_input( + &self, + handle: &ProcessHandle, + process_id: &ProcessId, + input: Vec, + close_input: bool, + ) -> ProcessExecResult<()> { + let started = Instant::now(); + let mut offset = 0usize; + loop { + let rest = &input[offset..]; + let accepted = self + .client + .write_process(&WriteProcessParams { + process_id: process_id.clone(), + data: ByteChunk::from(rest), + close_input, + }) + .await + .map_err(|error| map_handle_error(error, handle))? + .accepted_bytes; + offset += usize::try_from(accepted) + .unwrap_or(usize::MAX) + .min(rest.len()); + if offset >= input.len() { + return Ok(()); + } + if started.elapsed() >= INPUT_RETRY_WINDOW { + return Err(ProcessError::Failed { + message: format!( + "process {handle} accepted {offset} of {} input bytes; it is not reading its input", + input.len() + ), + }); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// Read until the process exits and its output settles, or until + /// `wait_ms` elapses. A finished command whose output exceeds the budget + /// shows its head and tail with the omitted count in between; a running + /// one keeps the rest behind the daemon cursor for the next call. + async fn read_output( + &self, + handle: &ProcessHandle, + process_id: &ProcessId, + wait_ms: Option, + max_output_bytes: Option, + ) -> ProcessExecResult { + let budget = max_output_bytes + .unwrap_or(DEFAULT_PROCESS_OUTPUT_BYTES) + .max(1); + let deadline = wait_ms.map(|wait| Instant::now() + Duration::from_millis(wait)); + let mut pages = Vec::new(); + let mut collected = 0u64; + let mut settled = false; + let last = loop { + // Once the exit has settled, further pages only drain output. + let wait = match deadline { + _ if settled => 0, + Some(deadline) => deadline + .saturating_duration_since(Instant::now()) + .as_millis() as u64, + None => self.max_wait_ms, + } + .min(self.max_wait_ms); + let response = self + .read_page( + handle, + ReadProcessParams { + process_id: process_id.clone(), + after: None, + tail_bytes: None, + ack: None, + max_bytes: (budget - collected).min(self.max_read_bytes), + wait_ms: wait, + return_on: ReturnOn::Exit, + }, + ) + .await?; + settled = response.state == ExecState::Exited; + collected += page_bytes(&response.output); + pages.push(response.output.clone()); + let more_output = response.output.next < response.output.end; + let out_of_time = deadline.is_some_and(|deadline| Instant::now() >= deadline); + match response.state { + ExecState::Exited if !more_output => break response, + ExecState::Exited if collected >= budget => { + let tail_bytes = budget / 2; + let tail = self + .read_page( + handle, + ReadProcessParams { + process_id: process_id.clone(), + after: None, + tail_bytes: Some(tail_bytes), + ack: None, + max_bytes: tail_bytes.min(self.max_read_bytes), + wait_ms: 0, + return_on: ReturnOn::AnyOutput, + }, + ) + .await?; + pages.push(tail.output.clone()); + break tail; + } + // The page ended before the budget (its chunk limit): read on. + ExecState::Exited => continue, + ExecState::Running if collected >= budget || out_of_time => break response, + ExecState::Running => continue, + } + }; + Ok(output_from_read(handle, budget, pages, last)) + } + + async fn read_page( + &self, + handle: &ProcessHandle, + params: ReadProcessParams, + ) -> ProcessExecResult { + self.client + .read_process(¶ms) + .await + .map_err(|error| map_handle_error(error, handle)) } } -fn output_from_read(process_id: ProcessId, response: ReadProcessResponse) -> ProcessOutput { - let (stdout, stderr) = split_output(response.chunks, response.omitted_bytes); - let status = match response.termination { - Some(ProcessTermination::TimedOut) => ProcessStatus::TimedOut, - Some(ProcessTermination::Killed) => ProcessStatus::Killed, - None if response.failure.is_some() => ProcessStatus::Failed, - None if response.exited => match response.exit_code { - Some(0) => ProcessStatus::Succeeded, - _ => ProcessStatus::Failed, - }, - None => ProcessStatus::Running, - }; - let handle = if response.closed || response.exited { - None +fn page_bytes(page: &OutputPage) -> u64 { + page.chunks + .iter() + .map(|chunk| chunk.data.len() as u64) + .sum() +} + +/// Shape the pages of one call into the model-facing output. The last page +/// may be a tail read; earlier pages are consecutive cursor reads. +fn output_from_read( + handle: &ProcessHandle, + budget: u64, + mut pages: Vec, + last: ReadProcessResponse, +) -> ProcessOutput { + let exited = last.state == ExecState::Exited; + let tail = (pages.len() > 1 && exited && last.output.next == last.output.end) + .then(|| pages.pop()) + .flatten(); + let head = merge_pages(pages); + let shown = match tail { + Some(tail) if tail.end > head.next => ShownOutput::from_pages(head, Some(tail)), + Some(tail) => ShownOutput::from_pages(merge_pages(vec![head, tail]), None), + None => ShownOutput::from_pages(head, None), + } + .trim_to(budget); + let (stdout, stderr) = split_output(&shown); + let (status, exit_code, failure) = if !exited { + (ProcessStatus::Running, None, None) } else { - Some(ProcessHandle::new(process_id.0)) + match (&last.stopped_by, &last.exit, &last.error) { + (Some(StoppedBy::Timeout), exit, _) => (ProcessStatus::TimedOut, exit_code(exit), None), + (Some(StoppedBy::Kill | StoppedBy::Terminate), exit, _) => { + (ProcessStatus::Killed, exit_code(exit), None) + } + (None, _, Some(error)) => (ProcessStatus::Failed, None, Some(error.to_string())), + (None, Some(ExitStatus::Code { code: 0 }), None) => { + (ProcessStatus::Succeeded, Some(0), None) + } + (None, Some(ExitStatus::Code { code }), None) => { + (ProcessStatus::Failed, Some(*code), None) + } + ( + None, + Some(ExitStatus::Signal { + signal, + core_dumped, + }), + None, + ) => ( + ProcessStatus::Failed, + None, + Some(format!( + "killed by {signal}{}", + if *core_dumped { " (core dumped)" } else { "" } + )), + ), + (None, None, None) => ( + ProcessStatus::Failed, + None, + Some("the environment reported no exit status".to_owned()), + ), + } }; ProcessOutput { status, - handle, - pid: response.pid, - exit_code: response.exit_code, - failure: response.failure, + handle: (!exited).then(|| handle.clone()), + pid: Some(last.pid), + exit_code, + failure, stdout, stderr, - omitted_bytes: response.omitted_bytes, - leftover_processes: response - .leftover_processes - .into_iter() - .map(|member| LeftoverProcess { - pid: member.pid, - command: member.command, + omitted_bytes: shown.omitted_bytes(), + leftover_processes: last + .leftovers + .map(|report| { + report + .processes + .into_iter() + .map(|member| LeftoverProcess { + pid: member.pid, + command: member.command.unwrap_or_default(), + }) + .collect() }) - .collect(), + .unwrap_or_default(), + } +} + +fn exit_code(exit: &Option) -> Option { + match exit { + Some(ExitStatus::Code { code }) => Some(*code), + _ => None, + } +} + +/// Consecutive pages as one: chunks and gaps in order, the last page's +/// position. +fn merge_pages(pages: Vec) -> OutputPage { + let mut merged = OutputPage::default(); + for page in pages { + merged.chunks.extend(page.chunks); + merged.gaps.extend(page.gaps); + merged.next = merged.next.max(page.next); + merged.end = merged.end.max(page.end); + } + merged +} + +/// Sorts chunks into their streams. The omission marker goes where output +/// was left out: before the first chunk after the omitted range, or at the +/// end of standard output when nothing follows it. +fn split_output(shown: &ShownOutput) -> (StreamOutput, StreamOutput) { + let mut stdout = StreamOutput::default(); + let mut stderr = StreamOutput::default(); + for chunk in &shown.head { + let stream = if is_stderr(chunk.stream) { + &mut stderr + } else { + &mut stdout + }; + stream.bytes.extend_from_slice(chunk.data.as_slice()); } + if shown.omitted.is_some() { + let stream = match shown.tail.first() { + Some(chunk) if is_stderr(chunk.stream) => &mut stderr, + _ => &mut stdout, + }; + stream.omitted_at = Some(stream.bytes.len()); + } + for chunk in &shown.tail { + let stream = if is_stderr(chunk.stream) { + &mut stderr + } else { + &mut stdout + }; + stream.bytes.extend_from_slice(chunk.data.as_slice()); + } + (stdout, stderr) } #[derive(Clone)] -pub struct RemoteJobExecutor { - client: Arc>>, - capabilities: EnvironmentCapabilities, +pub struct RemoteJobExecutor { + client: DataClient, + limits: JobLimits, } #[async_trait] -impl JobExecutor for RemoteJobExecutor -where - T: JsonRpcTransport + Send + 'static, -{ - async fn start_jobs( - &self, - request: remote_jobs::StartJobsParams, - ) -> JobExecResult { - if !self.capabilities.job_start { - return Err(JobError::Unsupported { - message: "provider target does not support job/start".to_owned(), - }); - } +impl JobExecutor for RemoteJobExecutor { + fn limits(&self) -> Option { + Some(self.limits.clone()) + } + + async fn start_jobs(&self, request: StartJobsParams) -> JobExecResult { self.client - .lock() - .await .start_jobs(&request) .await .map_err(map_job_error) } - async fn read_jobs( - &self, - request: remote_jobs::ReadJobsParams, - ) -> JobExecResult { - if !self.capabilities.job_read { - return Err(JobError::Unsupported { - message: "provider target does not support job/read".to_owned(), - }); - } - self.client - .lock() - .await - .read_jobs(&request) - .await - .map_err(map_job_error) + async fn read_jobs(&self, request: ReadJobsParams) -> JobExecResult { + self.client.read_jobs(&request).await.map_err(map_job_error) } - async fn cancel_jobs( - &self, - request: remote_jobs::CancelJobsParams, - ) -> JobExecResult { - if !self.capabilities.job_cancel { - return Err(JobError::Unsupported { - message: "provider target does not support job/cancel".to_owned(), - }); - } + async fn cancel_jobs(&self, request: CancelJobsParams) -> JobExecResult { self.client - .lock() - .await .cancel_jobs(&request) .await .map_err(map_job_error) } } -fn access_policy_for_capabilities(capabilities: &EnvironmentCapabilities) -> FileAccessPolicy { - if capabilities.filesystem_write { - FileAccessPolicy::FullReadWrite - } else { - FileAccessPolicy::FullReadOnly - } -} - fn environment_path(path: &FsPath) -> Result { EnvironmentPath::new(path.as_str()).map_err(|error| FsError::InvalidInput { message: error.to_string(), @@ -686,279 +1088,317 @@ fn process_environment_path(path: &FsPath) -> Result FsError { - match error { - EnvironmentClientError::Protocol(error) => match error.code { - EnvironmentProtocolErrorCode::NotFound => FsError::NotFound { path: path.clone() }, - EnvironmentProtocolErrorCode::Conflict => FsError::AlreadyExists { path: path.clone() }, - EnvironmentProtocolErrorCode::Unauthorized - | EnvironmentProtocolErrorCode::Forbidden => { - FsError::PermissionDenied { path: path.clone() } - } - EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable => FsError::Unsupported { - message: error.message, - }, - EnvironmentProtocolErrorCode::InvalidRequest => FsError::InvalidInput { - message: error.message, - }, - _ => FsError::Failed { - message: error.message, - }, - }, - other => FsError::Failed { - message: other.to_string(), +fn map_fs_error(error: ClientError, path: &FsPath) -> FsError { + let Some(protocol) = error.protocol_error() else { + return FsError::Failed { + message: error.to_string(), + }; + }; + let message = protocol.message.clone(); + match &protocol.data { + ErrorData::NotFound { .. } => FsError::NotFound { path: path.clone() }, + ErrorData::AlreadyExists {} => FsError::AlreadyExists { path: path.clone() }, + ErrorData::PermissionDenied { .. } => FsError::PermissionDenied { path: path.clone() }, + ErrorData::FailedPrecondition { + reason: PreconditionReason::IfMatchFailed, + } => FsError::Conflict { path: path.clone() }, + ErrorData::CapabilityUnavailable { + capability: Capability::FilesystemWrite, + } => FsError::PermissionDenied { path: path.clone() }, + ErrorData::CapabilityUnavailable { .. } => FsError::Unsupported { message }, + ErrorData::FailedPrecondition { .. } + | ErrorData::InvalidParams { .. } + | ErrorData::InvalidRequest {} + | ErrorData::PayloadTooLarge { .. } => FsError::InvalidInput { message }, + _ => FsError::Failed { + message: protocol.to_string(), }, } } -fn map_process_error(error: EnvironmentClientError) -> ProcessError { - match error { - EnvironmentClientError::Protocol(error) => match error.code { - EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable => ProcessError::Unsupported { - message: error.message, - }, - EnvironmentProtocolErrorCode::InvalidRequest - | EnvironmentProtocolErrorCode::NotFound => ProcessError::InvalidRequest { - message: error.message, - }, - _ => ProcessError::Failed { - message: error.message, - }, +fn map_process_error(error: ClientError) -> ProcessError { + let Some(protocol) = error.protocol_error() else { + return ProcessError::Failed { + message: error.to_string(), + }; + }; + let message = protocol.message.clone(); + match protocol.kind() { + ProtocolErrorKind::CapabilityUnavailable => ProcessError::Unsupported { message }, + ProtocolErrorKind::InvalidParams + | ProtocolErrorKind::InvalidRequest + | ProtocolErrorKind::NotFound + | ProtocolErrorKind::PermissionDenied + | ProtocolErrorKind::FailedPrecondition => ProcessError::InvalidRequest { + message: protocol.to_string(), }, - other => ProcessError::Failed { - message: other.to_string(), + _ => ProcessError::Failed { + message: protocol.to_string(), }, } } -fn map_job_error(error: EnvironmentClientError) -> JobError { - match error { - EnvironmentClientError::Protocol(error) => match error.code { - EnvironmentProtocolErrorCode::Unsupported - | EnvironmentProtocolErrorCode::CapabilityUnavailable => JobError::Unsupported { - message: error.message, - }, - EnvironmentProtocolErrorCode::InvalidRequest - | EnvironmentProtocolErrorCode::NotFound - | EnvironmentProtocolErrorCode::Conflict => JobError::InvalidRequest { - message: error.message, - }, - _ => JobError::Failed { - message: error.message, - }, +/// Errors of calls on an existing handle: an unknown, expired or earlier +/// daemon's handle is `notFound`, closed input `inputClosed`. +fn map_handle_error(error: ClientError, handle: &ProcessHandle) -> ProcessError { + match error.protocol_error().map(|protocol| &protocol.data) { + Some(ErrorData::NotFound { .. }) => ProcessError::UnknownHandle { + handle: handle.clone(), }, - other => JobError::Failed { - message: other.to_string(), + Some(ErrorData::FailedPrecondition { + reason: PreconditionReason::InputClosed, + }) => ProcessError::StdinClosed { + handle: handle.clone(), }, + _ => map_process_error(error), } } -/// Sorts chunks into their streams. The omission marker goes where the -/// daemon dropped the middle: at the first sequence gap, or at the start of -/// the output when the gap precedes everything returned. -fn split_output( - chunks: Vec, - omitted_bytes: u64, -) -> (StreamOutput, StreamOutput) { - let mut stdout = StreamOutput::default(); - let mut stderr = StreamOutput::default(); - let mut omission_placed = omitted_bytes == 0; - let gap_index = chunks - .windows(2) - .position(|pair| pair[1].seq != pair[0].seq + 1) - .map(|index| index + 1) - .unwrap_or(0); - for (index, chunk) in chunks.into_iter().enumerate() { - let stream = match chunk.stream { - ProcessOutputStream::Stdout | ProcessOutputStream::Pty => &mut stdout, - ProcessOutputStream::Stderr => &mut stderr, +fn map_job_error(error: ClientError) -> JobError { + let Some(protocol) = error.protocol_error() else { + return JobError::Failed { + message: error.to_string(), }; - if !omission_placed && index == gap_index { - stream.omitted_at = Some(stream.bytes.len()); - omission_placed = true; - } - stream.bytes.extend(chunk.chunk.into_inner()); - } - if !omission_placed { - stdout.omitted_at = Some(stdout.bytes.len()); + }; + let message = protocol.message.clone(); + match &protocol.data { + ErrorData::FailedPrecondition { + reason: PreconditionReason::JobIdInUse, + } => JobError::JobIdInUse { + message: format!("{message}; choose a new job_id for a new run"), + }, + ErrorData::CapabilityUnavailable { .. } => JobError::Unsupported { message }, + ErrorData::InvalidParams { .. } + | ErrorData::InvalidRequest {} + | ErrorData::FailedPrecondition { .. } + | ErrorData::NotFound { .. } => JobError::InvalidRequest { message }, + _ => JobError::Failed { + message: protocol.to_string(), + }, } - (stdout, stderr) } #[cfg(test)] mod tests { use std::{ collections::{BTreeMap, VecDeque}, - sync::{Arc, Mutex}, + sync::Mutex, }; - use environment_client::EnvironmentClientResult; + use environment_protocol::{ + data::exec::{OutputChunk, OutputStream}, + error::ProtocolError, + jsonrpc::{Request, Response}, + }; use harness::storage::InMemoryBlobStore; use serde_json::{Value, json}; use super::*; use crate::fs::tools::{ReadFileArgs, WriteFileArgs, invoke_read_file, invoke_write_file}; - struct MockTransport { - sent: Arc>>, - recv: Arc>>, + /// Answers each request with the next scripted result and records it. + struct Script { + replies: Mutex>>, + sent: Mutex>, } - impl MockTransport { - fn new(messages: impl IntoIterator) -> (Self, Arc>>) { - let sent = Arc::new(Mutex::new(Vec::new())); - ( - Self { - sent: sent.clone(), - recv: Arc::new(Mutex::new(messages.into_iter().collect())), - }, - sent, - ) - } + fn initialize(capabilities: Value) -> InitializeResponse { + let fixture: Value = serde_json::from_str(include_str!( + "../../../environment-protocol/fixtures/data/initialize.json" + )) + .unwrap(); + let mut response = fixture["response"].clone(); + response["capabilities"] = capabilities; + response["defaultCwd"] = json!("/workspace"); + serde_json::from_value(response).unwrap() } - #[async_trait] - impl JsonRpcTransport for MockTransport { - async fn send(&mut self, message: Value) -> EnvironmentClientResult<()> { - self.sent.lock().expect("sent lock").push(message); - Ok(()) - } + fn scripted( + capabilities: Value, + replies: impl IntoIterator>, + ) -> (DataClient, Arc