diff --git a/AGENTS.md b/AGENTS.md index 9f5d70be..8eda6554 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -654,12 +654,14 @@ Implemented command behavior: - `ti fs mount-file-system --file-system-id --mount-path ./workspace --mount-profile portable --pack-path /` - `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --read-cache-size-mb 256 --read-cache-max-file-mb 16` - `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --cache-dir ~/.ti/cache/workspace --write-back-cache=false` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --durability close-sync` - `ti fs mount-file-system --file-system-id --mount-path ./workspace --remote-path /workspace --driver fuse --layer-ref layer-1` - `ti fs mount-file-system --file-system-id --mount-path ./workspace-v1 --remote-path /workspace --driver fuse --layer-ref layer-1 --checkpoint-id cp-1` - `ti fs drain-file-system --mount-path ./workspace` - `ti fs drain-file-system --mount-path ./workspace --timeout 30s` - `ti fs unmount-file-system --mount-path ./workspace` - `ti fs unmount-file-system --mount-path ./workspace --ignore-absent` +- `ti fs unmount-file-system --mount-path ./workspace --force` - `ti fs-vault create-secret --secret-name db-prod --field DB_URL=mysql://example --field PASSWORD=@./password.txt` - `ti fs-vault replace-secret --secret-path /n/vault/db-prod --from-directory ./secret-fields` - `ti fs-vault read-secret --secret-name db-prod` @@ -1149,6 +1151,49 @@ Background FS and vault mounts write only a non-secret locator under original resource-scoped companion HOME without requiring the FS token again. Successful unmount removes the locator; failed unmount preserves it for retry. +`ti fs unmount-file-system` drains before it detaches a FUSE mount and must never +report success while write-back work is still local. Parse the companion's +`mount drain --json` report instead of discarding it: `ti fs drain-file-system` +returns it as `response`, prints the pending counters in text output with one +shared formatter, fails with `fs.drain_pending_work` when work remains, and +refuses mounts the dry-run calls "not applicable" (WebDAV, and any non-`fs` +locator kind such as `vault`) with `fs.drain_not_applicable` instead of a raw +companion error. The unmount records the driver the companion actually mounted +in the non-secret mount locator, gates only FUSE file-system mounts on the +drain, blocks with the locator preserved while work is pending, and treats +`--force` as the only "accept the loss" path (reported as `drain.status: +"skipped"`). `--ignore-absent` is a cleanup idempotency flag: it may continue +only past a drain failure that reports no pending work (no report at all, or a +parsed report whose counters are all zero with a non-pending classification), +but the block condition is the parsed pending counters, never the engine's +`error_kind` alone and never the rendered summary (`DrainPending.HasPendingWork` +is the single predicate, with one block authority: the error code that already +folds the counters in), and the outcome must still report `drain.status: +"failed"` on both the `unmounted` and `absent` paths. A non-pending classification with +queued work still blocks. A locator written before the driver was recorded is never gated, so +an upgrade cannot block an existing unmount. The reported `durability` must +match the driver the companion reported, and `--durability` must reject an +explicitly empty value instead of treating it as the default. Do not drop these +outcomes from the result, and do not reimplement the drain in ti: the mount +control socket belongs to the companion. The companion's `--mode auto` rule is +mirrored in exactly one place (`companionMountMode`), because +`mountdriver.Resolve` is prerequisite-aware and would disagree with the +companion; see `docs/spec/done/0038-fs-mount-durability.md`. + +`ti fs mount-file-system` forwards the selected write-durability tier to the +companion as `mount --durability`. The flag accepts `auto`, `interactive`, +`fsync`, `close-sync`, and `write-sync`, and defaults to `interactive` so an +ordinary mount keeps its documented behavior instead of following the engine's +`auto` default. Never collapse the tier back into the legacy +`--write-back-cache` boolean: keep that flag only as an alias (`true` means +`interactive`, `false` means `auto`), and refuse an inconsistent explicit +combination. FUSE mounts forward the tier, including `--driver auto` mounts on +platforms whose automatic choice is FUSE; WebDAV mounts forward none and reject +every explicit tier except `auto` with `fs.webdav_durability_unsupported`; +checkpoint mounts are read-only, force `auto`, and reject any other explicit +tier. Report the effective tier as `durability` in the mount result and do not +reintroduce the removed `write_back_cache` boolean. + Do not implement or expose ti commands for Drive9 internal APIs that are not part of Drive9's public CLI. In particular, do not reintroduce low-level layer entry/object/event commands, low-level Git workspace/tree/state/object-pack/ diff --git a/README.md b/README.md index 63fd9a53..c960f7df 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,16 @@ Automatic mounting uses FUSE on Linux and WebDAV on macOS and Windows. macOS use Mount commands start the companion runtime in the background, wait until the mount is ready, and then return a structured result. Use `ti fs unmount-file-system` or `ti fs-vault unmount-vault` to end a mount. The public CLI does not expose a foreground mount mode. +FUSE mounts choose their write durability with `--durability`: `auto` lets the companion measure round-trip time, `interactive` (the default) keeps writes durable in the local staging journal and uploads them after flush, `fsync` makes `fsync(2)` wait for the remote commit, `close-sync` makes `close(2)` remote-durable, and `write-sync` makes every write remote-durable before it returns. Use `close-sync` or `write-sync` when a caller must not lose an acknowledged write in the client; the interactive default leaves that window to the background uploader. The mount result reports the effective tier as `durability`. WebDAV mounts keep the engine's native write behavior: ti forwards no tier, reports `auto`, and rejects every other explicit tier. `--write-back-cache` remains as a legacy alias where `true` means `interactive` and `false` means `auto`. Checkpoint mounts are read-only, reject any tier other than `auto`, and report `auto`. + +```shell +ti fs mount-file-system --file-system-id --mount-path ~/my-workspace --driver fuse --durability close-sync +ti fs drain-file-system --mount-path ~/my-workspace +ti fs unmount-file-system --mount-path ~/my-workspace +``` + +`ti fs drain-file-system` reports the companion's drain result, including the pending counters (`open_handles`, `commit_queue_*`, `uploader_*`), so a caller can tell "fully uploaded" from "the background upload never finished"; it also refuses WebDAV and vault mounts, which have no drain, instead of failing with a raw companion error. `ti fs unmount-file-system` runs that drain itself before detaching a FUSE file-system mount and fails with `fs.drain_pending_work` while writes are still pending, keeping the mount locator so the command can be retried. `--force` is the only way to accept the undrained writes (the result then records `drain.status: "skipped"`); `--ignore-absent` stays a cleanup idempotency flag and continues only past a drain failure that reports no pending work - no report at all, or a zero-counter report with a non-pending classification (`drain.status: "failed"`) - never past pending work. WebDAV mounts keep their native write behavior and are not gated, and `ti fs-vault unmount-vault` is not gated either because the engine drains only file-system mounts. + Run a command against a remote Filesystem (Runtime is available only in AWS `us-east-1` and `us-west-2`): ```shell diff --git a/docs/spec/done/0038-fs-mount-durability.md b/docs/spec/done/0038-fs-mount-durability.md new file mode 100644 index 00000000..c5e8a2f2 --- /dev/null +++ b/docs/spec/done/0038-fs-mount-durability.md @@ -0,0 +1,127 @@ +# FUSE Mount Write Durability Selection + +## Goal + +Let a `ti fs mount-file-system` caller pick the write-durability tier the +ti-drive9 companion applies to a FUSE mount: + +```bash +ti fs mount-file-system --file-system-id --mount-path ./workspace +ti fs mount-file-system --file-system-id --mount-path ./workspace --durability close-sync +ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --durability write-sync +``` + +Before this spec, ti exposed only the boolean `--write-back-cache`, and only an +explicit `--driver fuse` mount forwarded anything: `true` (the default) pinned +the companion to `interactive`, and `false` forwarded nothing, so the companion +fell back to its own `auto` default. The companion has supported five tiers for +longer than ti has exposed them (`mount --durability auto|interactive|fsync| +close-sync|write-sync`), so a platform customer that needed `close-sync` or +`write-sync` had to mount around ti and drain by hand, which leaves the +client-side loss window unbounded and unobservable. + +## User Outcome + +A caller can state the durability it needs and get it, or get a clear error: + +- `interactive` (default) keeps the previous behavior: writes are durable in the + local staging journal and the remote upload runs in the background after + flush. +- `close-sync` makes `close(2)` remote-durable, so a completed close means the + data left the client. +- `write-sync` makes every `write(2)` remote-durable before it returns. +- `fsync` makes `fsync(2)` wait for the remote commit while ordinary writes keep + the write-back policy. +- `auto` leaves the choice to the companion's round-trip measurement. + +The mount result reports the effective tier, so a script can verify what it got +instead of assuming: + +```json +{ "status": "mounted", "driver": "fuse", "durability": "close-sync" } +``` + +## Command Contract + +`ti fs mount-file-system` (and its `ti fs mount` alias) gains: + +- `--durability `, default `interactive`, one of `auto`, `interactive`, + `fsync`, `close-sync`, `write-sync`. The value is trimmed and case-folded + before validation; anything else fails with `fs.invalid_mount_durability` and + exit code 2 before the companion runs. An explicitly empty value fails with + `fs.missing_mount_durability`, matching the CLI's rule for `--region` and + `--profile`, instead of being silently treated as the default. + +`--write-back-cache` stays as a legacy alias so existing scripts keep working: + +- `--write-back-cache=true` means `--durability=interactive`. +- `--write-back-cache=false` means `--durability=auto`. +- Setting both explicitly to inconsistent values fails with + `fs.conflicting_mount_durability` instead of silently preferring one. + +### Forwarding rules + +ti forwards the tier as `mount --durability ` only when the mount resolves +to FUSE, mirroring the companion's own `--mode auto` rule (WebDAV on macOS and +Windows, FUSE elsewhere). That rule lives in one place, +`internal/fs/mount_durability.go`'s `companionMountMode`. It deliberately does +not use `mountdriver.Resolve`, which is prerequisite-aware (macFUSE on darwin) +and would forward a tier to a WebDAV mount the companion chose. The mount result +reconciles the reported `durability` with the driver the companion reports, so +the result never claims a tier that is not in effect: + +- Explicit `--driver fuse`, and `--driver auto` on a platform whose automatic + choice is FUSE, forward the tier. An ordinary mount therefore keeps the + `interactive` default instead of following the companion's `auto` default. +- `--driver webdav` forwards no tier. WebDAV mounts keep the engine's native + write behavior, so an explicit tier other than `auto` fails with + `fs.webdav_durability_unsupported` and exit code 2. Reporting a tier that + cannot take effect would be worse than refusing it. +- Checkpoint mounts (`--layer-ref` plus `--checkpoint-id`) are read-only: they + force `auto`, forward nothing, and reject any other explicit tier with + `fs.checkpoint_mount_read_only`. `--write-back-cache=true` keeps its existing + rejection there. + +The result reports the effective tier as `durability` (`auto` for WebDAV and +checkpoint mounts). It replaces the previous `write_back_cache` boolean: a +boolean cannot express five tiers, and keeping a derived boolean alongside the +tier invites it to disagree with the mount. The removed field was never +documented as a stable contract. + +`--dry-run` reports the same effective tier in its request body and applies the +same validations, including the WebDAV rejection. + +## Scope + +In scope: + +- `internal/cli` flag registration and option wiring. +- `internal/fs` tier normalization, legacy alias reconciliation, FUSE/WebDAV + and checkpoint rules, companion argv construction, mount result, and dry-run. +- README, AGENTS.md, e2e (fake companion argv + failures), and live e2e mount + coverage. + +Out of scope: + +- `ti fs-vault mount-vault`, which forwards no tier and reports `auto`. +- Any ti-native mount implementation. Mounting still belongs to the companion. +- Drive9-side behavior: ti only selects an existing engine tier. + +## Verification + +- Unit tests pin the tier table, the legacy alias mapping, the conflict error, + the explicit-empty rejection, the WebDAV and checkpoint rejections, the local + mirror of the companion's `--mode auto` rule, the forwarded argv for every + tier, the interactive default, and the dry-run body. +- `make e2e` covers the default and a strict tier end to end through the fake + companion, plus the invalid-value and WebDAV rejections. Mounts that assert a + forwarded tier pin `--driver fuse`, and the automatic-mode case branches on + the host platform, so the gate is green on macOS and Windows too. +- `make live-e2e-fs` and `make live-e2e-fs-git` mount with an explicit tier + (`close-sync`) and assert the reported value, and the live WebDAV mount + asserts `auto`. + +The English PingCAP documentation pages under `docs/pingcap-docs/docs/` are a +submodule that was not checked out while this spec was implemented. The +published mount page must gain the `--durability` section; the tracked follow-up +is issue #98 in this repository and must land before the next release cut. diff --git a/docs/spec/done/0039-fs-unmount-drain-gate.md b/docs/spec/done/0039-fs-unmount-drain-gate.md new file mode 100644 index 00000000..7a3b6f49 --- /dev/null +++ b/docs/spec/done/0039-fs-unmount-drain-gate.md @@ -0,0 +1,203 @@ +# Drain Before Unmount + +## Goal + +Stop `ti fs unmount-file-system` from reporting success while local write-back +writes never reached the remote. An unmount must drain pending writes first, and +either refuse to detach while work is still pending or say explicitly that the +caller accepted the loss. `ti fs drain-file-system` must show what the engine +counted instead of discarding its report. + +## Problem + +Drive9's write-back policy keeps a write durable in the local staging journal +and uploads it in the background after flush. That makes the client's loss +window a background-upload race, and three holes made it invisible: + +1. `drive9 umount` stops the mount with a signal; the worker's SIGTERM handler + runs `flushOpenHandles` and `chain.Drain` but discards both errors + (`pkg/fuse/mount.go`, the shutdown goroutine), then exits with status 0 + because the stop was requested. A failed drain therefore looks like a clean + unmount, and the pending writes are gone with the mount. +2. `drive9 umount` never asks the mount's control socket for pending work, so it + cannot gate the detach on it. +3. `ti fs drain-file-system` ran `mount drain --json` and discarded the report, + and `ti fs unmount-file-system` never drained at all. + +Consequence: unmount returned success, the upload never happened, nothing +warned, and afterwards neither the user nor support could tell a client-side +loss from a server-side failure. + +## User Outcome + +```bash +# Reports the engine's drain report, including the pending counters: +ti fs drain-file-system --mount-path ./workspace + +# Refuses to detach while the drain reports pending work, keeping the locator: +ti fs unmount-file-system --mount-path ./workspace + +# Explicitly accepts the loss and detaches anyway: +ti fs unmount-file-system --mount-path ./workspace --force +``` + +A pending unmount fails with `fs.drain_pending_work`, names the engine's +`error_kind`, prints the non-zero pending counters, and shows the retry and +`--force` commands. A successful unmount reports the drain pass in its result: + +```json +{ + "status": "unmounted", + "mount_path": "./workspace", + "drain": { "status": "drained", "response": { "ok": true, "pending": { } } } +} +``` + +## Command Contract + +### `ti fs drain-file-system` + +- The companion's `mount drain --json` report is parsed into + `response` (same shape as `mountcontrol.DrainResponse`) instead of being + dropped. +- `status` is `drained`. A drain that leaves write-side work fails with + `fs.drain_pending_work`; a classified failure that reports none uses + `fs.drain_failed`; a failure with no usable report uses + `fs.drain_report_missing` (no stdout) or `fs.drain_report_invalid` + (unparseable stdout). Every error carries `error_kind`, the engine's error + text, and the non-zero pending counters when there are any, plus the remedy + command. +- Text output prints the pending counters, so `--output text` is usable in an + operator runbook. + +### `ti fs unmount-file-system` + +- Unless skipped, ti runs the same drain before `umount`. The drain uses the + command's `--timeout` (default 30s). +- The drain applies only to ti fs FUSE mounts. ti records the driver the + companion reported in the mount locator at mount time; WebDAV mounts keep + their native write behavior and have no control socket, so they are not + gated. The locator's mount kind gates too, because the engine drains only + file-system mounts: `ti fs-vault unmount-vault` (kind `vault`) is never + drained. A locator written before ti recorded the driver is also not gated, + so an upgrade never blocks an existing unmount. +- A failed or pending drain blocks the unmount and keeps the locator, so the + caller can close the writers and retry. +- `--force` skips the gate. The result records `drain.status: "skipped"` with + the reason, so a forced unmount is still attributable. It is the only way to + accept undrained writes. +- `--ignore-absent` keeps its cleanup idempotency meaning, not loss acceptance: + it may tolerate only a drain failure that reports **no pending work** — no + report at all (typically a mount that is already gone, or a companion that + predates the report), or a parsed report whose counters are all zero with a + non-pending classification. The result records `drain.status: "failed"` plus a + `drain` runtime check on both the `unmounted` and `absent` outcomes. The block + condition is the **parsed pending counters**, not the engine's classification: + a drain that hits its timeout against a write-back backlog reports a + non-pending `error_kind` while still listing queued work, so trusting the + classification alone would detach with exit 0 and print the counters it is + discarding. Cleanup scripts pass `--ignore-absent` routinely, so letting it + bypass the pending check would recreate the silent write loss this spec + closes. `--force` remains the only loss acceptance. +- The blocked-drain error tail is built from the classification: pending work + can only be bypassed with `--force` (and says `--ignore-absent` will not + help), while a non-pending or report-less failure points at + `--ignore-absent` for a mount that may be gone and at `--force` for accepting + the loss. The gate itself has one authority, the error code: the + `fs.drain_pending_work` classification already folds in both the engine's + `error_kind` and the parsed counters, so the block condition cannot drift from + the message. A drain that produced no report is a structured + `fs.drain_report_missing` (no stdout at all) or `fs.drain_report_invalid` + (unparseable stdout) with the same guidance, instead of a raw companion error. +- The result reports the recorded driver and the drain record. Text output adds + the `DRAIN` column and the pending counters. +- `--dry-run` reports the gate in a `pre_unmount_drain` check without running + the drain. + +### The pending predicate + +`DrainPending.HasPendingWork()` is the single data-safety predicate. It is an +explicit enumeration of the counters, never a parse of the operator-facing +summary, so rewording the text output cannot change a safety decision. + +The engine's own drain-failure exit record counts +`dirty_handles + commit_queue_pending + uploader_queued` as pending files and +`commit_queue_bytes + uploader_cached_bytes` as pending bytes, so those counters +are undrained work by construction. The remaining counters are informational, +but they are non-zero only while the mount still has unfinished work, so ti +treats any non-zero counter as pending. That is a deliberate conservative +over-block: refusing to detach is recoverable, silently detaching from queued +write-back is not. If Drive9 later publishes a loss-carrying versus +informational split, tighten the predicate and this section together. + +### Drain applicability + +`ti fs drain-file-system` runs only for mounts the engine can drain. A non-`fs` +locator kind (for example a `vault` mount) and a WebDAV driver fail with +`fs.drain_not_applicable` and an explanation instead of forwarding a request the +companion rejects with a raw error. A locator that records no driver stays +drainable: the command itself asks for a drain, and only the unmount gate needs +the stricter driver check. The drain `--dry-run` reports the same rule in its +`drain_support` check, with `status: "failed"` when the command would refuse to +run, so the dry run predicts the execution instead of contradicting it. + +The `--force` flag previously promised to "kill the mount process" but no code +read it; it now means "accept the undrained writes". + +## Scope + +In scope (this repository): + +- `internal/fs`: drain report parsing and rendering, the unmount gate, driver + recording in the mount locator, dry-run checks, result types. +- `internal/cli`: `--force` help text. +- `internal/fs/mountlocator`: the non-secret `driver` field. +- README, AGENTS.md, unit tests, fake-companion e2e, and live e2e. + +Out of scope (Drive9 repository, tracked there): + +- `drive9 umount` reporting and gating on pending work itself, so a non-ti + caller gets the same protection. +- The SIGTERM handler surfacing `flushOpenHandles`/`chain.Drain` failures + instead of discarding them. +- `ti fs-vault unmount-vault`, which forwards no durability tier and is not + gated: the engine drains only file-system mounts. + +ti can only refuse to detach; it cannot make the engine's shutdown path report +its own drain failure. Until Drive9 changes that, a `kill -TERM` of the mount +process (or a supervisor restart) can still lose staged writes, and ti's gate +remains the client-side mitigation. + +## Verification + +- Unit tests cover: a healthy drain report in the result and text output, a + pending drain error with counters and codes, the gate blocking and preserving + the locator, `--force` skipping it, WebDAV and legacy-locator skips, + `--ignore-absent` still reporting the failure, and the driver recorded at + mount time. +- `make e2e` drives a pending drain through `ti fs drain` and `ti fs unmount` + against the fake companion: both fail with the counters, `--force` unmounts + and reports `skipped`, a WebDAV unmount issues no drain call, and the drain + result carries `response.pending`. +- `make live-e2e-fs` asserts the real drain report fields on a live FUSE mount; + the live unmount now drains first. +- The e2e mount expectations pin `--driver fuse` wherever a forwarded tier is + asserted and branch the automatic-mode case on the host platform, so the + mandatory `make e2e` gate stays green on macOS and Windows as well as Linux. +- The `--ignore-absent` cases are covered both ways: pending work blocks even + when the engine classifies the failure as something else (unit and e2e), and a + drain that reports no pending work (no report, or zero counters with a + non-pending classification) continues with `drain.status: "failed"` on both + the `unmounted` and `absent` outcomes. +- The drain `--dry-run` check is pinned for both directions (WebDAV reports + `failed`, and a driver-less legacy locator reports `passed` because the drain + command still accepts it). +- Both tolerated shapes have their own fixture: a drain that produces no report + (`fs.drain_report_missing`, tolerated) and a parsed report with a non-pending + classification and zero counters (tolerated, `drain.status: "failed"`), plus + the pending shape that blocks under `--ignore-absent`. + +The English PingCAP pages under `docs/pingcap-docs/docs/` are a submodule that +was not checked out while this spec was implemented. The published mount and +unmount pages therefore still describe the pre-change CLI; the tracked follow-up +is issue #98 in this repository and must land before the next release cut. diff --git a/e2e/cli_test.go b/e2e/cli_test.go index 19761bf7..939b737c 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -1221,22 +1221,203 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { ) mixed.wantExitCode(0) + // FUSE is pinned wherever the test asserts a forwarded tier, so the + // expectations do not depend on the host's automatic mode (auto is WebDAV on + // macOS/Windows and FUSE elsewhere). The fake companion never touches real + // FUSE, so this costs no coverage. mountPath := filepath.Join(home, "mount") mount := runTIWithInput(t, bin, "", authEnv, "fs", "mount-file-system", "--mount-path", mountPath, + "--driver", "fuse", ) mount.wantExitCode(0) + mount.wantStdoutContains(`"durability": "interactive"`) + mount.wantStdoutContains(`"driver": "fuse"`) + mountText := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", filepath.Join(home, "mount-text"), + "--driver", "fuse", + "--output", "text", + ) + mountText.wantExitCode(0) + mountText.wantStdoutContains("Durability: interactive") + autoMountPath := filepath.Join(home, "mount-auto") + autoMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", autoMountPath, + ) + autoMount.wantExitCode(0) + // auto resolves to WebDAV on macOS/Windows and FUSE elsewhere. + wantAutoDurability := `"durability": "interactive"` + wantAutoDriver := `"driver": "fuse"` + wantAutoForwarded := "interactive" + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + wantAutoDurability = `"durability": "auto"` + wantAutoDriver = `"driver": "webdav"` + wantAutoForwarded = "" + } + autoMount.wantStdoutContains(wantAutoDurability) + autoMount.wantStdoutContains(wantAutoDriver) + strictMountPath := filepath.Join(home, "mount-close-sync") + strictMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", strictMountPath, + "--driver", "fuse", + "--durability", "close-sync", + ) + strictMount.wantExitCode(0) + strictMount.wantStdoutContains(`"durability": "close-sync"`) + invalidDurability := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", filepath.Join(home, "mount-invalid"), + "--durability", "turbo", + ) + invalidDurability.wantExitCode(2) + invalidDurability.wantStderrContains("auto, interactive, fsync, close-sync, write-sync") + webdavDurability := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", filepath.Join(home, "mount-webdav-strict"), + "--driver", "webdav", + "--durability", "write-sync", + ) + webdavDurability.wantExitCode(2) + webdavDurability.wantStderrContains("is not supported for WebDAV mounts") + webdavMountPath := filepath.Join(home, "mount-webdav") + webdavMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", webdavMountPath, + "--driver", "webdav", + ) + webdavMount.wantExitCode(0) + webdavMount.wantStdoutContains(`"durability": "auto"`) + // The drain dry-run predicts the drain command's own rule: a WebDAV mount + // cannot be drained, so the check fails instead of claiming success. + webdavDrainDryRun := runTIWithInput(t, bin, "", baseEnv, + "fs", "drain-file-system", + "--mount-path", webdavMountPath, + "--dry-run", + ) + webdavDrainDryRun.wantExitCode(0) + webdavDrainDryRun.wantStdoutContains(`"name": "drain_support"`) + webdavDrainDryRun.wantStdoutContains(`"status": "failed"`) + webdavDrain := runTIWithInput(t, bin, "", baseEnv, + "fs", "drain-file-system", + "--mount-path", webdavMountPath, + ) + webdavDrain.wantExitCode(2) + webdavDrain.wantStderrContains("no mount control socket") + for _, path := range []string{webdavMountPath, filepath.Join(home, "mount-text"), autoMountPath} { + extraUnmount := runTIWithInput(t, bin, "", baseEnv, + "fs", "unmount-file-system", + "--mount-path", path, + ) + extraUnmount.wantExitCode(0) + } drain := runTIWithInput(t, bin, "", baseEnv, "fs", "drain-file-system", "--mount-path", mountPath, ) drain.wantExitCode(0) + // The drain result must carry the engine's report instead of dropping the + // companion's --json output, and text output must show the counters. + drain.wantStdoutContains(`"response"`) + drain.wantStdoutContains(`"pending"`) + drainText := runTIWithInput(t, bin, "", baseEnv, + "fs", "drain-file-system", + "--mount-path", mountPath, + "--output", "text", + ) + drainText.wantExitCode(0) + drainText.wantStdoutContains("pending: none") unmount := runTIWithInput(t, bin, "", baseEnv, "fs", "unmount-file-system", "--mount-path", mountPath, ) unmount.wantExitCode(0) + unmount.wantStdoutContains(`"drain"`) + unmount.wantStdoutContains(`"status": "drained"`) + + // A pending drain blocks the unmount, keeps the locator for retry, and + // --force is the explicit way to accept the loss. + pendingDrainEnv := append(append([]string{}, baseEnv...), "FAKE_DRIVE9_DRAIN_PENDING=1") + pendingDrain := runTIWithInput(t, bin, "", pendingDrainEnv, + "fs", "drain-file-system", + "--mount-path", strictMountPath, + ) + pendingDrain.wantExitCode(1) + pendingDrain.wantStderrContains("error_kind=pending_work_remaining") + pendingDrain.wantStderrContains("commit_queue_pending=2") + pendingUnmount := runTIWithInput(t, bin, "", pendingDrainEnv, + "fs", "unmount-file-system", + "--mount-path", strictMountPath, + ) + pendingUnmount.wantExitCode(1) + pendingUnmount.wantStderrContains("error_kind=pending_work_remaining") + pendingUnmount.wantStderrContains("--force is the only way past this on unmount") + // --ignore-absent is the cleanup idempotency flag, not loss acceptance: it + // must not swallow pending writes either. + pendingIgnoreAbsent := runTIWithInput(t, bin, "", pendingDrainEnv, + "fs", "unmount-file-system", + "--mount-path", strictMountPath, + "--ignore-absent", + ) + pendingIgnoreAbsent.wantExitCode(1) + pendingIgnoreAbsent.wantStderrContains("--ignore-absent does not skip this check") + // A non-pending classification with queued work is still pending work. + pendingKindEnv := append(append([]string{}, baseEnv...), "FAKE_DRIVE9_DRAIN_PENDING_KIND=deadline_exceeded") + pendingKind := runTIWithInput(t, bin, "", pendingKindEnv, + "fs", "unmount-file-system", + "--mount-path", strictMountPath, + "--ignore-absent", + ) + pendingKind.wantExitCode(1) + pendingKind.wantStderrContains("error_kind=deadline_exceeded") + pendingKind.wantStderrContains("commit_queue_pending=3") + forcedUnmount := runTIWithInput(t, bin, "", pendingDrainEnv, + "fs", "unmount-file-system", + "--mount-path", strictMountPath, + "--force", + ) + forcedUnmount.wantExitCode(0) + forcedUnmount.wantStdoutContains(`"status": "skipped"`) + forcedUnmount.wantStdoutContains("--force skips the pre-unmount drain") + + // A parsed failure with a non-pending classification and no queued counters + // reports the failure but lets the cleanup flag continue. + zeroMountPath := filepath.Join(home, "mount-zero") + zeroMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", zeroMountPath, + "--driver", "fuse", + ) + zeroMount.wantExitCode(0) + zeroKindEnv := append(append([]string{}, baseEnv...), "FAKE_DRIVE9_DRAIN_ZERO_KIND=deadline_exceeded") + zeroKind := runTIWithInput(t, bin, "", zeroKindEnv, + "fs", "unmount-file-system", + "--mount-path", zeroMountPath, + "--ignore-absent", + ) + zeroKind.wantExitCode(0) + zeroKind.wantStdoutContains(`"status": "failed"`) + + // A drain that cannot reach the mount is not pending work, so the cleanup + // flag may continue, but the outcome still reports the failed drain. + goneMountPath := filepath.Join(home, "mount-gone") + goneMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", goneMountPath, + "--driver", "fuse", + ) + goneMount.wantExitCode(0) + goneEnv := append(append([]string{}, baseEnv...), "FAKE_DRIVE9_DRAIN_FAIL=1") + goneUnmount := runTIWithInput(t, bin, "", goneEnv, + "fs", "unmount-file-system", + "--mount-path", goneMountPath, + "--ignore-absent", + ) + goneUnmount.wantExitCode(0) + goneUnmount.wantStdoutContains(`"status": "failed"`) for _, path := range []string{ filepath.Join(home, ".tdc", "config"), @@ -1269,6 +1450,19 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { assertFakeDrive9Call(t, calls, []string{"fs", "ls"}, drive9TestToken("tenant-sandbox"), home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") assertFakeDrive9Call(t, calls, []string{"mount", "drain"}, "", home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") assertFakeDrive9Call(t, calls, []string{"umount"}, "", home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") + // The durability tier is forwarded verbatim, and a WebDAV mount forwards + // none because the engine has no durability tiers there. + assertFakeDrive9FlagValue(t, calls, "mount", mountPath, "--durability", "interactive") + assertFakeDrive9FlagValue(t, calls, "mount", strictMountPath, "--durability", "close-sync") + assertFakeDrive9FlagValue(t, calls, "mount", webdavMountPath, "--durability", "") + assertFakeDrive9FlagValue(t, calls, "mount", autoMountPath, "--durability", wantAutoForwarded) + // A FUSE unmount drains before detaching; a WebDAV unmount has no control + // socket and must not try. + for _, call := range calls { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" && call.Args[len(call.Args)-1] == webdavMountPath { + t.Fatalf("webdav unmount drained: %#v", call.Args) + } + } } func TestRuntimeExecCommandSurfaceAndFailureContract(t *testing.T) { @@ -1521,7 +1715,7 @@ func TestFSLayerForkWorkflowCommands(t *testing.T) { mounted := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", "/workspace", "--driver", "fuse", "--layer-ref", "child-id", "--checkpoint-id", "seed") mounted.wantExitCode(0) mounted.wantStdoutContains(`"read_only": true`) - mounted.wantStdoutContains(`"write_back_cache": false`) + mounted.wantStdoutContains(`"durability": "auto"`) mounted.wantStdoutContains(`"checkpoint_id": "seed"`) explicitWritable := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", filepath.Join(home, "writable-checkpoint"), "--driver", "fuse", "--layer-ref", "child-id", "--checkpoint-id", "seed", "--read-only=false") @@ -2101,6 +2295,32 @@ func assertFakeDrive9Args(t *testing.T, calls []fakeDrive9Call, want []string) { t.Fatalf("missing exact fake Drive9 args %v in %#v", want, calls) } +// assertFakeDrive9FlagValue finds the companion call for one mount path and +// asserts the flag/value pair it was invoked with. An empty value asserts the +// flag was not forwarded at all. +func assertFakeDrive9FlagValue(t *testing.T, calls []fakeDrive9Call, command, mountPath, flag, value string) { + t.Helper() + for _, call := range calls { + if len(call.Args) == 0 || call.Args[0] != command || call.Args[len(call.Args)-1] != mountPath { + continue + } + found := "" + for i := 0; i < len(call.Args); i++ { + if call.Args[i] != flag { + continue + } + if i+1 < len(call.Args) { + found = call.Args[i+1] + } + } + if found == value { + return + } + t.Fatalf("%s for %s used %s=%q, want %q in %#v", command, mountPath, flag, found, value, call.Args) + } + t.Fatalf("missing fake Drive9 %s call for mount path %s", command, mountPath) +} + func assertFakeDrive9TransientCall(t *testing.T, calls []fakeDrive9Call, prefix []string, apiKey, persistentHome, server, regionCode string) { t.Helper() for _, call := range calls { diff --git a/e2e/live_test.go b/e2e/live_test.go index c0e89cfa..650482e4 100644 --- a/e2e/live_test.go +++ b/e2e/live_test.go @@ -640,8 +640,9 @@ func TestLiveFSGitLifecycle(t *testing.T) { createDir := runTI(t, bin, "--profile", profileName, "fs", "create-directory", "--path", remoteRoot, "--mode", "0755") createDir.wantExitCode(0) - mount := runTI(t, bin, "--profile", profileName, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", remoteRoot, "--driver", "fuse", "--ready-timeout", "30s") + mount := runTI(t, bin, "--profile", profileName, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", remoteRoot, "--driver", "fuse", "--durability", "close-sync", "--ready-timeout", "30s") mount.wantExitCode(0) + mount.wantStdoutContains(`"durability": "close-sync"`) repoPath := filepath.Join(mountPath, "hello-world") clone := runTI(t, bin, "--profile", profileName, "fs-git", "clone-git-workspace", "--repo-url", "https://github.com/octocat/Hello-World.git", "--target-path", repoPath, "--blobless", "--hydrate", "sync") @@ -1586,6 +1587,11 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { if strings.Contains(mount.stdout, `"driver": "fuse"`) { drain := runTIWithInput(t, bin, "", locatorEnv, "fs", "drain-file-system", "--mount-path", mountPath, "--timeout", "30s") drain.wantExitCode(0) + // The drain report must be surfaced: the pending counters are how an + // operator tells "fully uploaded" from "the upload never finished". + drain.wantStdoutContains(`"response"`) + drain.wantStdoutContains(`"pending"`) + drain.wantStdoutContains(`"open_handles"`) } unmount := runTIWithInput(t, bin, "", locatorEnv, "fs", "unmount-file-system", "--mount-path", mountPath) unmount.wantExitCode(0) @@ -1720,6 +1726,8 @@ func TestLiveFSWebDAVMountRuntime(t *testing.T) { mount := runTI(t, bin, "--profile", profileName, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", remoteRoot, "--driver", "webdav", "--ready-timeout", "30s") mount.wantExitCode(0) + // WebDAV mounts have no durability tiers; ti forwards none and reports auto. + mount.wantStdoutContains(`"durability": "auto"`) mount.wantStdoutContains(`"status": "mounted"`) mount.wantStdoutContains(`"driver": "webdav"`) diff --git a/e2e/testdata/fake-drive9.go b/e2e/testdata/fake-drive9.go index 09e20b98..f437f404 100644 --- a/e2e/testdata/fake-drive9.go +++ b/e2e/testdata/fake-drive9.go @@ -7,6 +7,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "sort" "strings" "syscall" @@ -232,6 +233,41 @@ func main() { fmt.Println(`{"status":"unpacked"}`) return } + if hasPrefix(args, "mount", "drain") { + if os.Getenv("FAKE_DRIVE9_DRAIN_FAIL") == "1" { + fmt.Fprintln(os.Stderr, "drive9 mount drain: read mount state: no such file or directory") + os.Exit(1) + } + ok := os.Getenv("FAKE_DRIVE9_DRAIN_PENDING") != "1" + report := map[string]any{ + "ok": ok, "mount_point": args[len(args)-1], "duration_ms": 5, + "native_syncfs_supported": false, "pending": map[string]any{}, + } + if !ok { + report["error"] = "writeback uploader: 2 entries remain" + report["error_kind"] = "pending_work_remaining" + report["pending"] = map[string]any{"commit_queue_pending": 2, "uploader_queued": 2} + } + if kind := os.Getenv("FAKE_DRIVE9_DRAIN_ZERO_KIND"); kind != "" { + ok = false + report["ok"] = false + report["error"] = "drain deadline exceeded with nothing queued" + report["error_kind"] = kind + } + if kind := os.Getenv("FAKE_DRIVE9_DRAIN_PENDING_KIND"); kind != "" { + ok = false + report["ok"] = false + report["error"] = "drain deadline exceeded with queued work" + report["error_kind"] = kind + report["pending"] = map[string]any{"commit_queue_pending": 3, "uploader_queued": 4} + } + encoded, _ := json.Marshal(report) + fmt.Println(string(encoded)) + if !ok { + os.Exit(1) + } + return + } if hasPrefix(args, "mount") { fmt.Fprintln(os.Stderr, "drive9: mount mode: "+mountMode(args)) if mountPath := args[len(args)-1]; len(args) >= 2 && len(mountPath) > 0 && mountPath[0] != '-' { @@ -243,10 +279,15 @@ func main() { } func mountMode(args []string) string { - if value := flagValue(args, "--mode"); value != "" { + if value := flagValue(args, "--mode"); value != "" && value != "auto" { return value } - return "webdav" + // Mirror drive9's ResolveMountMode: auto is webdav on macOS/Windows and + // fuse everywhere else. + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return "webdav" + } + return "fuse" } func hasPrefix(args []string, want ...string) bool { diff --git a/internal/cli/commands.go b/internal/cli/commands.go index fee40ab8..13456f7f 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -2630,7 +2630,8 @@ func newFSMountFileSystemCommand(info version.Info) *cobra.Command { cmd.Flags().Int64("read-cache-size-mb", 128, "FUSE read cache size in MiB. 0 uses the default.") cmd.Flags().Int64("read-cache-max-file-mb", 4, "Maximum file size admitted to the FUSE read cache in MiB. 0 uses the default.") cmd.Flags().Duration("read-cache-ttl", 30*time.Second, "FUSE read cache Time-to-Live.") - cmd.Flags().Bool("write-back-cache", true, "Persist FUSE writes locally before writing them to the file system on flush; unavailable for checkpoint mounts.") + cmd.Flags().String("durability", tifs.DefaultMountDurability, "Write durability: auto, interactive, fsync, close-sync, or write-sync. FUSE only; WebDAV mounts reject every tier except auto.") + cmd.Flags().Bool("write-back-cache", true, "Legacy alias: true selects --durability=interactive and false selects --durability=auto. Unavailable for checkpoint mounts.") cmd.Flags().String("mount-profile", "", "Mount profile: coding-agent, portable, or none. Default: none.") cmd.Flags().String("local-root", "", "Local overlay root. Default: ~/.ti/local/fs/.") cmd.Flags().StringArray("pack-path", nil, "Local overlay path included by automatic or manual pack. Repeatable.") @@ -2674,7 +2675,7 @@ func newFSUnmountFileSystemCommand(info version.Info) *cobra.Command { }, info) cmd.Flags().String("mount-path", "", "The local mounted path.") cmd.Flags().Duration("timeout", 30*time.Second, "Time to wait for the mount process to exit.") - cmd.Flags().Bool("force", false, "Kill the mount process if graceful unmount times out.") + cmd.Flags().Bool("force", false, "Skip the pre-unmount drain gate and unmount even when local writes are still pending.") cmd.Flags().Bool("ignore-absent", false, "Return success when no file system mount state exists for the specified path.") cmd.Flags().String("pack-archive-path", "", "Pack archive to write after unmount") cmd.Flags().Bool("no-auto-pack", false, "Skip the portable mount profile's default auto-pack action.") @@ -3128,6 +3129,10 @@ func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFile if err != nil { return tifs.MountFileSystemOptions{}, err } + durability, err := ctx.StringFlag("durability") + if err != nil { + return tifs.MountFileSystemOptions{}, err + } writeBackCache, err := ctx.BoolFlag("write-back-cache") if err != nil { return tifs.MountFileSystemOptions{}, err @@ -3173,6 +3178,8 @@ func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFile ReadCacheMB: readCacheMB, ReadCacheFileMB: readCacheFileMB, ReadCacheTTL: readCacheTTL, + Durability: durability, + DurabilitySet: ctx.FlagChanged("durability"), WriteBackCache: writeBackCache, WriteBackCacheSet: ctx.FlagChanged("write-back-cache"), MountProfile: mountProfile, diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index e10a3415..6314011f 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -1271,7 +1271,7 @@ func (s Service) drive9MountVault(ctx context.Context, opts VaultMountOptions) ( if _, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{Profile: opts.Profile, Args: args, IncludeFSAPIKey: true, VaultToken: opts.VaultToken}); err != nil { return MountResult{}, err } - if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "vault"); err != nil { + if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "vault", "fuse"); err != nil { if umountErr := s.umountMount(ctx, MountFileSystemOptions{Profile: opts.Profile, MountPath: opts.MountPath}); umountErr != nil { // Compose an apperr error: the CLI renders apperr.MessageFor, which // reads only the embedded apperr message, so a fmt.Errorf wrapper would @@ -1283,7 +1283,7 @@ func (s Service) drive9MountVault(ctx context.Context, opts VaultMountOptions) ( if err := s.waitForMountReady(ctx, opts.MountPath, opts.ReadyTimeout, fmt.Sprintf("; to stop it run: ti fs-vault unmount-vault --mount-path %q", opts.MountPath)); err != nil { return MountResult{}, err } - return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: "vault", MountPath: opts.MountPath, RemotePath: "/n/vault", Driver: "fuse"}, nil + return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: "vault", MountPath: opts.MountPath, RemotePath: "/n/vault", Driver: "fuse", Durability: MountDurabilityAuto}, nil } func (s Service) drive9PackFileSystem(ctx context.Context, opts PackFileSystemOptions) (PackFileSystemResult, error) { @@ -1359,29 +1359,51 @@ func (s Service) drive9UnpackFileSystem(ctx context.Context, opts UnpackFileSyst } func (s Service) drive9DrainFileSystem(ctx context.Context, opts DrainFileSystemOptions) (DrainResult, error) { - profile, err := s.drive9MountLocatorProfile(opts.Profile, opts.MountPath) + locator, profile, err := s.mountLocatorProfile(opts.Profile, opts.MountPath) if err != nil { return DrainResult{}, err } - args := []string{"mount", "drain", "--json"} - if opts.Timeout > 0 { - args = append(args, "--timeout", opts.Timeout.String()) + // The companion's report is the only place the undrained work is counted, + // so keep it in the result instead of discarding it. + if drainable, reason := drainApplicability(locator); !drainable { + return DrainResult{}, apperr.New("fs.drain_not_applicable", "usage", 2, fmt.Sprintf("ti fs drain %s: %s", opts.MountPath, reason)) } - args = append(args, opts.MountPath) - if _, err := s.drive9Run(ctx, profile, args, true); err != nil { + drain, err := s.drainMount(ctx, profile, opts.MountPath, opts.Timeout) + if err != nil { return DrainResult{}, err } - return DrainResult{Status: "drained", MountPath: opts.MountPath}, nil + return DrainResult{Status: drain.Status, MountPath: opts.MountPath, Driver: locator.Driver, Response: drain.Response}, nil } func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSystemOptions) (UnmountResult, error) { - profile, err := s.drive9MountLocatorProfile(opts.Profile, opts.MountPath) + locator, profile, err := s.mountLocatorProfile(opts.Profile, opts.MountPath) if err != nil { if opts.IgnoreAbsent && apperr.CodeFor(err) == "fs.mount_locator_not_found" { return UnmountResult{Status: "absent", MountPath: opts.MountPath}, nil } return UnmountResult{}, err } + // Detaching the kernel mount stops the client from noticing that local + // write-back work never reached the remote, so drain first and refuse to + // detach while writes are still pending. --force skips the gate, and + // --ignore-absent tolerates a drain that fails because the mount is already + // gone. + drain, drainErr := s.drainBeforeUnmount(ctx, profile, locator, opts) + if drainErr != nil { + // --ignore-absent is the idempotency flag cleanup scripts pass, not a + // licence to drop writes. The error code is the single authority here: + // fs.drain_pending_work folds in both the engine's classification and + // the parsed pending counters, so a drain that times out against a + // write-back backlog still blocks. --ignore-absent may only tolerate a + // drain failure that reports no pending work: no report at all + // (typically a mount that is already gone, or a companion that predates + // the report) or a parsed report whose counters are all zero with a + // non-pending classification. Accepting actual loss stays --force's job + // alone. + if apperr.CodeFor(drainErr) == "fs.drain_pending_work" || !opts.IgnoreAbsent { + return UnmountResult{}, drainErr + } + } args := []string{"umount"} if opts.Timeout > 0 { args = append(args, "--timeout", opts.Timeout.String()) @@ -1402,7 +1424,11 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy if removeErr := mountlocator.Remove(homeDir, opts.MountPath); removeErr != nil { return UnmountResult{}, apperr.Wrap("fs.remove_mount_locator", "runtime", 1, "remove stale ti fs mount locator", removeErr) } - return UnmountResult{Status: "absent", MountPath: opts.MountPath}, nil + absent := UnmountResult{Status: "absent", MountPath: opts.MountPath, Driver: locator.Driver, Drain: &drain} + if drainErr != nil { + absent.Checks = append(absent.Checks, MountRuntimeCheck{Name: "drain", Status: "failed", Message: apperr.MessageFor(drainErr)}) + } + return absent, nil } return UnmountResult{}, err } @@ -1413,10 +1439,16 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy if err := mountlocator.Remove(homeDir, opts.MountPath); err != nil { return UnmountResult{}, apperr.Wrap("fs.remove_mount_locator", "runtime", 1, "remove ti fs mount locator", err) } - return UnmountResult{Status: "unmounted", MountPath: opts.MountPath}, nil + result := UnmountResult{Status: "unmounted", MountPath: opts.MountPath, Driver: locator.Driver, Drain: &drain} + if drainErr != nil { + // --ignore-absent asked ti to continue past a drain it could not + // complete, so the outcome must still say the writes may be lost. + result.Checks = append(result.Checks, MountRuntimeCheck{Name: "drain", Status: "failed", Message: apperr.MessageFor(drainErr)}) + } + return result, nil } -func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kind string, mountOpts ...MountFileSystemOptions) error { +func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kind, driver string, mountOpts ...MountFileSystemOptions) error { if profile == nil { return apperr.New("fs.missing_profile", "config", 2, "active profile is required") } @@ -1432,7 +1464,7 @@ func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kin if err != nil { return apperr.Wrap("fs.write_mount_locator", "runtime", 1, "construct ti fs mount locator", err) } - locator = locator.WithTokenCorrelation(profile.FSTenantID, profile.FSTokenID, fsTokenFingerprint(profile.FSAPIKey)).WithServiceAddress(profile.FSServiceAddress) + locator = locator.WithTokenCorrelation(profile.FSTenantID, profile.FSTokenID, fsTokenFingerprint(profile.FSAPIKey)).WithServiceAddress(profile.FSServiceAddress).WithDriver(driver) if len(mountOpts) > 0 { opts := mountOpts[0] locator = locator.WithLayerView(opts.RemotePath, opts.LayerRef, opts.CheckpointID, opts.ReadOnly) @@ -1443,28 +1475,31 @@ func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kin return nil } -func (s Service) drive9MountLocatorProfile(base *config.Profile, mountPath string) (*config.Profile, error) { +// mountLocatorProfile resolves the ti-owned mount locator for a mount path and +// rebuilds the companion profile from it. Callers that also need locator facts +// (the recorded driver, for example) use this instead of reading it twice. +func (s Service) mountLocatorProfile(base *config.Profile, mountPath string) (mountlocator.Locator, *config.Profile, error) { homeDir, err := s.homeDir() if err != nil { - return nil, err + return mountlocator.Locator{}, nil, err } locator, _, err := mountlocator.Read(homeDir, mountPath) if err != nil { if os.IsNotExist(err) { - return nil, apperr.New("fs.mount_locator_not_found", "runtime", 1, fmt.Sprintf("no ti fs mount locator found for %q", mountPath)) + return mountlocator.Locator{}, nil, apperr.New("fs.mount_locator_not_found", "runtime", 1, fmt.Sprintf("no ti fs mount locator found for %q", mountPath)) } - return nil, apperr.Wrap("fs.read_mount_locator", "runtime", 1, fmt.Sprintf("read ti fs mount locator for %q", mountPath), err) + return mountlocator.Locator{}, nil, apperr.Wrap("fs.read_mount_locator", "runtime", 1, fmt.Sprintf("read ti fs mount locator for %q", mountPath), err) } expectedHome, err := fscred.CompanionHome(homeDir, locator.Profile, locator.FileSystemName) if err != nil { - return nil, err + return mountlocator.Locator{}, nil, err } if filepath.Clean(expectedHome) != filepath.Clean(locator.CompanionHome) { - return nil, apperr.New("fs.mount_locator_invalid", "config", 2, "ti fs mount locator companion home does not match its profile and file system") + return mountlocator.Locator{}, nil, apperr.New("fs.mount_locator_invalid", "config", 2, "ti fs mount locator companion home does not match its profile and file system") } placement, err := region.ParseCanonicalCode(locator.RegionCode) if err != nil { - return nil, apperr.Wrap("fs.mount_locator_invalid", "config", 2, "ti fs mount locator has an invalid region", err) + return mountlocator.Locator{}, nil, apperr.Wrap("fs.mount_locator_invalid", "config", 2, "ti fs mount locator has an invalid region", err) } profile := config.Profile{HomeDir: homeDir} if base != nil { @@ -1488,7 +1523,7 @@ func (s Service) drive9MountLocatorProfile(base *config.Profile, mountPath strin profile.FSServiceAddress = credential.ServiceAddress } } - return &profile, nil + return locator, &profile, nil } func drive9CopyArgs(opts CopyFileOptions) ([]string, string, string, error) { diff --git a/internal/fs/drive9_fake_companion_test.go b/internal/fs/drive9_fake_companion_test.go index f7f81aff..fac7dd04 100644 --- a/internal/fs/drive9_fake_companion_test.go +++ b/internal/fs/drive9_fake_companion_test.go @@ -28,6 +28,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strconv" "time" ) @@ -98,6 +99,36 @@ func main() { "region_code": os.Getenv("DRIVE9_REGION_CODE"), "server": os.Getenv("DRIVE9_SERVER"), }) + case len(args) >= 2 && args[0] == "mount" && args[1] == "drain": + if os.Getenv("TI_FAKE_DRIVE9_DRAIN_FAIL") == "1" { + fmt.Fprintln(os.Stderr, "drive9 mount drain: read mount state: no such file or directory") + os.Exit(1) + } + pending := os.Getenv("TI_FAKE_DRIVE9_DRAIN_PENDING") == "1" + report := map[string]any{ + "ok": !pending, "mount_point": args[len(args)-1], "duration_ms": 7, + "native_syncfs_supported": false, "pending": map[string]any{}, + } + if pending { + report["error"] = "writeback uploader: 2 entries remain" + report["error_kind"] = "pending_work_remaining" + report["pending"] = map[string]any{"commit_queue_pending": 2, "uploader_queued": 2} + } + if kind := os.Getenv("TI_FAKE_DRIVE9_DRAIN_ZERO_KIND"); kind != "" { + report["ok"] = false + report["error"] = "drain deadline exceeded with nothing queued" + report["error_kind"] = kind + } + if kind := os.Getenv("TI_FAKE_DRIVE9_DRAIN_PENDING_KIND"); kind != "" { + report["ok"] = false + report["error"] = "drain deadline exceeded with queued work" + report["error_kind"] = kind + report["pending"] = map[string]any{"commit_queue_pending": 3, "uploader_queued": 4} + } + _ = json.NewEncoder(os.Stdout).Encode(report) + if pending { + os.Exit(1) + } case args[0] == "umount": if os.Getenv("TI_FAKE_UMOUNT_FAIL") == "1" { fmt.Fprintln(os.Stderr, "umount: mount is busy") @@ -105,11 +136,7 @@ func main() { } case args[0] == "mount" && (len(args) == 1 || args[1] != "drain"): fmt.Fprintln(os.Stderr, "component: drive9 mount") - if flagValue(args, "--mode") == "fuse" { - fmt.Fprintln(os.Stderr, "drive9: mount mode: fuse") - } else { - fmt.Fprintln(os.Stderr, "drive9: mount mode: webdav") - } + fmt.Fprintln(os.Stderr, "drive9: mount mode: "+fakeMountMode(args)) if os.Getenv("TI_FAKE_DRIVE9_MOUNT_FAIL") == "1" { fmt.Fprintln(os.Stderr, "mount: drive9 mount: background mount exited before becoming ready") os.Exit(1) @@ -268,6 +295,18 @@ func record(args []string) { _ = json.NewEncoder(f).Encode(callRecord{Args: args, Env: env}) } +// fakeMountMode mirrors drive9's ResolveMountMode: an explicit mode wins, and +// auto means webdav on macOS/Windows and fuse elsewhere. +func fakeMountMode(args []string) string { + if mode := flagValue(args, "--mode"); mode != "" && mode != "auto" { + return mode + } + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return "webdav" + } + return "fuse" +} + func flagValue(args []string, name string) string { for i := 0; i+1 < len(args); i++ { if args[i] == name { diff --git a/internal/fs/drive9_mount_audit.go b/internal/fs/drive9_mount_audit.go index 6f909f74..6b2b7745 100644 --- a/internal/fs/drive9_mount_audit.go +++ b/internal/fs/drive9_mount_audit.go @@ -96,9 +96,9 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem if opts.ReadCacheTTL > 0 { args = append(args, "--read-cache-ttl", opts.ReadCacheTTL.String()) } - if opts.WriteBackCache { - args = append(args, "--durability", "interactive") - } + } + if mountDurabilityForwarded(opts) { + appendFlagValue(&args, "--durability", opts.Durability) } appendFlagValue(&args, "--profile", opts.MountProfile) appendFlagValue(&args, "--local-root", opts.LocalRoot) @@ -126,7 +126,10 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem } return MountResult{}, runErr } - if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "fs", opts); err != nil { + // Record the driver the companion actually mounted with, so the matching + // unmount knows whether the mount supports the drain gate. + mountedDriver := drive9MountedDriver(result.Stderr, opts.Driver) + if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "fs", mountedDriver, opts); err != nil { // A partially completed unpack can leave an escaping link behind, and a // later mount would record it as pre-existing state, so audit before // deciding whether the failed mount can be torn down. @@ -161,11 +164,19 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem return MountResult{}, readyErr } endpoint, _ := s.resolveFS(opts.Profile) - driver := drive9MountedDriver(result.Stderr, opts.Driver) + driver := mountedDriver if driver == "" { driver = "auto" } - return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: opts.Profile.FSResourceName, MountPath: opts.MountPath, RemotePath: remotePath, Driver: driver, Endpoint: &endpoint, MountProfile: opts.MountProfile, LocalRoot: auditRoot, PackPaths: opts.PackPaths, WriteBackCache: opts.WriteBackCache, LayerRef: opts.LayerRef, CheckpointID: opts.CheckpointID, ReadOnly: opts.ReadOnly}, nil + // Report the tier that is actually in effect. ti forwards --durability only + // for mounts its companion-mode mirror calls FUSE; if the companion still + // mounted WebDAV, no tier applies and reporting "interactive" would claim a + // guarantee the mount does not have. + durability := opts.Durability + if driver == "webdav" { + durability = MountDurabilityAuto + } + return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: opts.Profile.FSResourceName, MountPath: opts.MountPath, RemotePath: remotePath, Driver: driver, Endpoint: &endpoint, MountProfile: opts.MountProfile, LocalRoot: auditRoot, PackPaths: opts.PackPaths, Durability: durability, LayerRef: opts.LayerRef, CheckpointID: opts.CheckpointID, ReadOnly: opts.ReadOnly}, nil } // stopMountAfterUnsafeAudit tears down a mount whose overlay failed the audit. It diff --git a/internal/fs/drive9_mount_audit_test.go b/internal/fs/drive9_mount_audit_test.go index bb760e05..8f957dcf 100644 --- a/internal/fs/drive9_mount_audit_test.go +++ b/internal/fs/drive9_mount_audit_test.go @@ -26,11 +26,14 @@ func TestDrive9MountLocatorRoutesDrainAndUnmountWithoutCredentials(t *testing.T) mountPath := filepath.Join(t.TempDir(), "workspace") profile := dataProfile() + // An explicit FUSE driver keeps this routing test independent of the host's + // automatic mode: on macOS/Windows `auto` is WebDAV, which has no drain. if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ Profile: profile, FileSystemName: "workspace", MountPath: mountPath, RemotePath: "/", + Driver: "fuse", }); err != nil { t.Fatalf("MountFileSystem failed: %v", err) } @@ -707,7 +710,13 @@ func TestDrive9MountSuppressesCompanionSuccessChatter(t *testing.T) { if stdout.Len() != 0 || stderr.Len() != 0 { t.Fatalf("companion mount output leaked to ti output: stdout=%q stderr=%q", stdout.String(), stderr.String()) } - if result.Driver != "webdav" || result.Status != "mounted" { + // auto is webdav on macOS/Windows and fuse elsewhere; the point is that the + // driver comes from the companion's own report, not from opts.Driver. + wantDriver := "fuse" + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + wantDriver = "webdav" + } + if result.Driver != wantDriver || result.Status != "mounted" { t.Fatalf("mount result did not retain captured companion state: %#v", result) } } diff --git a/internal/fs/drive9_mount_drain.go b/internal/fs/drive9_mount_drain.go new file mode 100644 index 00000000..b5b2fc05 --- /dev/null +++ b/internal/fs/drive9_mount_drain.go @@ -0,0 +1,215 @@ +package fs + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/tidbcloud/ti-cli/internal/apperr" + "github.com/tidbcloud/ti-cli/internal/config" + "github.com/tidbcloud/ti-cli/internal/fs/mountcontrol" + "github.com/tidbcloud/ti-cli/internal/fs/mountlocator" +) + +// decodeDrive9DrainResponse parses the `mount drain --json` report. The +// companion prints the report before it exits, including when the drain failed, +// so a caller can report the undrained work instead of only an exit status. +func decodeDrive9DrainResponse(stdout []byte) (*mountcontrol.DrainResponse, error) { + trimmed := bytes.TrimSpace(stdout) + if len(trimmed) == 0 { + return nil, apperr.New("fs.drain_report_missing", "runtime", 1, "the ti fs companion printed no drain report; update ti-drive9 so `mount drain --json` reports pending work") + } + var response mountcontrol.DrainResponse + if err := json.Unmarshal(trimmed, &response); err != nil { + return nil, apperr.Wrap("fs.drain_report_invalid", "runtime", 1, "decode the ti-drive9 drain report", err) + } + return &response, nil +} + +// drainPendingSummary renders the undrained counters the engine reports, so an +// operator can tell "nothing was pending" from "the upload of eight files never +// finished" without reading the raw JSON. +func drainPendingSummary(pending mountcontrol.DrainPending) string { + counters := []struct { + name string + value int64 + }{ + {"open_handles", int64(pending.OpenHandles)}, + {"dirty_handles", int64(pending.DirtyHandles)}, + {"commit_queue_pending", int64(pending.CommitQueuePending)}, + {"commit_queue_bytes", pending.CommitQueueBytes}, + {"commit_queue_in_flight", int64(pending.CommitQueueInFlight)}, + {"commit_queue_delayed", int64(pending.CommitQueueDelayed)}, + {"commit_queue_conflicts", int64(pending.CommitQueueConflicts)}, + {"uploader_queued", int64(pending.UploaderQueued)}, + {"uploader_in_flight", int64(pending.UploaderInFlight)}, + {"uploader_cached", int64(pending.UploaderCached)}, + {"uploader_cached_bytes", pending.UploaderCachedBytes}, + } + rendered := make([]string, 0, len(counters)) + for _, counter := range counters { + if counter.value == 0 { + continue + } + rendered = append(rendered, fmt.Sprintf("%s=%d", counter.name, counter.value)) + } + if !pending.HasPendingWork() { + return "none" + } + return strings.Join(rendered, " ") +} + +// describeDrainFailure renders the engine's drain report for a user-facing +// error: what failed, which path, and what work is still undrained. +func describeDrainFailure(response *mountcontrol.DrainResponse) string { + if response == nil { + return "the companion reported no drain detail" + } + parts := make([]string, 0, 5) + if response.MountPoint != "" { + parts = append(parts, fmt.Sprintf("mount %q", response.MountPoint)) + } + if response.ErrorKind != "" { + parts = append(parts, "error_kind="+response.ErrorKind) + } + if response.Error != "" { + parts = append(parts, "error="+response.Error) + } + parts = append(parts, "pending: "+drainPendingSummary(response.Pending)) + return strings.Join(parts, "; ") +} + +// drainErrorCode classifies a drain failure. Undrained write-back work is the +// dangerous case (the data is still only local), so it gets its own code. +func drainErrorCode(response *mountcontrol.DrainResponse) string { + if response != nil && (response.ErrorKind == "pending_work_remaining" || response.Pending.HasPendingWork()) { + return "fs.drain_pending_work" + } + return "fs.drain_failed" +} + +// mountKindFileSystem marks a ti fs mount in the mount locator. Vault mounts +// record their own kind and are never drained: the engine drains only FUSE +// file-system mounts, and a vault mount has its own control plane. +const mountKindFileSystem = "fs" + +// mountDriverSupportsDrain reports whether a recorded mount driver can be +// drained. A FUSE mount exposes a control socket; WebDAV keeps the engine's +// native write behavior and has no drain. An unknown driver (a locator written +// before the driver was recorded) is not gated, so an upgrade never blocks an +// existing mount. +func mountDriverSupportsDrain(driver string) bool { + switch strings.TrimSpace(driver) { + case "": + return false + case "webdav": + return false + case "fuse": + return true + default: + return companionMountMode(driver) == "fuse" + } +} + +// mountDrainGate reports whether an unmount of this locator should drain first, +// and why it will not. Only a ti fs FUSE mount has a drain gate. +func mountDrainGate(locator mountlocator.Locator, force bool) (bool, string) { + if force { + return false, "unmount --force skips the pre-unmount drain" + } + switch strings.TrimSpace(locator.Kind) { + case mountKindFileSystem: + case "": + return false, "the mount locator records no mount kind, so ti cannot confirm the mount supports drain" + default: + return false, fmt.Sprintf("%s mounts have no drain gate; the companion stops them as they are", strings.TrimSpace(locator.Kind)) + } + if !mountDriverSupportsDrain(locator.Driver) { + return false, mountDrainSkipReason(locator.Driver) + } + return true, "" +} + +// drainApplicability reports whether `ti fs drain-file-system` can run at all. +// It rejects the mounts whose dry-run already says "not applicable" so the +// command fails with the same explanation instead of a raw companion error. An +// unrecorded driver stays drainable: the command itself asks for a drain, and +// only the unmount gate needs the stricter driver check. +func drainApplicability(locator mountlocator.Locator) (bool, string) { + if kind := strings.TrimSpace(locator.Kind); kind != "" && kind != mountKindFileSystem { + return false, fmt.Sprintf("%s mounts have no drain; the engine drains only file-system mounts", kind) + } + if strings.TrimSpace(locator.Driver) == "webdav" { + return false, mountDrainSkipReason(locator.Driver) + } + return true, "" +} + +// mountDrainSkipReason explains why a driver cannot be drained. +func mountDrainSkipReason(driver string) string { + if strings.TrimSpace(driver) == "" { + return "the mount locator records no driver, so ti cannot confirm the mount supports drain" + } + return fmt.Sprintf("%s mounts have no mount control socket to drain; the companion flushes them when it unmounts", strings.TrimSpace(driver)) +} + +// drainRemedy builds the actionable tail of a drain error from its +// classification. Pending work can only be skipped with --force; a failure that +// produced no report is typically a mount that is already gone, which +// --ignore-absent tolerates on unmount. +func drainRemedy(code, mountPath string) string { + if code == "fs.drain_pending_work" { + return fmt.Sprintf("close the writers or let the background upload finish, then retry ti fs drain-file-system --mount-path %q; --force is the only way past this on unmount (--ignore-absent does not skip this check)", mountPath) + } + return fmt.Sprintf("if the mount is already gone, ti fs unmount-file-system --mount-path %q --ignore-absent continues; --force accepts losing any undrained writes", mountPath) +} + +// drainMount runs the companion's drain and reports its result. It returns the +// drain record even when the drain failed, so a caller that continues anyway +// (unmount --force or --ignore-absent) can say what was left undrained. +func (s Service) drainMount(ctx context.Context, profile *config.Profile, mountPath string, timeout time.Duration) (MountDrain, error) { + args := []string{"mount", "drain", "--json"} + if timeout > 0 { + args = append(args, "--timeout", timeout.String()) + } + args = append(args, mountPath) + result, runErr := s.drive9Run(ctx, profile, args, true) + response, decodeErr := decodeDrive9DrainResponse(result.Stdout) + if decodeErr != nil { + // No usable report: the mount is most likely gone, or the companion + // predates the report. That is a drain failure, not pending work, so + // the error has to carry the same unmount escape hatches as a classified + // failure instead of surfacing a raw companion error. + record := MountDrain{Status: "failed", Message: apperr.MessageFor(decodeErr)} + reason := apperr.MessageFor(decodeErr) + if runErr != nil { + reason = fmt.Sprintf("%v", runErr) + } + // Keep the report-shaped code: "no report" and "unparseable report" + // tell an operator different things (a gone mount or an old companion + // versus a broken companion), and both are tolerated by + // --ignore-absent because neither shows pending work. + code := apperr.CodeFor(decodeErr) + return record, apperr.New(code, "runtime", 1, fmt.Sprintf("drain %s produced no usable report: %s; %s", mountPath, reason, drainRemedy(code, mountPath))) + } + if runErr != nil || !response.OK { + code := drainErrorCode(response) + record := MountDrain{Status: "failed", Message: describeDrainFailure(response), Response: response} + return record, apperr.New(code, "runtime", 1, fmt.Sprintf("drain %s did not finish: %s; %s", mountPath, describeDrainFailure(response), drainRemedy(code, mountPath))) + } + return MountDrain{Status: "drained", Response: response}, nil +} + +// drainBeforeUnmount is the unmount gate: an ordinary unmount drains first and +// refuses to detach while local writes are still pending. --force skips the +// gate explicitly, and a mount that cannot be drained (WebDAV, or a locator +// written before ti recorded the driver) is not gated. +func (s Service) drainBeforeUnmount(ctx context.Context, profile *config.Profile, locator mountlocator.Locator, opts UnmountFileSystemOptions) (MountDrain, error) { + if drainable, reason := mountDrainGate(locator, opts.Force); !drainable { + return MountDrain{Status: "skipped", Message: reason}, nil + } + return s.drainMount(ctx, profile, opts.MountPath, drainTimeout(opts.Timeout)) +} diff --git a/internal/fs/drive9_mount_drain_test.go b/internal/fs/drive9_mount_drain_test.go new file mode 100644 index 00000000..592f67ba --- /dev/null +++ b/internal/fs/drive9_mount_drain_test.go @@ -0,0 +1,589 @@ +package fs + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tidbcloud/ti-cli/internal/apperr" + "github.com/tidbcloud/ti-cli/internal/fs/fscred" + "github.com/tidbcloud/ti-cli/internal/fs/mountcontrol" + "github.com/tidbcloud/ti-cli/internal/fs/mountlocator" +) + +// TestDrive9DrainReportsPendingCounters keeps the engine's drain report in the +// result instead of discarding the `--json` output. +func TestDrive9DrainReportsPendingCounters(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + + result, err := service.DrainFileSystem(context.Background(), DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("drain failed: %v", err) + } + if result.Status != "drained" || result.Response == nil || !result.Response.OK { + t.Fatalf("unexpected drain result: %#v", result) + } + if result.Driver != "fuse" { + t.Fatalf("drain driver = %q, want fuse", result.Driver) + } + if result.Response.DurationMS != 7 { + t.Fatalf("drain duration = %d, want the companion's report", result.Response.DurationMS) + } + human := result.Human() + if !strings.Contains(human, "pending: none") { + t.Fatalf("text output dropped the pending counters:\n%s", human) + } + mountHuman := MountResult{Status: "mounted", MountPath: "/mnt", Driver: "fuse", Durability: MountDurabilityCloseSync}.Human() + if !strings.Contains(mountHuman, "Durability: close-sync") { + t.Fatalf("mount text output dropped the effective tier:\n%s", mountHuman) + } +} + +// TestDrive9DrainFailureNamesPendingWork turns an undrained mount into an +// actionable error with the counters that explain it. +func TestDrive9DrainFailureNamesPendingWork(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + + _, err := service.DrainFileSystem(context.Background(), DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if apperr.CodeFor(err) != "fs.drain_pending_work" { + t.Fatalf("drain error = %v, want fs.drain_pending_work", err) + } + message := apperr.MessageFor(err) + for _, want := range []string{"pending_work_remaining", "commit_queue_pending=2", "uploader_queued=2", "ti fs drain-file-system"} { + if !strings.Contains(message, want) { + t.Fatalf("drain error %q missing %q", message, want) + } + } +} + +// TestUnmountDrainsBeforeDetach is the core fix: unmount must not detach while +// local writes are still pending. +func TestUnmountDrainsBeforeDetach(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("unmount failed: %v", err) + } + if result.Status != "unmounted" || result.Drain == nil || result.Drain.Status != "drained" { + t.Fatalf("unexpected unmount result: %#v", result) + } + drainIndex, umountIndex := -1, -1 + for i, call := range readFakeDrive9Calls(t, recordPath) { + switch { + case len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain": + drainIndex = i + case len(call.Args) >= 1 && call.Args[0] == "umount": + umountIndex = i + } + } + if drainIndex < 0 || umountIndex < 0 || drainIndex > umountIndex { + t.Fatalf("drain did not run before umount: drain=%d umount=%d", drainIndex, umountIndex) + } + if !strings.Contains(result.Human(), "drain") { + t.Fatalf("text output dropped the drain outcome:\n%s", result.Human()) + } +} + +// TestUnmountRefusesToDetachWithPendingWork blocks the unmount and keeps the +// locator so the write loss is explicit and retryable. +func TestUnmountRefusesToDetachWithPendingWork(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + + _, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if apperr.CodeFor(err) != "fs.drain_pending_work" { + t.Fatalf("unmount error = %v, want fs.drain_pending_work", err) + } + if _, _, locErr := mountlocator.Read(home, mountPath); locErr != nil { + t.Fatalf("blocked unmount must preserve the locator: %v", locErr) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) > 0 && call.Args[0] == "umount" { + t.Fatalf("unmount ran despite pending writes: %#v", call.Args) + } + } +} + +// TestUnmountForceSkipsDrainGate gives the operator an explicit way to accept +// the loss, and records that it was skipped. +func TestUnmountForceSkipsDrainGate(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, Force: true}) + if err != nil { + t.Fatalf("forced unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "skipped" || !strings.Contains(result.Drain.Message, "--force") { + t.Fatalf("forced unmount did not record the skipped gate: %#v", result.Drain) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("--force still drained: %#v", call.Args) + } + } +} + +// TestUnmountSkipsDrainForWebDAV keeps WebDAV unmount working: there is no +// control socket to drain. +func TestUnmountSkipsDrainForWebDAV(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "webdav", + }); err != nil { + t.Fatalf("webdav mount failed: %v", err) + } + locator, _, err := mountlocator.Read(home, mountPath) + if err != nil { + t.Fatalf("read locator: %v", err) + } + if locator.Driver != "webdav" { + t.Fatalf("locator driver = %q, want webdav", locator.Driver) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("webdav unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "skipped" || !strings.Contains(result.Drain.Message, "webdav") { + t.Fatalf("webdav unmount drain record = %#v", result.Drain) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("webdav unmount drained: %#v", call.Args) + } + } +} + +// TestUnmountSkipsDrainForVaultMount keeps `ti fs-vault unmount-vault` working: +// a vault mount has no file-system drain, and the engine rejects one. +func TestUnmountSkipsDrainForVaultMount(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + mountPath := filepath.Join(t.TempDir(), "vault") + companionHome, err := fscred.CompanionHome(home, "default", "tenant-1") + if err != nil { + t.Fatalf("resolve companion home: %v", err) + } + locator, err := mountlocator.New("default", "tenant-1", "aws-us-east-1", companionHome, mountPath, "vault") + if err != nil { + t.Fatalf("construct locator: %v", err) + } + if _, err := mountlocator.Write(home, locator.WithDriver("fuse")); err != nil { + t.Fatalf("write locator: %v", err) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("vault unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "skipped" || !strings.Contains(result.Drain.Message, "vault") { + t.Fatalf("vault drain record = %#v", result.Drain) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("vault unmount drained: %#v", call.Args) + } + } +} + +// TestUnmountSkipsDrainForLegacyLocator never gates a mount recorded before the +// driver was written, so an upgrade cannot block an existing unmount. +func TestUnmountSkipsDrainForLegacyLocator(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + profile := dataProfile() + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := mountlocator.Write(home, legacyTestLocator(t, home, mountPath)); err != nil { + t.Fatalf("write locator: %v", err) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: profile, MountPath: mountPath}) + if err != nil { + t.Fatalf("legacy unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "skipped" || !strings.Contains(result.Drain.Message, "no driver") { + t.Fatalf("legacy locator drain record = %#v", result.Drain) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("legacy locator drained: %#v", call.Args) + } + } +} + +// TestUnmountIgnoreAbsentStillBlocksPendingWork keeps --ignore-absent from +// becoming a silent loss-acceptance flag: it is the idempotency flag cleanup +// scripts pass, so pending writes must still block the detach. +func TestUnmountIgnoreAbsentStillBlocksPendingWork(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + + _, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if apperr.CodeFor(err) != "fs.drain_pending_work" { + t.Fatalf("ignore-absent unmount error = %v, want fs.drain_pending_work", err) + } + if !strings.Contains(apperr.MessageFor(err), "--force") || !strings.Contains(apperr.MessageFor(err), "--ignore-absent does not skip") { + t.Fatalf("blocked drain error must name both flags: %v", apperr.MessageFor(err)) + } + if _, _, locErr := mountlocator.Read(home, mountPath); locErr != nil { + t.Fatalf("blocked unmount must preserve the locator: %v", locErr) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) > 0 && call.Args[0] == "umount" { + t.Fatalf("unmount ran despite pending writes: %#v", call.Args) + } + } +} + +// TestUnmountIgnoreAbsentToleratesGoneMount keeps cleanup idempotent: a drain +// that cannot reach the mount (not pending work) may continue. +func TestUnmountIgnoreAbsentToleratesGoneMount(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_FAIL", "1") + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if err != nil { + t.Fatalf("ignore-absent unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "failed" { + t.Fatalf("drain record = %#v, want failed", result.Drain) + } + found := false + for _, check := range result.Checks { + if check.Name == "drain" && check.Status == "failed" { + found = true + } + } + if !found { + t.Fatalf("unmount did not report the failed drain: %#v", result.Checks) + } +} + +// TestUnmountIgnoreAbsentAbsentPathStillReportsDrainFailure keeps the two +// success shapes aligned: when the unmount itself reports the mount absent, the +// tolerated drain failure is still recorded. +func TestUnmountIgnoreAbsentAbsentPathStillReportsDrainFailure(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_FAIL", "1") + t.Setenv("TI_FAKE_UMOUNT_FAIL", "1") + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if err != nil { + t.Fatalf("ignore-absent unmount failed: %v", err) + } + if result.Status != "absent" || result.Drain == nil || result.Drain.Status != "failed" { + t.Fatalf("unexpected absent result: %#v", result) + } + found := false + for _, check := range result.Checks { + if check.Name == "drain" && check.Status == "failed" { + found = true + } + } + if !found { + t.Fatalf("absent path did not report the tolerated drain failure: %#v", result.Checks) + } +} + +// TestUnmountIgnoreAbsentStillBlocksParsedPendingWork covers the classification +// hole: a drain that fails with a non-pending error_kind while the report shows +// queued work must still block. +func TestUnmountIgnoreAbsentStillBlocksParsedPendingWork(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING_KIND", "deadline_exceeded") + + _, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if apperr.CodeFor(err) != "fs.drain_pending_work" { + t.Fatalf("ignore-absent unmount error = %v, want fs.drain_pending_work", err) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) > 0 && call.Args[0] == "umount" { + t.Fatalf("unmount ran with queued work: %#v", call.Args) + } + } +} + +// TestUnmountIgnoreAbsentToleratesZeroCounterFailure pins the tolerated shape: +// a parsed report with a non-pending classification and no queued counters +// reports drain.status "failed" but lets the cleanup unmount continue. +func TestUnmountIgnoreAbsentToleratesZeroCounterFailure(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_ZERO_KIND", "deadline_exceeded") + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if err != nil { + t.Fatalf("ignore-absent unmount failed: %v", err) + } + if result.Status != "unmounted" || result.Drain == nil || result.Drain.Status != "failed" { + t.Fatalf("unexpected unmount result: %#v", result) + } + if result.Drain.Response == nil || result.Drain.Response.Pending.HasPendingWork() { + t.Fatalf("fixture was supposed to report no queued work: %#v", result.Drain.Response) + } +} + +// TestDryRunDrainSupportForDriverlessLocator keeps the drain dry run honest for +// mounts whose locator predates the recorded driver: the drain command still +// accepts them, so the check must not claim it cannot run. +func TestDryRunDrainSupportForDriverlessLocator(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + service := testCompanionService(home, companion) + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := mountlocator.Write(home, legacyTestLocator(t, home, mountPath)); err != nil { + t.Fatalf("write locator: %v", err) + } + result, err := service.DryRunDrainFileSystem(context.Background(), "ti fs drain-file-system", DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("dry run failed: %v", err) + } + for _, check := range result.Checks { + if check.Name != "drain_support" { + continue + } + if check.Status != "passed" { + t.Fatalf("driver-less drain dry run = %#v, want passed", check) + } + return + } + t.Fatal("drain_support check missing") +} + +// TestHasPendingWorkIsIndependentOfRendering pins every counter the predicate +// enumerates and keeps it from ever depending on the operator-facing summary. +func TestHasPendingWorkIsIndependentOfRendering(t *testing.T) { + for _, tc := range []struct { + name string + pending mountcontrol.DrainPending + want bool + }{ + {name: "empty", pending: mountcontrol.DrainPending{}, want: false}, + {name: "open handles", pending: mountcontrol.DrainPending{OpenHandles: 1}, want: true}, + {name: "dirty handles", pending: mountcontrol.DrainPending{DirtyHandles: 1}, want: true}, + {name: "commit queue pending", pending: mountcontrol.DrainPending{CommitQueuePending: 1}, want: true}, + {name: "commit queue bytes", pending: mountcontrol.DrainPending{CommitQueueBytes: 1}, want: true}, + {name: "commit queue in flight", pending: mountcontrol.DrainPending{CommitQueueInFlight: 1}, want: true}, + {name: "commit queue delayed", pending: mountcontrol.DrainPending{CommitQueueDelayed: 1}, want: true}, + {name: "commit queue conflicts", pending: mountcontrol.DrainPending{CommitQueueConflicts: 1}, want: true}, + {name: "uploader queued", pending: mountcontrol.DrainPending{UploaderQueued: 1}, want: true}, + {name: "uploader in flight", pending: mountcontrol.DrainPending{UploaderInFlight: 1}, want: true}, + {name: "uploader cached", pending: mountcontrol.DrainPending{UploaderCached: 1}, want: true}, + {name: "uploader cached bytes", pending: mountcontrol.DrainPending{UploaderCachedBytes: 1}, want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got := tc.pending.HasPendingWork() + if got != tc.want { + t.Fatalf("HasPendingWork(%+v) = %t, want %t", tc.pending, got, tc.want) + } + summary := drainPendingSummary(tc.pending) + if tc.want != (summary != "none") { + t.Fatalf("summary %q disagrees with the predicate for %+v", summary, tc.pending) + } + }) + } +} + +// TestDrainRemedyMatchesClassification keeps the error guidance accurate: only +// pending work is unbailable by --ignore-absent. +func TestDrainRemedyMatchesClassification(t *testing.T) { + pending := drainRemedy("fs.drain_pending_work", "/mnt") + if !strings.Contains(pending, "--force") || !strings.Contains(pending, "--ignore-absent does not skip") { + t.Fatalf("pending remedy = %q", pending) + } + other := drainRemedy("fs.drain_failed", "/mnt") + if strings.Contains(other, "does not skip") || !strings.Contains(other, "--ignore-absent") || !strings.Contains(other, "--force") { + t.Fatalf("non-pending remedy = %q", other) + } +} + +// TestDrainRejectsNonFileSystemMount keeps the drain command from reaching the +// companion with a mount the engine cannot drain. +func TestDrainRejectsNonFileSystemMount(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + mountPath := filepath.Join(t.TempDir(), "vault") + if _, err := mountlocator.Write(home, legacyTestLocatorWithKind(t, home, mountPath, "vault", "fuse")); err != nil { + t.Fatalf("write locator: %v", err) + } + if drainable, _ := drainApplicability(mountlocator.Locator{Kind: mountKindFileSystem, Driver: "webdav"}); drainable { + t.Fatal("webdav locator reported drainable") + } + if drainable, _ := drainApplicability(mountlocator.Locator{Kind: mountKindFileSystem}); !drainable { + t.Fatal("locator with an unrecorded driver must stay drainable") + } + if _, err := service.DrainFileSystem(context.Background(), DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}); apperr.CodeFor(err) != "fs.drain_not_applicable" { + t.Fatalf("vault drain error = %v, want fs.drain_not_applicable", err) + } + if _, statErr := os.Stat(recordPath); statErr == nil { + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("vault drain reached the companion: %#v", call.Args) + } + } + } +} + +// TestMountRecordsDriverInLocator records what the companion actually mounted, +// because the unmount gate depends on it. +func TestMountRecordsDriverInLocator(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + locator, _, err := mountlocator.Read(home, mountPath) + if err != nil { + t.Fatalf("read locator: %v", err) + } + if locator.Driver != "fuse" { + t.Fatalf("locator driver = %q, want fuse", locator.Driver) + } +} + +func legacyTestLocatorWithKind(t *testing.T, home, mountPath, kind, driver string) mountlocator.Locator { + t.Helper() + companionHome, err := fscred.CompanionHome(home, "default", "tenant-1") + if err != nil { + t.Fatalf("resolve companion home: %v", err) + } + locator, err := mountlocator.New("default", "tenant-1", "aws-us-east-1", companionHome, mountPath, kind) + if err != nil { + t.Fatalf("construct locator: %v", err) + } + return locator.WithDriver(driver) +} + +func legacyTestLocator(t *testing.T, home, mountPath string) mountlocator.Locator { + t.Helper() + companionHome, err := fscred.CompanionHome(home, "default", "tenant-1") + if err != nil { + t.Fatalf("resolve companion home: %v", err) + } + locator, err := mountlocator.New("default", "tenant-1", "aws-us-east-1", companionHome, mountPath, "fs") + if err != nil { + t.Fatalf("construct locator: %v", err) + } + return locator +} diff --git a/internal/fs/mount.go b/internal/fs/mount.go index e05afad0..e5cfbca3 100644 --- a/internal/fs/mount.go +++ b/internal/fs/mount.go @@ -20,6 +20,7 @@ import ( "github.com/tidbcloud/ti-cli/internal/dryrun" "github.com/tidbcloud/ti-cli/internal/fs/mountcontrol" "github.com/tidbcloud/ti-cli/internal/fs/mountdriver" + "github.com/tidbcloud/ti-cli/internal/fs/mountlocator" "github.com/tidbcloud/ti-cli/internal/fs/mountstate" ) @@ -30,18 +31,28 @@ const ( ) type MountFileSystemOptions struct { - Profile *config.Profile - FileSystemName string - MountPath string - RemotePath string - Driver string - ReadOnly bool - ReadOnlySet bool - ReadyTimeout time.Duration - CacheDir string - ReadCacheMB int64 - ReadCacheFileMB int64 - ReadCacheTTL time.Duration + Profile *config.Profile + FileSystemName string + MountPath string + RemotePath string + Driver string + ReadOnly bool + ReadOnlySet bool + ReadyTimeout time.Duration + CacheDir string + ReadCacheMB int64 + ReadCacheFileMB int64 + ReadCacheTTL time.Duration + // Durability is the write-durability tier forwarded to the companion as + // `mount --durability`. Normalization always leaves an effective tier here. + Durability string + // DurabilitySet records whether --durability was explicitly supplied. + DurabilitySet bool + // DurabilityExplicit records whether either --durability or the legacy + // --write-back-cache alias was explicitly supplied. Normalization sets it. + DurabilityExplicit bool + // WriteBackCache is the legacy boolean alias of --durability: true means + // interactive, false means auto. WriteBackCache bool WriteBackCacheSet bool MountProfile string @@ -84,7 +95,7 @@ type MountResult struct { Checks []MountRuntimeCheck `json:"checks,omitempty"` Remote *MountRemoteSnapshot `json:"remote,omitempty"` CacheDir string `json:"cache_dir,omitempty"` - WriteBackCache bool `json:"write_back_cache"` + Durability string `json:"durability"` MountProfile string `json:"mount_profile,omitempty"` LocalRoot string `json:"local_root,omitempty"` PackPaths []string `json:"pack_paths,omitempty"` @@ -114,16 +125,27 @@ type UnmountResult struct { Status string `json:"status"` MountPath string `json:"mount_path"` Driver string `json:"driver,omitempty"` - PID int `json:"pid,omitempty"` StateFile string `json:"state_file,omitempty"` Checks []MountRuntimeCheck `json:"checks,omitempty"` + // Drain records the pre-unmount drain pass: drained, failed, or skipped + // with the reason. A failed drain is only reported when the caller asked to + // continue anyway (--force or --ignore-absent). + Drain *MountDrain `json:"drain,omitempty"` +} + +// MountDrain is the pre-unmount drain outcome. Response carries the engine's +// own drain report, including the pending counters, so a lost write is +// attributable instead of silent. +type MountDrain struct { + Status string `json:"status"` + Message string `json:"message,omitempty"` + Response *mountcontrol.DrainResponse `json:"response,omitempty"` } type DrainResult struct { Status string `json:"status"` MountPath string `json:"mount_path"` Driver string `json:"driver,omitempty"` - PID int `json:"pid,omitempty"` StateFile string `json:"state_file,omitempty"` ControlSocket string `json:"control_socket,omitempty"` Response *mountcontrol.DrainResponse `json:"response,omitempty"` @@ -146,10 +168,20 @@ func (s Service) MountFileSystem(ctx context.Context, opts MountFileSystemOption } // normalizeMountOptions trims the flag values the unpack predicate and the -// companion argv must agree on, then applies the layer rules. +// companion argv must agree on, resolves the durability tier, applies the layer +// rules, and refuses a tier the effective driver cannot honor. func normalizeMountOptions(opts MountFileSystemOptions) (MountFileSystemOptions, error) { opts.UnpackArchivePath = strings.TrimSpace(opts.UnpackArchivePath) - return normalizeLayerMountOptions(opts) + var err error + opts, err = normalizeMountDurability(opts) + if err != nil { + return MountFileSystemOptions{}, err + } + opts, err = normalizeLayerMountOptions(opts) + if err != nil { + return MountFileSystemOptions{}, err + } + return rejectUnsupportedMountDurability(opts) } func (s Service) DryRunMountFileSystem(ctx context.Context, commandPath string, opts MountFileSystemOptions) (dryrun.Result, error) { @@ -198,6 +230,7 @@ func (s Service) DryRunMountFileSystem(ctx context.Context, commandPath string, Path: "/v1/status", Body: map[string]any{ "driver": opts.Driver, + "durability": opts.Durability, "layer_ref": opts.LayerRef, "checkpoint_id": opts.CheckpointID, "read_only": opts.ReadOnly, @@ -229,8 +262,14 @@ func normalizeLayerMountOptions(opts MountFileSystemOptions) (MountFileSystemOpt if opts.WriteBackCacheSet && opts.WriteBackCache { return MountFileSystemOptions{}, apperr.New("fs.checkpoint_mount_read_only", "usage", 2, "checkpoint mounts are read-only; remove --write-back-cache=true") } + if opts.DurabilityExplicit && opts.Durability != MountDurabilityAuto { + return MountFileSystemOptions{}, apperr.New("fs.checkpoint_mount_read_only", "usage", 2, fmt.Sprintf("checkpoint mounts are read-only; remove --durability=%s", opts.Durability)) + } opts.ReadOnly = true opts.WriteBackCache = false + // A checkpoint view never writes, so ti forwards no tier and reports + // the engine's own default instead of the interactive mount default. + opts.Durability = MountDurabilityAuto } driverName := strings.TrimSpace(opts.Driver) if driverName == "" { @@ -297,7 +336,7 @@ func (s Service) DrainFileSystem(ctx context.Context, opts DrainFileSystemOption } func (s Service) DryRunDrainFileSystem(ctx context.Context, commandPath string, opts DrainFileSystemOptions) (dryrun.Result, error) { - profile, err := s.drive9MountLocatorProfile(opts.Profile, opts.MountPath) + locator, profile, err := s.mountLocatorProfile(opts.Profile, opts.MountPath) if err != nil { return dryrun.Result{}, err } @@ -312,11 +351,12 @@ func (s Service) DryRunDrainFileSystem(ctx context.Context, commandPath string, dryrun.Check{Name: "mount_locator", Status: "passed", Message: opts.MountPath}, dryrun.Check{Name: "file_system_id", Status: "passed", Message: profile.FSTenantID}, dryrun.Check{Name: "region", Status: "passed", Message: profile.FSPlacementRegionCode}, + drainSupportCheck(locator), ), nil } func (s Service) DryRunUnmountFileSystem(ctx context.Context, commandPath string, opts UnmountFileSystemOptions) (dryrun.Result, error) { - profile, err := s.drive9MountLocatorProfile(opts.Profile, opts.MountPath) + locator, profile, err := s.mountLocatorProfile(opts.Profile, opts.MountPath) if err != nil { if opts.IgnoreAbsent && apperr.CodeFor(err) == "fs.mount_locator_not_found" { return dryrun.New( @@ -339,9 +379,29 @@ func (s Service) DryRunUnmountFileSystem(ctx context.Context, commandPath string dryrun.Check{Name: "mount_locator", Status: "passed", Message: opts.MountPath}, dryrun.Check{Name: "file_system_id", Status: "passed", Message: profile.FSTenantID}, dryrun.Check{Name: "region", Status: "passed", Message: profile.FSPlacementRegionCode}, + dryrun.Check{Name: "pre_unmount_drain", Status: "passed", Message: describeDryRunDrainGate(locator, opts)}, ), nil } +// drainSupportCheck predicts whether `ti fs drain-file-system` will run for +// this mount. It uses the drain command's own rule, so a dry run cannot claim a +// drain that normal execution deterministically refuses. +func drainSupportCheck(locator mountlocator.Locator) dryrun.Check { + if drainable, reason := drainApplicability(locator); !drainable { + return dryrun.Check{Name: "drain_support", Status: "failed", Message: reason} + } + return dryrun.Check{Name: "drain_support", Status: "passed", Message: "file-system mount; the drain command asks the companion"} +} + +// describeDryRunDrainGate reports what the pre-unmount drain would do without +// running it. +func describeDryRunDrainGate(locator mountlocator.Locator, opts UnmountFileSystemOptions) string { + if drainable, reason := mountDrainGate(locator, opts.Force); !drainable { + return "skipped: " + reason + } + return "drain pending writes before detaching the mount" +} + func (s Service) readDrainMountState(mountPathInput string) (mountstate.State, string, string, []MountRuntimeCheck, error) { homeDir, err := s.homeDir() if err != nil { @@ -389,6 +449,7 @@ type mountInputs struct { readCacheFileBytes int64 readCacheTTL time.Duration writeBackCache bool + durability string mountProfile string localRoot string packPaths []string @@ -493,7 +554,8 @@ func (s Service) mountInputs(opts MountFileSystemOptions) (mountInputs, error) { readCacheBytes: readCacheBytes, readCacheFileBytes: readCacheFileBytes, readCacheTTL: readCacheTTL, - writeBackCache: opts.WriteBackCache, + writeBackCache: mountDurabilityUsesWriteBack(opts.Durability), + durability: opts.Durability, mountProfile: mountProfile, localRoot: localRoot, packPaths: packPaths, @@ -572,7 +634,7 @@ func mountResult(status string, inputs mountInputs, remote apifs.StatusResponse, Checks: checks, Remote: &snapshot, CacheDir: inputs.cacheDir, - WriteBackCache: inputs.writeBackCache, + Durability: inputs.durability, MountProfile: inputs.mountProfile, LocalRoot: inputs.localRoot, PackPaths: append([]string(nil), inputs.packPaths...), @@ -718,6 +780,9 @@ func (r MountResult) Human() string { "Remote path: " + r.RemotePath, "Driver: " + r.Driver, } + if r.Durability != "" { + lines = append(lines, "Durability: "+r.Durability) + } if r.LayerRef != "" { lines = append(lines, "Layer: "+r.LayerRef) } @@ -745,36 +810,36 @@ func (r MountResult) Human() string { func (r UnmountResult) Human() string { var out strings.Builder writer := tabwriter.NewWriter(&out, 0, 0, 2, ' ', 0) - _, _ = fmt.Fprintln(writer, "STATUS\tMOUNT_PATH\tDRIVER\tPID") - _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%d\n", r.Status, r.MountPath, r.Driver, r.PID) + _, _ = fmt.Fprintln(writer, "STATUS\tMOUNT_PATH\tDRIVER\tDRAIN") + drain := "" + if r.Drain != nil { + drain = r.Drain.Status + } + _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%s\n", r.Status, r.MountPath, r.Driver, drain) _ = writer.Flush() + if r.Drain != nil { + if r.Drain.Message != "" { + _, _ = fmt.Fprintf(&out, "drain: %s\n", r.Drain.Message) + } + if r.Drain.Response != nil { + _, _ = fmt.Fprintf(&out, "pending: %s\n", drainPendingSummary(r.Drain.Response.Pending)) + } + } return strings.TrimRight(out.String(), "\n") } func (r DrainResult) Human() string { var out strings.Builder writer := tabwriter.NewWriter(&out, 0, 0, 2, ' ', 0) - _, _ = fmt.Fprintln(writer, "STATUS\tMOUNT_PATH\tDRIVER\tPID\tDURATION_MS") + _, _ = fmt.Fprintln(writer, "STATUS\tMOUNT_PATH\tDRIVER\tDURATION_MS") duration := int64(0) if r.Response != nil { duration = r.Response.DurationMS } - _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%d\t%d\n", r.Status, r.MountPath, r.Driver, r.PID, duration) + _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%d\n", r.Status, r.MountPath, r.Driver, duration) _ = writer.Flush() if r.Response != nil { - p := r.Response.Pending - _, _ = fmt.Fprintf(&out, "pending: open_handles=%d dirty_handles=%d commit_queue=%d commit_bytes=%d commit_in_flight=%d commit_delayed=%d commit_conflicts=%d uploader_queued=%d uploader_in_flight=%d uploader_cached=%d\n", - p.OpenHandles, - p.DirtyHandles, - p.CommitQueuePending, - p.CommitQueueBytes, - p.CommitQueueInFlight, - p.CommitQueueDelayed, - p.CommitQueueConflicts, - p.UploaderQueued, - p.UploaderInFlight, - p.UploaderCached, - ) + _, _ = fmt.Fprintf(&out, "pending: %s\n", drainPendingSummary(r.Response.Pending)) } return strings.TrimRight(out.String(), "\n") } diff --git a/internal/fs/mount_durability.go b/internal/fs/mount_durability.go new file mode 100644 index 00000000..0bc7f3e5 --- /dev/null +++ b/internal/fs/mount_durability.go @@ -0,0 +1,158 @@ +package fs + +import ( + "fmt" + "runtime" + "strings" + + "github.com/tidbcloud/ti-cli/internal/apperr" +) + +// Write-durability tiers accepted by the ti-drive9 companion's +// `mount --durability`. Each tier selects the engine's sync mode and write +// policy; ti forwards one explicitly instead of letting the companion pick its +// own default, because the engine default is auto and ti has always mounted +// with interactive semantics. +const ( + // MountDurabilityAuto lets the engine measure round-trip time at mount + // time and choose interactive or strict accordingly. It is the engine + // default and the only tier a WebDAV mount can be described with. + MountDurabilityAuto = "auto" + // MountDurabilityInteractive keeps writes durable in the local staging + // journal and uploads them asynchronously after flush. + MountDurabilityInteractive = "interactive" + // MountDurabilityFsync makes fsync(2) block until the remote commit + // succeeds while ordinary writes still stage locally. + MountDurabilityFsync = "fsync" + // MountDurabilityCloseSync makes close(2) remote-durable by uploading from + // Flush, so a completed close means the data left the client. + MountDurabilityCloseSync = "close-sync" + // MountDurabilityWriteSync makes every write remote-durable before it + // returns success. It is the strictest and most expensive tier. + MountDurabilityWriteSync = "write-sync" +) + +// DefaultMountDurability is the tier an ordinary ti mount uses. It preserves +// the behavior ti had before the tier was selectable. +const DefaultMountDurability = MountDurabilityInteractive + +// mountDurabilityValues lists the accepted tiers in increasing strictness for +// help text and error messages. +var mountDurabilityValues = []string{ + MountDurabilityAuto, + MountDurabilityInteractive, + MountDurabilityFsync, + MountDurabilityCloseSync, + MountDurabilityWriteSync, +} + +// MountDurabilityHelp renders the accepted tiers for flag and error text. +func MountDurabilityHelp() string { + return strings.Join(mountDurabilityValues, ", ") +} + +func validMountDurability(value string) bool { + for _, candidate := range mountDurabilityValues { + if value == candidate { + return true + } + } + return false +} + +// mountDurabilityUsesWriteBack reports whether a tier stages writes locally and +// leaves the remote upload to the background (drive9's writeback policy). The +// close-sync and write-sync tiers make the close or the write itself +// remote-durable instead. +func mountDurabilityUsesWriteBack(value string) bool { + switch value { + case MountDurabilityAuto, MountDurabilityInteractive, MountDurabilityFsync: + return true + default: + return false + } +} + +// normalizeMountDurability resolves --durability together with its legacy +// --write-back-cache alias into the single tier ti forwards. +func normalizeMountDurability(opts MountFileSystemOptions) (MountFileSystemOptions, error) { + value := strings.ToLower(strings.TrimSpace(opts.Durability)) + if value == "" { + // An explicit empty value is a mistake, not a request for the default: + // the rest of the CLI rejects explicit empty --region/--profile values + // for the same reason, and treating "" as both "unset" and "explicit" + // makes --durability "" --write-back-cache=false look like a conflict. + if opts.DurabilitySet { + return MountFileSystemOptions{}, apperr.New("fs.missing_mount_durability", "usage", 2, fmt.Sprintf("--durability must not be empty; supported values: %s", MountDurabilityHelp())) + } + value = DefaultMountDurability + } + if !validMountDurability(value) { + return MountFileSystemOptions{}, apperr.New("fs.invalid_mount_durability", "usage", 2, fmt.Sprintf("unsupported --durability value %q; supported values: %s", strings.TrimSpace(opts.Durability), MountDurabilityHelp())) + } + explicit := opts.DurabilitySet + if opts.WriteBackCacheSet { + legacy := MountDurabilityInteractive + if !opts.WriteBackCache { + legacy = MountDurabilityAuto + } + if opts.DurabilitySet && value != legacy { + return MountFileSystemOptions{}, apperr.New("fs.conflicting_mount_durability", "usage", 2, fmt.Sprintf("--durability=%s conflicts with --write-back-cache=%t; pass only --durability (--write-back-cache=true means interactive and --write-back-cache=false means auto)", value, opts.WriteBackCache)) + } + value = legacy + explicit = true + } + opts.Durability = value + opts.DurabilityExplicit = explicit + return opts, nil +} + +// rejectUnsupportedMountDurability refuses a tier the effective mount driver +// cannot honor. WebDAV mounts keep the engine's native write behavior, so ti +// forwards no tier for them and reports auto. An explicit strict tier is a +// usage error instead of a silent downgrade, so a caller that asked for +// close-sync or write-sync never believes it got it. +func rejectUnsupportedMountDurability(opts MountFileSystemOptions) (MountFileSystemOptions, error) { + if companionMountMode(opts.Driver) != "webdav" { + return opts, nil + } + if opts.DurabilityExplicit && opts.Durability != MountDurabilityAuto { + return MountFileSystemOptions{}, apperr.New("fs.webdav_durability_unsupported", "usage", 2, fmt.Sprintf("--durability=%s is not supported for WebDAV mounts, which always keep their native write behavior; remove --durability or use --driver fuse where FUSE is available", opts.Durability)) + } + opts.Durability = MountDurabilityAuto + return opts, nil +} + +// companionMountMode mirrors the companion's own `--mode auto` resolution +// (WebDAV on macOS and Windows, FUSE everywhere else) so ti's durability +// decision matches the mode the companion will actually mount with. +// +// It deliberately does not use mountdriver.Resolve: that resolver is +// prerequisite-aware and prefers FUSE whenever it is installable (macFUSE on +// darwin), while the companion's `--mode auto` is GOOS-only. Forwarding a tier +// based on ti's optimistic resolution would hand a durability flag to a WebDAV +// mount the companion chose, which the companion rejects. This function is the +// companion's rule, kept in one place and pinned by +// TestCompanionMountModeMatchesCompanionAutoResolution; the mount result +// reconciles the reported tier with the driver the companion reports. +func companionMountMode(driver string) string { + switch strings.TrimSpace(driver) { + case "", "auto": + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return "webdav" + } + return "fuse" + case "webdav": + return "webdav" + default: + return "fuse" + } +} + +// mountDurabilityForwarded reports whether ti pins the tier on the companion +// argv. It pins FUSE mounts, including automatic ones that resolve to FUSE, so +// an ordinary mount keeps the documented interactive default instead of the +// engine's auto default. Checkpoint mounts are read-only and forward nothing. +func mountDurabilityForwarded(opts MountFileSystemOptions) bool { + return opts.CheckpointID == "" && companionMountMode(opts.Driver) == "fuse" +} diff --git a/internal/fs/mount_durability_test.go b/internal/fs/mount_durability_test.go new file mode 100644 index 00000000..1fe6644a --- /dev/null +++ b/internal/fs/mount_durability_test.go @@ -0,0 +1,301 @@ +package fs + +import ( + "context" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/tidbcloud/ti-cli/internal/apperr" +) + +// TestNormalizeMountDurability pins the tier contract: --durability keeps the +// interactive default, the legacy boolean alias maps onto two tiers, and an +// explicit but unusable tier is refused instead of silently downgraded. +func TestNormalizeMountDurability(t *testing.T) { + for _, tc := range []struct { + name string + opts MountFileSystemOptions + want string + wantCode string + forwarded bool + writeBack bool + explicitly bool + }{ + { + name: "default keeps interactive", + opts: MountFileSystemOptions{Driver: "fuse"}, + want: MountDurabilityInteractive, + forwarded: true, + writeBack: true, + explicitly: false, + }, + { + name: "auto resolves to the engine default", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityAuto, DurabilitySet: true}, + want: MountDurabilityAuto, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "fsync stays a write-back tier", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityFsync, DurabilitySet: true}, + want: MountDurabilityFsync, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "close-sync leaves the write-back policy", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityCloseSync, DurabilitySet: true}, + want: MountDurabilityCloseSync, + forwarded: true, + writeBack: false, + explicitly: true, + }, + { + name: "write-sync leaves the write-back policy", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityWriteSync, DurabilitySet: true}, + want: MountDurabilityWriteSync, + forwarded: true, + writeBack: false, + explicitly: true, + }, + { + name: "value is trimmed and case-folded", + opts: MountFileSystemOptions{Driver: "fuse", Durability: " Close-Sync ", DurabilitySet: true}, + want: MountDurabilityCloseSync, + forwarded: true, + writeBack: false, + explicitly: true, + }, + { + name: "legacy true means interactive", + opts: MountFileSystemOptions{Driver: "fuse", WriteBackCache: true, WriteBackCacheSet: true}, + want: MountDurabilityInteractive, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "legacy false means auto", + opts: MountFileSystemOptions{Driver: "fuse", WriteBackCache: false, WriteBackCacheSet: true}, + want: MountDurabilityAuto, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "legacy alias may agree with the tier", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityInteractive, DurabilitySet: true, WriteBackCache: true, WriteBackCacheSet: true}, + want: MountDurabilityInteractive, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "explicit empty value is refused", + opts: MountFileSystemOptions{Driver: "fuse", Durability: " ", DurabilitySet: true}, + wantCode: "fs.missing_mount_durability", + }, + { + name: "unknown tier is refused", + opts: MountFileSystemOptions{Driver: "fuse", Durability: "turbo", DurabilitySet: true}, + wantCode: "fs.invalid_mount_durability", + }, + { + name: "conflicting alias is refused", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityAuto, DurabilitySet: true, WriteBackCache: true, WriteBackCacheSet: true}, + wantCode: "fs.conflicting_mount_durability", + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeMountOptions(tc.opts) + if tc.wantCode != "" { + if apperr.CodeFor(err) != tc.wantCode { + t.Fatalf("error = %v, want %s", err, tc.wantCode) + } + return + } + if err != nil { + t.Fatalf("normalizeMountOptions failed: %v", err) + } + if got.Durability != tc.want { + t.Fatalf("durability = %q, want %q", got.Durability, tc.want) + } + if mountDurabilityForwarded(got) != tc.forwarded { + t.Fatalf("forwarded = %t, want %t", mountDurabilityForwarded(got), tc.forwarded) + } + if mountDurabilityUsesWriteBack(got.Durability) != tc.writeBack { + t.Fatalf("write-back policy = %t, want %t", mountDurabilityUsesWriteBack(got.Durability), tc.writeBack) + } + if got.DurabilityExplicit != tc.explicitly { + t.Fatalf("explicit = %t, want %t", got.DurabilityExplicit, tc.explicitly) + } + }) + } +} + +// TestNormalizeMountDurabilityWebDAV keeps the WebDAV contract honest: the +// engine has no tiers there, so ti forwards none and refuses an explicit tier +// that would otherwise look honored. +func TestNormalizeMountDurabilityWebDAV(t *testing.T) { + got, err := normalizeMountOptions(MountFileSystemOptions{Driver: "webdav"}) + if err != nil { + t.Fatalf("default webdav mount failed: %v", err) + } + if got.Durability != MountDurabilityAuto { + t.Fatalf("webdav durability = %q, want %s", got.Durability, MountDurabilityAuto) + } + if mountDurabilityForwarded(got) { + t.Fatal("webdav mount must not forward --durability") + } + if _, err := normalizeMountOptions(MountFileSystemOptions{Driver: "webdav", Durability: MountDurabilityAuto, DurabilitySet: true}); err != nil { + t.Fatalf("explicit auto webdav mount failed: %v", err) + } + for _, tier := range []string{MountDurabilityInteractive, MountDurabilityFsync, MountDurabilityCloseSync, MountDurabilityWriteSync} { + if _, err := normalizeMountOptions(MountFileSystemOptions{Driver: "webdav", Durability: tier, DurabilitySet: true}); apperr.CodeFor(err) != "fs.webdav_durability_unsupported" { + t.Fatalf("webdav %s error = %v, want fs.webdav_durability_unsupported", tier, err) + } + } + if _, err := normalizeMountOptions(MountFileSystemOptions{Driver: "webdav", WriteBackCache: true, WriteBackCacheSet: true}); apperr.CodeFor(err) != "fs.webdav_durability_unsupported" { + t.Fatalf("webdav write-back-cache error = %v, want fs.webdav_durability_unsupported", err) + } +} + +// TestNormalizeMountDurabilityCheckpoint keeps read-only checkpoint mounts on +// the engine default and refuses a tier that cannot take effect there. +func TestNormalizeMountDurabilityCheckpoint(t *testing.T) { + base := MountFileSystemOptions{Driver: "fuse", LayerRef: "layer", CheckpointID: "cp-1"} + got, err := normalizeMountOptions(base) + if err != nil { + t.Fatalf("default checkpoint mount failed: %v", err) + } + if !got.ReadOnly { + t.Fatal("checkpoint mount must be read-only") + } + if got.Durability != MountDurabilityAuto { + t.Fatalf("checkpoint durability = %q, want %s", got.Durability, MountDurabilityAuto) + } + if mountDurabilityForwarded(got) { + t.Fatal("checkpoint mount must not forward --durability") + } + + explicitAuto := base + explicitAuto.Durability = MountDurabilityAuto + explicitAuto.DurabilitySet = true + if _, err := normalizeMountOptions(explicitAuto); err != nil { + t.Fatalf("explicit auto checkpoint mount failed: %v", err) + } + explicitLegacyOff := base + explicitLegacyOff.WriteBackCache = false + explicitLegacyOff.WriteBackCacheSet = true + if _, err := normalizeMountOptions(explicitLegacyOff); err != nil { + t.Fatalf("explicit write-back-cache=false checkpoint mount failed: %v", err) + } + + for _, tier := range []string{MountDurabilityInteractive, MountDurabilityFsync, MountDurabilityCloseSync, MountDurabilityWriteSync} { + explicit := base + explicit.Durability = tier + explicit.DurabilitySet = true + if _, err := normalizeMountOptions(explicit); apperr.CodeFor(err) != "fs.checkpoint_mount_read_only" { + t.Fatalf("checkpoint %s error = %v, want fs.checkpoint_mount_read_only", tier, err) + } + } +} + +// TestCompanionMountModeMatchesCompanionAutoResolution pins the local mirror of +// the companion's --mode auto rule, which decides whether a tier is forwarded. +func TestCompanionMountModeMatchesCompanionAutoResolution(t *testing.T) { + wantAuto := "fuse" + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + wantAuto = "webdav" + } + for _, tc := range []struct{ driver, want string }{ + {driver: "", want: wantAuto}, + {driver: "auto", want: wantAuto}, + {driver: " fuse ", want: "fuse"}, + {driver: "webdav", want: "webdav"}, + } { + if got := companionMountMode(tc.driver); got != tc.want { + t.Fatalf("companionMountMode(%q) = %q, want %q", tc.driver, got, tc.want) + } + } +} + +// TestDrive9MountForwardsDurability proves the selected tier reaches the +// companion argv instead of being collapsed into a boolean. +func TestDrive9MountForwardsDurability(t *testing.T) { + for _, tier := range []string{MountDurabilityAuto, MountDurabilityInteractive, MountDurabilityFsync, MountDurabilityCloseSync, MountDurabilityWriteSync} { + t.Run(tier, func(t *testing.T) { + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(t.TempDir(), companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + result, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + Durability: tier, DurabilitySet: true, + }) + if err != nil { + t.Fatalf("mount failed: %v", err) + } + if result.Durability != tier { + t.Fatalf("result durability = %q, want %q", result.Durability, tier) + } + call := requireFakeDrive9Call(t, recordPath, "mount") + want := []string{"--durability", tier} + for i := 0; i+1 < len(call.Args); i++ { + if call.Args[i] == want[0] && call.Args[i+1] == want[1] { + return + } + } + t.Fatalf("mount args %#v missing %#v", call.Args, want) + }) + } +} + +// TestDrive9MountDefaultKeepsInteractiveDurability pins the default so an +// ordinary mount never silently follows the engine's auto default. +func TestDrive9MountDefaultKeepsInteractiveDurability(t *testing.T) { + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(t.TempDir(), companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + result, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }) + if err != nil { + t.Fatalf("mount failed: %v", err) + } + if result.Durability != MountDurabilityInteractive { + t.Fatalf("result durability = %q, want %q", result.Durability, MountDurabilityInteractive) + } + call := requireFakeDrive9Call(t, recordPath, "mount") + joined := strings.Join(call.Args, " ") + if !strings.Contains(joined, "--durability "+MountDurabilityInteractive) { + t.Fatalf("mount args %#v missing the interactive default", call.Args) + } +} + +// TestDryRunMountDurability keeps --dry-run aligned with the live path. +func TestDryRunMountDurability(t *testing.T) { + service := Service{} + profile := dataProfile() + opts := MountFileSystemOptions{Profile: profile, MountPath: filepath.Join(t.TempDir(), "mnt"), Driver: "fuse", Durability: MountDurabilityCloseSync, DurabilitySet: true} + result, err := service.DryRunMountFileSystem(context.Background(), "ti fs mount-file-system", opts) + if err != nil { + t.Fatalf("dry run failed: %v", err) + } + if body, ok := result.Request.Body.(map[string]any); !ok || body["durability"] != MountDurabilityCloseSync { + t.Fatalf("dry-run body = %#v", result.Request.Body) + } + opts.Durability = MountDurabilityWriteSync + opts.Driver = "webdav" + if _, err := service.DryRunMountFileSystem(context.Background(), "ti fs mount-file-system", opts); apperr.CodeFor(err) != "fs.webdav_durability_unsupported" { + t.Fatalf("dry-run webdav error = %v, want fs.webdav_durability_unsupported", err) + } +} diff --git a/internal/fs/mountcontrol/control.go b/internal/fs/mountcontrol/control.go index 532ad723..37f35b1a 100644 --- a/internal/fs/mountcontrol/control.go +++ b/internal/fs/mountcontrol/control.go @@ -52,6 +52,31 @@ type DrainPending struct { UploaderCachedBytes int64 `json:"uploader_cached_bytes"` } +// HasPendingWork reports whether the snapshot lists work the engine has not made +// durable remotely. +// +// The engine's own drain-failure exit record counts exactly +// dirty_handles + commit_queue_pending + uploader_queued as pending files and +// commit_queue_bytes + uploader_cached_bytes as pending bytes, so those counters +// are undrained work by construction. The remaining counters are informational, +// but they are non-zero only while the mount still has unfinished work, so ti +// treats any non-zero counter as pending: over-blocking an unmount is safe, +// silently detaching from queued write-back is not. Keep this conservative until +// the engine publishes a loss-carrying/informational split. +func (p DrainPending) HasPendingWork() bool { + return p.OpenHandles != 0 || + p.DirtyHandles != 0 || + p.CommitQueuePending != 0 || + p.CommitQueueBytes != 0 || + p.CommitQueueInFlight != 0 || + p.CommitQueueDelayed != 0 || + p.CommitQueueConflicts != 0 || + p.UploaderQueued != 0 || + p.UploaderInFlight != 0 || + p.UploaderCached != 0 || + p.UploaderCachedBytes != 0 +} + type DrainPhase struct { Name string `json:"name"` DurationMS int64 `json:"duration_ms"` diff --git a/internal/fs/mountlocator/locator.go b/internal/fs/mountlocator/locator.go index 6fe529d0..40ff3c88 100644 --- a/internal/fs/mountlocator/locator.go +++ b/internal/fs/mountlocator/locator.go @@ -21,10 +21,15 @@ type Locator struct { // was created. It is a stable, non-secret placement property that lets drain // and unmount rebuild the companion environment without fetching the region // catalog. Older locators may omit it. - ServiceAddress string `json:"service_address,omitempty"` - CompanionHome string `json:"companion_home"` - MountPath string `json:"mount_path"` - Kind string `json:"kind,omitempty"` + ServiceAddress string `json:"service_address,omitempty"` + CompanionHome string `json:"companion_home"` + MountPath string `json:"mount_path"` + Kind string `json:"kind,omitempty"` + // Driver is the mount driver the companion reported when the mount was + // created. Unmount uses it to decide whether the mount supports drain: + // WebDAV mounts have no control socket and no durability tiers. Older + // locators may omit it, and an omitted driver means "unknown, do not gate". + Driver string `json:"driver,omitempty"` FileSystemID string `json:"file_system_id,omitempty"` TokenID string `json:"token_id,omitempty"` TokenFingerprint string `json:"token_fingerprint,omitempty"` @@ -67,6 +72,12 @@ func New(profile, fileSystemName, regionCode, companionHome, mountPath, kind str }, nil } +// WithDriver records the mount driver reported by the companion. +func (l Locator) WithDriver(driver string) Locator { + l.Driver = strings.TrimSpace(driver) + return l +} + // WithServiceAddress records the resolved regional service address. func (l Locator) WithServiceAddress(address string) Locator { l.ServiceAddress = strings.TrimSpace(address) diff --git a/internal/fs/vault.go b/internal/fs/vault.go index 787b6ffe..4088d8ee 100644 --- a/internal/fs/vault.go +++ b/internal/fs/vault.go @@ -274,7 +274,9 @@ func vaultMountResult(status string, inputs vaultMountInputs, checks []MountRunt LogFile: logFile, Endpoint: nil, Checks: checks, - WriteBackCache: false, + // A vault mount forwards no write-durability tier, so it runs with the + // engine default. + Durability: MountDurabilityAuto, } }