From 76aad638037ccf8a0c2d2dbd17a5175c37b7c406 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 19 Nov 2026 04:08:50 +0800 Subject: [PATCH 1/7] feat(fs): pass mount write durability through to the companion `ti fs mount-file-system` only exposed the boolean `--write-back-cache`, and only an explicit `--driver fuse` mount forwarded anything: true pinned the companion to `interactive` and false forwarded nothing, so the engine fell back to `auto`. The companion has long supported five tiers (`mount --durability auto|interactive|fsync|close-sync|write-sync`), so a caller that needed close-sync or write-sync had to mount around ti and drain by hand, leaving the client-side loss window unbounded. Add `--durability` and forward the tier to the companion for FUSE mounts. The default stays `interactive`, so an ordinary mount keeps its documented behavior instead of following the engine's `auto` default, including on `--driver auto` mounts that resolve to FUSE. - Keep `--write-back-cache` as a legacy alias (true = interactive, false = auto) and refuse an inconsistent explicit combination. - WebDAV mounts have no tiers: forward none, report `auto`, and reject every explicit tier except `auto` instead of silently downgrading. - Checkpoint mounts are read-only: force `auto` and reject any other explicit tier. - Replace the `write_back_cache` result boolean with the effective `durability` tier, because a boolean cannot express five tiers and a derived boolean invites disagreeing with the mount. - Report the same tier in `--dry-run`. Spec: docs/spec/done/0038-fs-mount-durability.md --- AGENTS.md | 15 ++ README.md | 8 + docs/spec/done/0038-fs-mount-durability.md | 117 ++++++++ e2e/cli_test.go | 72 ++++- e2e/live_test.go | 5 +- internal/cli/commands.go | 9 +- internal/fs/drive9_companion.go | 2 +- internal/fs/drive9_mount_audit.go | 8 +- internal/fs/mount.go | 63 +++-- internal/fs/mount_durability.go | 144 ++++++++++ internal/fs/mount_durability_test.go | 296 +++++++++++++++++++++ internal/fs/vault.go | 4 +- 12 files changed, 717 insertions(+), 26 deletions(-) create mode 100644 docs/spec/done/0038-fs-mount-durability.md create mode 100644 internal/fs/mount_durability.go create mode 100644 internal/fs/mount_durability_test.go diff --git a/AGENTS.md b/AGENTS.md index 9f5d70be..fb5403e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -654,6 +654,7 @@ Implemented command behavior: - `ti fs mount-file-system --file-system-id --mount-path ./workspace --mount-profile portable --pack-path /` - `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --read-cache-size-mb 256 --read-cache-max-file-mb 16` - `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --cache-dir ~/.ti/cache/workspace --write-back-cache=false` +- `ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --durability close-sync` - `ti fs mount-file-system --file-system-id --mount-path ./workspace --remote-path /workspace --driver fuse --layer-ref layer-1` - `ti fs mount-file-system --file-system-id --mount-path ./workspace-v1 --remote-path /workspace --driver fuse --layer-ref layer-1 --checkpoint-id cp-1` - `ti fs drain-file-system --mount-path ./workspace` @@ -1149,6 +1150,20 @@ Background FS and vault mounts write only a non-secret locator under original resource-scoped companion HOME without requiring the FS token again. Successful unmount removes the locator; failed unmount preserves it for retry. +`ti fs mount-file-system` forwards the selected write-durability tier to the +companion as `mount --durability`. The flag accepts `auto`, `interactive`, +`fsync`, `close-sync`, and `write-sync`, and defaults to `interactive` so an +ordinary mount keeps its documented behavior instead of following the engine's +`auto` default. Never collapse the tier back into the legacy +`--write-back-cache` boolean: keep that flag only as an alias (`true` means +`interactive`, `false` means `auto`), and refuse an inconsistent explicit +combination. FUSE mounts forward the tier, including `--driver auto` mounts on +platforms whose automatic choice is FUSE; WebDAV mounts forward none and reject +every explicit tier except `auto` with `fs.webdav_durability_unsupported`; +checkpoint mounts are read-only, force `auto`, and reject any other explicit +tier. Report the effective tier as `durability` in the mount result and do not +reintroduce the removed `write_back_cache` boolean. + Do not implement or expose ti commands for Drive9 internal APIs that are not part of Drive9's public CLI. In particular, do not reintroduce low-level layer entry/object/event commands, low-level Git workspace/tree/state/object-pack/ diff --git a/README.md b/README.md index 63fd9a53..6fd8fad0 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,14 @@ Automatic mounting uses FUSE on Linux and WebDAV on macOS and Windows. macOS use Mount commands start the companion runtime in the background, wait until the mount is ready, and then return a structured result. Use `ti fs unmount-file-system` or `ti fs-vault unmount-vault` to end a mount. The public CLI does not expose a foreground mount mode. +FUSE mounts choose their write durability with `--durability`: `auto` lets the companion measure round-trip time, `interactive` (the default) keeps writes durable in the local staging journal and uploads them after flush, `fsync` makes `fsync(2)` wait for the remote commit, `close-sync` makes `close(2)` remote-durable, and `write-sync` makes every write remote-durable before it returns. Use `close-sync` or `write-sync` when a caller must not lose an acknowledged write in the client; the interactive default leaves that window to the background uploader. The mount result reports the effective tier as `durability`. WebDAV mounts keep the engine's native write behavior: ti forwards no tier, reports `auto`, and rejects every other explicit tier. `--write-back-cache` remains as a legacy alias where `true` means `interactive` and `false` means `auto`. Checkpoint mounts are read-only, reject any tier other than `auto`, and report `auto`. + +```shell +ti fs mount-file-system --file-system-id --mount-path ~/my-workspace --driver fuse --durability close-sync +ti fs drain-file-system --mount-path ~/my-workspace +ti fs unmount-file-system --mount-path ~/my-workspace +``` + Run a command against a remote Filesystem (Runtime is available only in AWS `us-east-1` and `us-west-2`): ```shell diff --git a/docs/spec/done/0038-fs-mount-durability.md b/docs/spec/done/0038-fs-mount-durability.md new file mode 100644 index 00000000..f25c86fc --- /dev/null +++ b/docs/spec/done/0038-fs-mount-durability.md @@ -0,0 +1,117 @@ +# FUSE Mount Write Durability Selection + +## Goal + +Let a `ti fs mount-file-system` caller pick the write-durability tier the +ti-drive9 companion applies to a FUSE mount: + +```bash +ti fs mount-file-system --file-system-id --mount-path ./workspace +ti fs mount-file-system --file-system-id --mount-path ./workspace --durability close-sync +ti fs mount-file-system --file-system-id --mount-path ./workspace --driver fuse --durability write-sync +``` + +Before this spec, ti exposed only the boolean `--write-back-cache`, and only an +explicit `--driver fuse` mount forwarded anything: `true` (the default) pinned +the companion to `interactive`, and `false` forwarded nothing, so the companion +fell back to its own `auto` default. The companion has supported five tiers for +longer than ti has exposed them (`mount --durability auto|interactive|fsync| +close-sync|write-sync`), so a platform customer that needed `close-sync` or +`write-sync` had to mount around ti and drain by hand, which leaves the +client-side loss window unbounded and unobservable. + +## User Outcome + +A caller can state the durability it needs and get it, or get a clear error: + +- `interactive` (default) keeps the previous behavior: writes are durable in the + local staging journal and the remote upload runs in the background after + flush. +- `close-sync` makes `close(2)` remote-durable, so a completed close means the + data left the client. +- `write-sync` makes every `write(2)` remote-durable before it returns. +- `fsync` makes `fsync(2)` wait for the remote commit while ordinary writes keep + the write-back policy. +- `auto` leaves the choice to the companion's round-trip measurement. + +The mount result reports the effective tier, so a script can verify what it got +instead of assuming: + +```json +{ "status": "mounted", "driver": "fuse", "durability": "close-sync" } +``` + +## Command Contract + +`ti fs mount-file-system` (and its `ti fs mount` alias) gains: + +- `--durability `, default `interactive`, one of `auto`, `interactive`, + `fsync`, `close-sync`, `write-sync`. The value is trimmed and case-folded + before validation; anything else fails with `fs.invalid_mount_durability` and + exit code 2 before the companion runs. + +`--write-back-cache` stays as a legacy alias so existing scripts keep working: + +- `--write-back-cache=true` means `--durability=interactive`. +- `--write-back-cache=false` means `--durability=auto`. +- Setting both explicitly to inconsistent values fails with + `fs.conflicting_mount_durability` instead of silently preferring one. + +### Forwarding rules + +ti forwards the tier as `mount --durability ` only when the mount resolves +to FUSE, mirroring the companion's own `--mode auto` rule (WebDAV on macOS and +Windows, FUSE elsewhere): + +- Explicit `--driver fuse`, and `--driver auto` on a platform whose automatic + choice is FUSE, forward the tier. An ordinary mount therefore keeps the + `interactive` default instead of following the companion's `auto` default. +- `--driver webdav` forwards no tier. WebDAV mounts keep the engine's native + write behavior, so an explicit tier other than `auto` fails with + `fs.webdav_durability_unsupported` and exit code 2. Reporting a tier that + cannot take effect would be worse than refusing it. +- Checkpoint mounts (`--layer-ref` plus `--checkpoint-id`) are read-only: they + force `auto`, forward nothing, and reject any other explicit tier with + `fs.checkpoint_mount_read_only`. `--write-back-cache=true` keeps its existing + rejection there. + +The result reports the effective tier as `durability` (`auto` for WebDAV and +checkpoint mounts). It replaces the previous `write_back_cache` boolean: a +boolean cannot express five tiers, and keeping a derived boolean alongside the +tier invites it to disagree with the mount. The removed field was never +documented as a stable contract. + +`--dry-run` reports the same effective tier in its request body and applies the +same validations, including the WebDAV rejection. + +## Scope + +In scope: + +- `internal/cli` flag registration and option wiring. +- `internal/fs` tier normalization, legacy alias reconciliation, FUSE/WebDAV + and checkpoint rules, companion argv construction, mount result, and dry-run. +- README, AGENTS.md, e2e (fake companion argv + failures), and live e2e mount + coverage. + +Out of scope: + +- `ti fs-vault mount-vault`, which forwards no tier and reports `auto`. +- Any ti-native mount implementation. Mounting still belongs to the companion. +- Drive9-side behavior: ti only selects an existing engine tier. + +## Verification + +- Unit tests pin the tier table, the legacy alias mapping, the conflict error, + the WebDAV and checkpoint rejections, the local mirror of the companion's + `--mode auto` rule, the forwarded argv for every tier, the interactive + default, and the dry-run body. +- `make e2e` covers the default and a strict tier end to end through the fake + companion, plus the invalid-value and WebDAV rejections. +- `make live-e2e-fs` and `make live-e2e-fs-git` mount with an explicit tier + (`close-sync`) and assert the reported value, and the live WebDAV mount + asserts `auto`. + +The English PingCAP documentation pages under `docs/pingcap-docs/docs/` are a +submodule that was not checked out while this spec was implemented. The mount +page must gain the `--durability` section when that submodule is available. diff --git a/e2e/cli_test.go b/e2e/cli_test.go index 19761bf7..3b04e137 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -1227,6 +1227,45 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { "--mount-path", mountPath, ) mount.wantExitCode(0) + mount.wantStdoutContains(`"durability": "interactive"`) + strictMountPath := filepath.Join(home, "mount-close-sync") + strictMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", strictMountPath, + "--durability", "close-sync", + ) + strictMount.wantExitCode(0) + strictMount.wantStdoutContains(`"durability": "close-sync"`) + invalidDurability := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", filepath.Join(home, "mount-invalid"), + "--durability", "turbo", + ) + invalidDurability.wantExitCode(2) + invalidDurability.wantStderrContains("auto, interactive, fsync, close-sync, write-sync") + webdavDurability := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", filepath.Join(home, "mount-webdav-strict"), + "--driver", "webdav", + "--durability", "write-sync", + ) + webdavDurability.wantExitCode(2) + webdavDurability.wantStderrContains("is not supported for WebDAV mounts") + webdavMountPath := filepath.Join(home, "mount-webdav") + webdavMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", webdavMountPath, + "--driver", "webdav", + ) + webdavMount.wantExitCode(0) + webdavMount.wantStdoutContains(`"durability": "auto"`) + for _, path := range []string{strictMountPath, webdavMountPath} { + strictUnmount := runTIWithInput(t, bin, "", baseEnv, + "fs", "unmount-file-system", + "--mount-path", path, + ) + strictUnmount.wantExitCode(0) + } drain := runTIWithInput(t, bin, "", baseEnv, "fs", "drain-file-system", "--mount-path", mountPath, @@ -1269,6 +1308,11 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { assertFakeDrive9Call(t, calls, []string{"fs", "ls"}, drive9TestToken("tenant-sandbox"), home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") assertFakeDrive9Call(t, calls, []string{"mount", "drain"}, "", home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") assertFakeDrive9Call(t, calls, []string{"umount"}, "", home, "default", "tenant-sandbox", "https://fs-east.test", "aws-us-east-1") + // The durability tier is forwarded verbatim, and a WebDAV mount forwards + // none because the engine has no durability tiers there. + assertFakeDrive9FlagValue(t, calls, "mount", mountPath, "--durability", "interactive") + assertFakeDrive9FlagValue(t, calls, "mount", strictMountPath, "--durability", "close-sync") + assertFakeDrive9FlagValue(t, calls, "mount", webdavMountPath, "--durability", "") } func TestRuntimeExecCommandSurfaceAndFailureContract(t *testing.T) { @@ -1521,7 +1565,7 @@ func TestFSLayerForkWorkflowCommands(t *testing.T) { mounted := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", "/workspace", "--driver", "fuse", "--layer-ref", "child-id", "--checkpoint-id", "seed") mounted.wantExitCode(0) mounted.wantStdoutContains(`"read_only": true`) - mounted.wantStdoutContains(`"write_back_cache": false`) + mounted.wantStdoutContains(`"durability": "auto"`) mounted.wantStdoutContains(`"checkpoint_id": "seed"`) explicitWritable := runTIWithInput(t, bin, "", env, "fs", "mount-file-system", "--mount-path", filepath.Join(home, "writable-checkpoint"), "--driver", "fuse", "--layer-ref", "child-id", "--checkpoint-id", "seed", "--read-only=false") @@ -2101,6 +2145,32 @@ func assertFakeDrive9Args(t *testing.T, calls []fakeDrive9Call, want []string) { t.Fatalf("missing exact fake Drive9 args %v in %#v", want, calls) } +// assertFakeDrive9FlagValue finds the companion call for one mount path and +// asserts the flag/value pair it was invoked with. An empty value asserts the +// flag was not forwarded at all. +func assertFakeDrive9FlagValue(t *testing.T, calls []fakeDrive9Call, command, mountPath, flag, value string) { + t.Helper() + for _, call := range calls { + if len(call.Args) == 0 || call.Args[0] != command || call.Args[len(call.Args)-1] != mountPath { + continue + } + found := "" + for i := 0; i < len(call.Args); i++ { + if call.Args[i] != flag { + continue + } + if i+1 < len(call.Args) { + found = call.Args[i+1] + } + } + if found == value { + return + } + t.Fatalf("%s for %s used %s=%q, want %q in %#v", command, mountPath, flag, found, value, call.Args) + } + t.Fatalf("missing fake Drive9 %s call for mount path %s", command, mountPath) +} + func assertFakeDrive9TransientCall(t *testing.T, calls []fakeDrive9Call, prefix []string, apiKey, persistentHome, server, regionCode string) { t.Helper() for _, call := range calls { diff --git a/e2e/live_test.go b/e2e/live_test.go index c0e89cfa..492f05fb 100644 --- a/e2e/live_test.go +++ b/e2e/live_test.go @@ -640,8 +640,9 @@ func TestLiveFSGitLifecycle(t *testing.T) { createDir := runTI(t, bin, "--profile", profileName, "fs", "create-directory", "--path", remoteRoot, "--mode", "0755") createDir.wantExitCode(0) - mount := runTI(t, bin, "--profile", profileName, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", remoteRoot, "--driver", "fuse", "--ready-timeout", "30s") + mount := runTI(t, bin, "--profile", profileName, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", remoteRoot, "--driver", "fuse", "--durability", "close-sync", "--ready-timeout", "30s") mount.wantExitCode(0) + mount.wantStdoutContains(`"durability": "close-sync"`) repoPath := filepath.Join(mountPath, "hello-world") clone := runTI(t, bin, "--profile", profileName, "fs-git", "clone-git-workspace", "--repo-url", "https://github.com/octocat/Hello-World.git", "--target-path", repoPath, "--blobless", "--hydrate", "sync") @@ -1720,6 +1721,8 @@ func TestLiveFSWebDAVMountRuntime(t *testing.T) { mount := runTI(t, bin, "--profile", profileName, "fs", "mount-file-system", "--mount-path", mountPath, "--remote-path", remoteRoot, "--driver", "webdav", "--ready-timeout", "30s") mount.wantExitCode(0) + // WebDAV mounts have no durability tiers; ti forwards none and reports auto. + mount.wantStdoutContains(`"durability": "auto"`) mount.wantStdoutContains(`"status": "mounted"`) mount.wantStdoutContains(`"driver": "webdav"`) diff --git a/internal/cli/commands.go b/internal/cli/commands.go index fee40ab8..b41d0de9 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -2630,7 +2630,8 @@ func newFSMountFileSystemCommand(info version.Info) *cobra.Command { cmd.Flags().Int64("read-cache-size-mb", 128, "FUSE read cache size in MiB. 0 uses the default.") cmd.Flags().Int64("read-cache-max-file-mb", 4, "Maximum file size admitted to the FUSE read cache in MiB. 0 uses the default.") cmd.Flags().Duration("read-cache-ttl", 30*time.Second, "FUSE read cache Time-to-Live.") - cmd.Flags().Bool("write-back-cache", true, "Persist FUSE writes locally before writing them to the file system on flush; unavailable for checkpoint mounts.") + cmd.Flags().String("durability", tifs.DefaultMountDurability, "Write durability: auto, interactive, fsync, close-sync, or write-sync. FUSE only; WebDAV mounts reject every tier except auto.") + cmd.Flags().Bool("write-back-cache", true, "Legacy alias: true selects --durability=interactive and false selects --durability=auto. Unavailable for checkpoint mounts.") cmd.Flags().String("mount-profile", "", "Mount profile: coding-agent, portable, or none. Default: none.") cmd.Flags().String("local-root", "", "Local overlay root. Default: ~/.ti/local/fs/.") cmd.Flags().StringArray("pack-path", nil, "Local overlay path included by automatic or manual pack. Repeatable.") @@ -3128,6 +3129,10 @@ func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFile if err != nil { return tifs.MountFileSystemOptions{}, err } + durability, err := ctx.StringFlag("durability") + if err != nil { + return tifs.MountFileSystemOptions{}, err + } writeBackCache, err := ctx.BoolFlag("write-back-cache") if err != nil { return tifs.MountFileSystemOptions{}, err @@ -3173,6 +3178,8 @@ func fsMountOptions(ctx commandContext, profile *config.Profile) (tifs.MountFile ReadCacheMB: readCacheMB, ReadCacheFileMB: readCacheFileMB, ReadCacheTTL: readCacheTTL, + Durability: durability, + DurabilitySet: ctx.FlagChanged("durability"), WriteBackCache: writeBackCache, WriteBackCacheSet: ctx.FlagChanged("write-back-cache"), MountProfile: mountProfile, diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index e10a3415..b959d30b 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -1283,7 +1283,7 @@ func (s Service) drive9MountVault(ctx context.Context, opts VaultMountOptions) ( if err := s.waitForMountReady(ctx, opts.MountPath, opts.ReadyTimeout, fmt.Sprintf("; to stop it run: ti fs-vault unmount-vault --mount-path %q", opts.MountPath)); err != nil { return MountResult{}, err } - return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: "vault", MountPath: opts.MountPath, RemotePath: "/n/vault", Driver: "fuse"}, nil + return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: "vault", MountPath: opts.MountPath, RemotePath: "/n/vault", Driver: "fuse", Durability: MountDurabilityAuto}, nil } func (s Service) drive9PackFileSystem(ctx context.Context, opts PackFileSystemOptions) (PackFileSystemResult, error) { diff --git a/internal/fs/drive9_mount_audit.go b/internal/fs/drive9_mount_audit.go index 6f909f74..fbfcd9a9 100644 --- a/internal/fs/drive9_mount_audit.go +++ b/internal/fs/drive9_mount_audit.go @@ -96,9 +96,9 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem if opts.ReadCacheTTL > 0 { args = append(args, "--read-cache-ttl", opts.ReadCacheTTL.String()) } - if opts.WriteBackCache { - args = append(args, "--durability", "interactive") - } + } + if mountDurabilityForwarded(opts) { + appendFlagValue(&args, "--durability", opts.Durability) } appendFlagValue(&args, "--profile", opts.MountProfile) appendFlagValue(&args, "--local-root", opts.LocalRoot) @@ -165,7 +165,7 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem if driver == "" { driver = "auto" } - return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: opts.Profile.FSResourceName, MountPath: opts.MountPath, RemotePath: remotePath, Driver: driver, Endpoint: &endpoint, MountProfile: opts.MountProfile, LocalRoot: auditRoot, PackPaths: opts.PackPaths, WriteBackCache: opts.WriteBackCache, LayerRef: opts.LayerRef, CheckpointID: opts.CheckpointID, ReadOnly: opts.ReadOnly}, nil + return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: opts.Profile.FSResourceName, MountPath: opts.MountPath, RemotePath: remotePath, Driver: driver, Endpoint: &endpoint, MountProfile: opts.MountProfile, LocalRoot: auditRoot, PackPaths: opts.PackPaths, Durability: opts.Durability, LayerRef: opts.LayerRef, CheckpointID: opts.CheckpointID, ReadOnly: opts.ReadOnly}, nil } // stopMountAfterUnsafeAudit tears down a mount whose overlay failed the audit. It diff --git a/internal/fs/mount.go b/internal/fs/mount.go index e05afad0..c6735e7e 100644 --- a/internal/fs/mount.go +++ b/internal/fs/mount.go @@ -30,18 +30,28 @@ const ( ) type MountFileSystemOptions struct { - Profile *config.Profile - FileSystemName string - MountPath string - RemotePath string - Driver string - ReadOnly bool - ReadOnlySet bool - ReadyTimeout time.Duration - CacheDir string - ReadCacheMB int64 - ReadCacheFileMB int64 - ReadCacheTTL time.Duration + Profile *config.Profile + FileSystemName string + MountPath string + RemotePath string + Driver string + ReadOnly bool + ReadOnlySet bool + ReadyTimeout time.Duration + CacheDir string + ReadCacheMB int64 + ReadCacheFileMB int64 + ReadCacheTTL time.Duration + // Durability is the write-durability tier forwarded to the companion as + // `mount --durability`. Normalization always leaves an effective tier here. + Durability string + // DurabilitySet records whether --durability was explicitly supplied. + DurabilitySet bool + // DurabilityExplicit records whether either --durability or the legacy + // --write-back-cache alias was explicitly supplied. Normalization sets it. + DurabilityExplicit bool + // WriteBackCache is the legacy boolean alias of --durability: true means + // interactive, false means auto. WriteBackCache bool WriteBackCacheSet bool MountProfile string @@ -84,7 +94,7 @@ type MountResult struct { Checks []MountRuntimeCheck `json:"checks,omitempty"` Remote *MountRemoteSnapshot `json:"remote,omitempty"` CacheDir string `json:"cache_dir,omitempty"` - WriteBackCache bool `json:"write_back_cache"` + Durability string `json:"durability"` MountProfile string `json:"mount_profile,omitempty"` LocalRoot string `json:"local_root,omitempty"` PackPaths []string `json:"pack_paths,omitempty"` @@ -146,10 +156,20 @@ func (s Service) MountFileSystem(ctx context.Context, opts MountFileSystemOption } // normalizeMountOptions trims the flag values the unpack predicate and the -// companion argv must agree on, then applies the layer rules. +// companion argv must agree on, resolves the durability tier, applies the layer +// rules, and refuses a tier the effective driver cannot honor. func normalizeMountOptions(opts MountFileSystemOptions) (MountFileSystemOptions, error) { opts.UnpackArchivePath = strings.TrimSpace(opts.UnpackArchivePath) - return normalizeLayerMountOptions(opts) + var err error + opts, err = normalizeMountDurability(opts) + if err != nil { + return MountFileSystemOptions{}, err + } + opts, err = normalizeLayerMountOptions(opts) + if err != nil { + return MountFileSystemOptions{}, err + } + return rejectUnsupportedMountDurability(opts) } func (s Service) DryRunMountFileSystem(ctx context.Context, commandPath string, opts MountFileSystemOptions) (dryrun.Result, error) { @@ -198,6 +218,7 @@ func (s Service) DryRunMountFileSystem(ctx context.Context, commandPath string, Path: "/v1/status", Body: map[string]any{ "driver": opts.Driver, + "durability": opts.Durability, "layer_ref": opts.LayerRef, "checkpoint_id": opts.CheckpointID, "read_only": opts.ReadOnly, @@ -229,8 +250,14 @@ func normalizeLayerMountOptions(opts MountFileSystemOptions) (MountFileSystemOpt if opts.WriteBackCacheSet && opts.WriteBackCache { return MountFileSystemOptions{}, apperr.New("fs.checkpoint_mount_read_only", "usage", 2, "checkpoint mounts are read-only; remove --write-back-cache=true") } + if opts.DurabilityExplicit && opts.Durability != MountDurabilityAuto { + return MountFileSystemOptions{}, apperr.New("fs.checkpoint_mount_read_only", "usage", 2, fmt.Sprintf("checkpoint mounts are read-only; remove --durability=%s", opts.Durability)) + } opts.ReadOnly = true opts.WriteBackCache = false + // A checkpoint view never writes, so ti forwards no tier and reports + // the engine's own default instead of the interactive mount default. + opts.Durability = MountDurabilityAuto } driverName := strings.TrimSpace(opts.Driver) if driverName == "" { @@ -389,6 +416,7 @@ type mountInputs struct { readCacheFileBytes int64 readCacheTTL time.Duration writeBackCache bool + durability string mountProfile string localRoot string packPaths []string @@ -493,7 +521,8 @@ func (s Service) mountInputs(opts MountFileSystemOptions) (mountInputs, error) { readCacheBytes: readCacheBytes, readCacheFileBytes: readCacheFileBytes, readCacheTTL: readCacheTTL, - writeBackCache: opts.WriteBackCache, + writeBackCache: mountDurabilityUsesWriteBack(opts.Durability), + durability: opts.Durability, mountProfile: mountProfile, localRoot: localRoot, packPaths: packPaths, @@ -572,7 +601,7 @@ func mountResult(status string, inputs mountInputs, remote apifs.StatusResponse, Checks: checks, Remote: &snapshot, CacheDir: inputs.cacheDir, - WriteBackCache: inputs.writeBackCache, + Durability: inputs.durability, MountProfile: inputs.mountProfile, LocalRoot: inputs.localRoot, PackPaths: append([]string(nil), inputs.packPaths...), diff --git a/internal/fs/mount_durability.go b/internal/fs/mount_durability.go new file mode 100644 index 00000000..b1465ff1 --- /dev/null +++ b/internal/fs/mount_durability.go @@ -0,0 +1,144 @@ +package fs + +import ( + "fmt" + "runtime" + "strings" + + "github.com/tidbcloud/ti-cli/internal/apperr" +) + +// Write-durability tiers accepted by the ti-drive9 companion's +// `mount --durability`. Each tier selects the engine's sync mode and write +// policy; ti forwards one explicitly instead of letting the companion pick its +// own default, because the engine default is auto and ti has always mounted +// with interactive semantics. +const ( + // MountDurabilityAuto lets the engine measure round-trip time at mount + // time and choose interactive or strict accordingly. It is the engine + // default and the only tier a WebDAV mount can be described with. + MountDurabilityAuto = "auto" + // MountDurabilityInteractive keeps writes durable in the local staging + // journal and uploads them asynchronously after flush. + MountDurabilityInteractive = "interactive" + // MountDurabilityFsync makes fsync(2) block until the remote commit + // succeeds while ordinary writes still stage locally. + MountDurabilityFsync = "fsync" + // MountDurabilityCloseSync makes close(2) remote-durable by uploading from + // Flush, so a completed close means the data left the client. + MountDurabilityCloseSync = "close-sync" + // MountDurabilityWriteSync makes every write remote-durable before it + // returns success. It is the strictest and most expensive tier. + MountDurabilityWriteSync = "write-sync" +) + +// DefaultMountDurability is the tier an ordinary ti mount uses. It preserves +// the behavior ti had before the tier was selectable. +const DefaultMountDurability = MountDurabilityInteractive + +// mountDurabilityValues lists the accepted tiers in increasing strictness for +// help text and error messages. +var mountDurabilityValues = []string{ + MountDurabilityAuto, + MountDurabilityInteractive, + MountDurabilityFsync, + MountDurabilityCloseSync, + MountDurabilityWriteSync, +} + +// MountDurabilityHelp renders the accepted tiers for flag and error text. +func MountDurabilityHelp() string { + return strings.Join(mountDurabilityValues, ", ") +} + +func validMountDurability(value string) bool { + for _, candidate := range mountDurabilityValues { + if value == candidate { + return true + } + } + return false +} + +// mountDurabilityUsesWriteBack reports whether a tier stages writes locally and +// leaves the remote upload to the background (drive9's writeback policy). The +// close-sync and write-sync tiers make the close or the write itself +// remote-durable instead. +func mountDurabilityUsesWriteBack(value string) bool { + switch value { + case MountDurabilityAuto, MountDurabilityInteractive, MountDurabilityFsync: + return true + default: + return false + } +} + +// normalizeMountDurability resolves --durability together with its legacy +// --write-back-cache alias into the single tier ti forwards. +func normalizeMountDurability(opts MountFileSystemOptions) (MountFileSystemOptions, error) { + value := strings.ToLower(strings.TrimSpace(opts.Durability)) + if value == "" { + value = DefaultMountDurability + } + if !validMountDurability(value) { + return MountFileSystemOptions{}, apperr.New("fs.invalid_mount_durability", "usage", 2, fmt.Sprintf("unsupported --durability value %q; supported values: %s", strings.TrimSpace(opts.Durability), MountDurabilityHelp())) + } + explicit := opts.DurabilitySet + if opts.WriteBackCacheSet { + legacy := MountDurabilityInteractive + if !opts.WriteBackCache { + legacy = MountDurabilityAuto + } + if opts.DurabilitySet && value != legacy { + return MountFileSystemOptions{}, apperr.New("fs.conflicting_mount_durability", "usage", 2, fmt.Sprintf("--durability=%s conflicts with --write-back-cache=%t; pass only --durability (--write-back-cache=true means interactive and --write-back-cache=false means auto)", value, opts.WriteBackCache)) + } + value = legacy + explicit = true + } + opts.Durability = value + opts.DurabilityExplicit = explicit + return opts, nil +} + +// rejectUnsupportedMountDurability refuses a tier the effective mount driver +// cannot honor. WebDAV mounts keep the engine's native write behavior, so ti +// forwards no tier for them and reports auto. An explicit strict tier is a +// usage error instead of a silent downgrade, so a caller that asked for +// close-sync or write-sync never believes it got it. +func rejectUnsupportedMountDurability(opts MountFileSystemOptions) (MountFileSystemOptions, error) { + if effectiveMountMode(opts.Driver) != "webdav" { + return opts, nil + } + if opts.DurabilityExplicit && opts.Durability != MountDurabilityAuto { + return MountFileSystemOptions{}, apperr.New("fs.webdav_durability_unsupported", "usage", 2, fmt.Sprintf("--durability=%s is not supported for WebDAV mounts, which always keep their native write behavior; remove --durability or use --driver fuse where FUSE is available", opts.Durability)) + } + opts.Durability = MountDurabilityAuto + return opts, nil +} + +// effectiveMountMode mirrors the companion's own `--mode auto` resolution +// (WebDAV on macOS and Windows, FUSE everywhere else) so ti's durability +// decision matches the mode the companion will actually mount with. It +// duplicates the companion rule on purpose: resolving it locally keeps the +// flag decision deterministic instead of depending on host prerequisites. +func effectiveMountMode(driver string) string { + switch strings.TrimSpace(driver) { + case "", "auto": + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return "webdav" + } + return "fuse" + case "webdav": + return "webdav" + default: + return "fuse" + } +} + +// mountDurabilityForwarded reports whether ti pins the tier on the companion +// argv. It pins FUSE mounts, including automatic ones that resolve to FUSE, so +// an ordinary mount keeps the documented interactive default instead of the +// engine's auto default. Checkpoint mounts are read-only and forward nothing. +func mountDurabilityForwarded(opts MountFileSystemOptions) bool { + return opts.CheckpointID == "" && effectiveMountMode(opts.Driver) == "fuse" +} diff --git a/internal/fs/mount_durability_test.go b/internal/fs/mount_durability_test.go new file mode 100644 index 00000000..c5d02bbd --- /dev/null +++ b/internal/fs/mount_durability_test.go @@ -0,0 +1,296 @@ +package fs + +import ( + "context" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/tidbcloud/ti-cli/internal/apperr" +) + +// TestNormalizeMountDurability pins the tier contract: --durability keeps the +// interactive default, the legacy boolean alias maps onto two tiers, and an +// explicit but unusable tier is refused instead of silently downgraded. +func TestNormalizeMountDurability(t *testing.T) { + for _, tc := range []struct { + name string + opts MountFileSystemOptions + want string + wantCode string + forwarded bool + writeBack bool + explicitly bool + }{ + { + name: "default keeps interactive", + opts: MountFileSystemOptions{Driver: "fuse"}, + want: MountDurabilityInteractive, + forwarded: true, + writeBack: true, + explicitly: false, + }, + { + name: "auto resolves to the engine default", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityAuto, DurabilitySet: true}, + want: MountDurabilityAuto, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "fsync stays a write-back tier", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityFsync, DurabilitySet: true}, + want: MountDurabilityFsync, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "close-sync leaves the write-back policy", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityCloseSync, DurabilitySet: true}, + want: MountDurabilityCloseSync, + forwarded: true, + writeBack: false, + explicitly: true, + }, + { + name: "write-sync leaves the write-back policy", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityWriteSync, DurabilitySet: true}, + want: MountDurabilityWriteSync, + forwarded: true, + writeBack: false, + explicitly: true, + }, + { + name: "value is trimmed and case-folded", + opts: MountFileSystemOptions{Driver: "fuse", Durability: " Close-Sync ", DurabilitySet: true}, + want: MountDurabilityCloseSync, + forwarded: true, + writeBack: false, + explicitly: true, + }, + { + name: "legacy true means interactive", + opts: MountFileSystemOptions{Driver: "fuse", WriteBackCache: true, WriteBackCacheSet: true}, + want: MountDurabilityInteractive, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "legacy false means auto", + opts: MountFileSystemOptions{Driver: "fuse", WriteBackCache: false, WriteBackCacheSet: true}, + want: MountDurabilityAuto, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "legacy alias may agree with the tier", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityInteractive, DurabilitySet: true, WriteBackCache: true, WriteBackCacheSet: true}, + want: MountDurabilityInteractive, + forwarded: true, + writeBack: true, + explicitly: true, + }, + { + name: "unknown tier is refused", + opts: MountFileSystemOptions{Driver: "fuse", Durability: "turbo", DurabilitySet: true}, + wantCode: "fs.invalid_mount_durability", + }, + { + name: "conflicting alias is refused", + opts: MountFileSystemOptions{Driver: "fuse", Durability: MountDurabilityAuto, DurabilitySet: true, WriteBackCache: true, WriteBackCacheSet: true}, + wantCode: "fs.conflicting_mount_durability", + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeMountOptions(tc.opts) + if tc.wantCode != "" { + if apperr.CodeFor(err) != tc.wantCode { + t.Fatalf("error = %v, want %s", err, tc.wantCode) + } + return + } + if err != nil { + t.Fatalf("normalizeMountOptions failed: %v", err) + } + if got.Durability != tc.want { + t.Fatalf("durability = %q, want %q", got.Durability, tc.want) + } + if mountDurabilityForwarded(got) != tc.forwarded { + t.Fatalf("forwarded = %t, want %t", mountDurabilityForwarded(got), tc.forwarded) + } + if mountDurabilityUsesWriteBack(got.Durability) != tc.writeBack { + t.Fatalf("write-back policy = %t, want %t", mountDurabilityUsesWriteBack(got.Durability), tc.writeBack) + } + if got.DurabilityExplicit != tc.explicitly { + t.Fatalf("explicit = %t, want %t", got.DurabilityExplicit, tc.explicitly) + } + }) + } +} + +// TestNormalizeMountDurabilityWebDAV keeps the WebDAV contract honest: the +// engine has no tiers there, so ti forwards none and refuses an explicit tier +// that would otherwise look honored. +func TestNormalizeMountDurabilityWebDAV(t *testing.T) { + got, err := normalizeMountOptions(MountFileSystemOptions{Driver: "webdav"}) + if err != nil { + t.Fatalf("default webdav mount failed: %v", err) + } + if got.Durability != MountDurabilityAuto { + t.Fatalf("webdav durability = %q, want %s", got.Durability, MountDurabilityAuto) + } + if mountDurabilityForwarded(got) { + t.Fatal("webdav mount must not forward --durability") + } + if _, err := normalizeMountOptions(MountFileSystemOptions{Driver: "webdav", Durability: MountDurabilityAuto, DurabilitySet: true}); err != nil { + t.Fatalf("explicit auto webdav mount failed: %v", err) + } + for _, tier := range []string{MountDurabilityInteractive, MountDurabilityFsync, MountDurabilityCloseSync, MountDurabilityWriteSync} { + if _, err := normalizeMountOptions(MountFileSystemOptions{Driver: "webdav", Durability: tier, DurabilitySet: true}); apperr.CodeFor(err) != "fs.webdav_durability_unsupported" { + t.Fatalf("webdav %s error = %v, want fs.webdav_durability_unsupported", tier, err) + } + } + if _, err := normalizeMountOptions(MountFileSystemOptions{Driver: "webdav", WriteBackCache: true, WriteBackCacheSet: true}); apperr.CodeFor(err) != "fs.webdav_durability_unsupported" { + t.Fatalf("webdav write-back-cache error = %v, want fs.webdav_durability_unsupported", err) + } +} + +// TestNormalizeMountDurabilityCheckpoint keeps read-only checkpoint mounts on +// the engine default and refuses a tier that cannot take effect there. +func TestNormalizeMountDurabilityCheckpoint(t *testing.T) { + base := MountFileSystemOptions{Driver: "fuse", LayerRef: "layer", CheckpointID: "cp-1"} + got, err := normalizeMountOptions(base) + if err != nil { + t.Fatalf("default checkpoint mount failed: %v", err) + } + if !got.ReadOnly { + t.Fatal("checkpoint mount must be read-only") + } + if got.Durability != MountDurabilityAuto { + t.Fatalf("checkpoint durability = %q, want %s", got.Durability, MountDurabilityAuto) + } + if mountDurabilityForwarded(got) { + t.Fatal("checkpoint mount must not forward --durability") + } + + explicitAuto := base + explicitAuto.Durability = MountDurabilityAuto + explicitAuto.DurabilitySet = true + if _, err := normalizeMountOptions(explicitAuto); err != nil { + t.Fatalf("explicit auto checkpoint mount failed: %v", err) + } + explicitLegacyOff := base + explicitLegacyOff.WriteBackCache = false + explicitLegacyOff.WriteBackCacheSet = true + if _, err := normalizeMountOptions(explicitLegacyOff); err != nil { + t.Fatalf("explicit write-back-cache=false checkpoint mount failed: %v", err) + } + + for _, tier := range []string{MountDurabilityInteractive, MountDurabilityFsync, MountDurabilityCloseSync, MountDurabilityWriteSync} { + explicit := base + explicit.Durability = tier + explicit.DurabilitySet = true + if _, err := normalizeMountOptions(explicit); apperr.CodeFor(err) != "fs.checkpoint_mount_read_only" { + t.Fatalf("checkpoint %s error = %v, want fs.checkpoint_mount_read_only", tier, err) + } + } +} + +// TestEffectiveMountModeMatchesCompanionAutoResolution pins the local mirror of +// the companion's --mode auto rule, which decides whether a tier is forwarded. +func TestEffectiveMountModeMatchesCompanionAutoResolution(t *testing.T) { + wantAuto := "fuse" + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + wantAuto = "webdav" + } + for _, tc := range []struct{ driver, want string }{ + {driver: "", want: wantAuto}, + {driver: "auto", want: wantAuto}, + {driver: " fuse ", want: "fuse"}, + {driver: "webdav", want: "webdav"}, + } { + if got := effectiveMountMode(tc.driver); got != tc.want { + t.Fatalf("effectiveMountMode(%q) = %q, want %q", tc.driver, got, tc.want) + } + } +} + +// TestDrive9MountForwardsDurability proves the selected tier reaches the +// companion argv instead of being collapsed into a boolean. +func TestDrive9MountForwardsDurability(t *testing.T) { + for _, tier := range []string{MountDurabilityAuto, MountDurabilityInteractive, MountDurabilityFsync, MountDurabilityCloseSync, MountDurabilityWriteSync} { + t.Run(tier, func(t *testing.T) { + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(t.TempDir(), companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + result, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + Durability: tier, DurabilitySet: true, + }) + if err != nil { + t.Fatalf("mount failed: %v", err) + } + if result.Durability != tier { + t.Fatalf("result durability = %q, want %q", result.Durability, tier) + } + call := requireFakeDrive9Call(t, recordPath, "mount") + want := []string{"--durability", tier} + for i := 0; i+1 < len(call.Args); i++ { + if call.Args[i] == want[0] && call.Args[i+1] == want[1] { + return + } + } + t.Fatalf("mount args %#v missing %#v", call.Args, want) + }) + } +} + +// TestDrive9MountDefaultKeepsInteractiveDurability pins the default so an +// ordinary mount never silently follows the engine's auto default. +func TestDrive9MountDefaultKeepsInteractiveDurability(t *testing.T) { + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(t.TempDir(), companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + result, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }) + if err != nil { + t.Fatalf("mount failed: %v", err) + } + if result.Durability != MountDurabilityInteractive { + t.Fatalf("result durability = %q, want %q", result.Durability, MountDurabilityInteractive) + } + call := requireFakeDrive9Call(t, recordPath, "mount") + joined := strings.Join(call.Args, " ") + if !strings.Contains(joined, "--durability "+MountDurabilityInteractive) { + t.Fatalf("mount args %#v missing the interactive default", call.Args) + } +} + +// TestDryRunMountDurability keeps --dry-run aligned with the live path. +func TestDryRunMountDurability(t *testing.T) { + service := Service{} + profile := dataProfile() + opts := MountFileSystemOptions{Profile: profile, MountPath: filepath.Join(t.TempDir(), "mnt"), Driver: "fuse", Durability: MountDurabilityCloseSync, DurabilitySet: true} + result, err := service.DryRunMountFileSystem(context.Background(), "ti fs mount-file-system", opts) + if err != nil { + t.Fatalf("dry run failed: %v", err) + } + if body, ok := result.Request.Body.(map[string]any); !ok || body["durability"] != MountDurabilityCloseSync { + t.Fatalf("dry-run body = %#v", result.Request.Body) + } + opts.Durability = MountDurabilityWriteSync + opts.Driver = "webdav" + if _, err := service.DryRunMountFileSystem(context.Background(), "ti fs mount-file-system", opts); apperr.CodeFor(err) != "fs.webdav_durability_unsupported" { + t.Fatalf("dry-run webdav error = %v, want fs.webdav_durability_unsupported", err) + } +} diff --git a/internal/fs/vault.go b/internal/fs/vault.go index 787b6ffe..4088d8ee 100644 --- a/internal/fs/vault.go +++ b/internal/fs/vault.go @@ -274,7 +274,9 @@ func vaultMountResult(status string, inputs vaultMountInputs, checks []MountRunt LogFile: logFile, Endpoint: nil, Checks: checks, - WriteBackCache: false, + // A vault mount forwards no write-durability tier, so it runs with the + // engine default. + Durability: MountDurabilityAuto, } } From 038ed4511b3662eddc9d1810029a181412f7ab67 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 19 Nov 2026 04:20:47 +0800 Subject: [PATCH 2/7] fix(fs): drain before unmount and report the engine's drain counts Write-back uploads happen after flush, so detaching a mount is the last moment a client-side loss can still be prevented. Three holes made that loss invisible: `ti fs drain-file-system` discarded the companion's `mount drain --json` report, `ti fs unmount-file-system` never drained, and `--force` was parsed but never read. - Parse the drain report into the result `response`: `ti fs drain` now returns `ok`, `error_kind`, `failed_path`, the duration and the pending counters, and prints those counters in text output. - A drain that leaves write-side work fails with `fs.drain_pending_work` and names the counters plus the retry command. - Unmount drains first, blocks with the mount locator preserved while work is pending, and gates only ti fs FUSE mounts: WebDAV has no control socket, a `vault` locator is not a file-system mount (the engine drains only those), and a locator written before the driver was recorded is not gated so an upgrade cannot block an existing mount. The mount now records the driver the companion actually used in the non-secret locator. - `--force` becomes the explicit "accept the loss" escape hatch and is reported as `drain.status: "skipped"`; `--ignore-absent` may continue past a failed drain but still reports it. - `--dry-run` reports the gate without running the drain. Spec: docs/spec/done/0039-fs-unmount-drain-gate.md --- AGENTS.md | 16 + README.md | 2 + docs/spec/done/0039-fs-unmount-drain-gate.md | 137 ++++++++ e2e/cli_test.go | 59 +++- e2e/live_test.go | 5 + e2e/testdata/fake-drive9.go | 18 + internal/cli/commands.go | 2 +- internal/fs/drive9_companion.go | 63 ++-- internal/fs/drive9_fake_companion_test.go | 15 + internal/fs/drive9_mount_audit.go | 7 +- internal/fs/drive9_mount_drain.go | 181 ++++++++++ internal/fs/drive9_mount_drain_test.go | 340 +++++++++++++++++++ internal/fs/mount.go | 57 +++- internal/fs/mountlocator/locator.go | 19 +- 14 files changed, 880 insertions(+), 41 deletions(-) create mode 100644 docs/spec/done/0039-fs-unmount-drain-gate.md create mode 100644 internal/fs/drive9_mount_drain.go create mode 100644 internal/fs/drive9_mount_drain_test.go diff --git a/AGENTS.md b/AGENTS.md index fb5403e3..569c93d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -661,6 +661,7 @@ Implemented command behavior: - `ti fs drain-file-system --mount-path ./workspace --timeout 30s` - `ti fs unmount-file-system --mount-path ./workspace` - `ti fs unmount-file-system --mount-path ./workspace --ignore-absent` +- `ti fs unmount-file-system --mount-path ./workspace --force` - `ti fs-vault create-secret --secret-name db-prod --field DB_URL=mysql://example --field PASSWORD=@./password.txt` - `ti fs-vault replace-secret --secret-path /n/vault/db-prod --from-directory ./secret-fields` - `ti fs-vault read-secret --secret-name db-prod` @@ -1150,6 +1151,21 @@ Background FS and vault mounts write only a non-secret locator under original resource-scoped companion HOME without requiring the FS token again. Successful unmount removes the locator; failed unmount preserves it for retry. +`ti fs unmount-file-system` drains before it detaches a FUSE mount and must never +report success while write-back work is still local. Parse the companion's +`mount drain --json` report instead of discarding it: `ti fs drain-file-system` +returns it as `response`, prints the pending counters in text output, and fails +with `fs.drain_pending_work` when work remains. The unmount records the driver +the companion actually mounted in the non-secret mount locator, gates only FUSE +mounts on the drain (and never a `vault` locator: the engine drains only +file-system mounts), blocks with the locator preserved while work is pending, +and treats `--force` as the explicit "accept the loss" escape hatch (reported as +`drain.status: "skipped"`). `--ignore-absent` may continue past a drain failure +but must still report `drain.status: "failed"`. A locator written before the +driver was recorded is never gated, so an upgrade cannot block an existing +unmount. Do not drop these outcomes from the result, and do not reimplement the +drain in ti: the mount control socket belongs to the companion. + `ti fs mount-file-system` forwards the selected write-durability tier to the companion as `mount --durability`. The flag accepts `auto`, `interactive`, `fsync`, `close-sync`, and `write-sync`, and defaults to `interactive` so an diff --git a/README.md b/README.md index 6fd8fad0..1e4cfbbb 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,8 @@ ti fs drain-file-system --mount-path ~/my-workspace ti fs unmount-file-system --mount-path ~/my-workspace ``` +`ti fs drain-file-system` reports the companion's drain result, including the pending counters (`open_handles`, `commit_queue_*`, `uploader_*`), so a caller can tell "fully uploaded" from "the background upload never finished". `ti fs unmount-file-system` runs that drain itself before detaching a FUSE mount and fails with `fs.drain_pending_work` while writes are still pending, keeping the mount locator so the command can be retried; use `--force` to accept the undrained writes anyway, and note that the result then records `drain.status: "skipped"`. `--ignore-absent` continues past a drain that fails because the mount is already gone but still reports `drain.status: "failed"`. WebDAV mounts keep their native write behavior and are not gated, and `ti fs-vault unmount-vault` is not gated either because the engine drains only file-system mounts. + Run a command against a remote Filesystem (Runtime is available only in AWS `us-east-1` and `us-west-2`): ```shell diff --git a/docs/spec/done/0039-fs-unmount-drain-gate.md b/docs/spec/done/0039-fs-unmount-drain-gate.md new file mode 100644 index 00000000..e20c8630 --- /dev/null +++ b/docs/spec/done/0039-fs-unmount-drain-gate.md @@ -0,0 +1,137 @@ +# Drain Before Unmount + +## Goal + +Stop `ti fs unmount-file-system` from reporting success while local write-back +writes never reached the remote. An unmount must drain pending writes first, and +either refuse to detach while work is still pending or say explicitly that the +caller accepted the loss. `ti fs drain-file-system` must show what the engine +counted instead of discarding its report. + +## Problem + +Drive9's write-back policy keeps a write durable in the local staging journal +and uploads it in the background after flush. That makes the client's loss +window a background-upload race, and three holes made it invisible: + +1. `drive9 umount` stops the mount with a signal; the worker's SIGTERM handler + runs `flushOpenHandles` and `chain.Drain` but discards both errors + (`pkg/fuse/mount.go`, the shutdown goroutine), then exits with status 0 + because the stop was requested. A failed drain therefore looks like a clean + unmount, and the pending writes are gone with the mount. +2. `drive9 umount` never asks the mount's control socket for pending work, so it + cannot gate the detach on it. +3. `ti fs drain-file-system` ran `mount drain --json` and discarded the report, + and `ti fs unmount-file-system` never drained at all. + +Consequence: unmount returned success, the upload never happened, nothing +warned, and afterwards neither the user nor support could tell a client-side +loss from a server-side failure. + +## User Outcome + +```bash +# Reports the engine's drain report, including the pending counters: +ti fs drain-file-system --mount-path ./workspace + +# Refuses to detach while the drain reports pending work, keeping the locator: +ti fs unmount-file-system --mount-path ./workspace + +# Explicitly accepts the loss and detaches anyway: +ti fs unmount-file-system --mount-path ./workspace --force +``` + +A pending unmount fails with `fs.drain_pending_work`, names the engine's +`error_kind`, prints the non-zero pending counters, and shows the retry and +`--force` commands. A successful unmount reports the drain pass in its result: + +```json +{ + "status": "unmounted", + "mount_path": "./workspace", + "drain": { "status": "drained", "response": { "ok": true, "pending": { } } } +} +``` + +## Command Contract + +### `ti fs drain-file-system` + +- The companion's `mount drain --json` report is parsed into + `response` (same shape as `mountcontrol.DrainResponse`) instead of being + dropped. +- `status` is `drained`. A drain that leaves write-side work fails with + `fs.drain_pending_work`; any other drain failure uses `fs.drain_failed`. Both + errors carry `error_kind`, the engine's error text, and the non-zero pending + counters, plus the retry command. +- Text output prints the pending counters, so `--output text` is usable in an + operator runbook. + +### `ti fs unmount-file-system` + +- Unless skipped, ti runs the same drain before `umount`. The drain uses the + command's `--timeout` (default 30s). +- The drain applies only to ti fs FUSE mounts. ti records the driver the + companion reported in the mount locator at mount time; WebDAV mounts keep + their native write behavior and have no control socket, so they are not + gated. The locator's mount kind gates too, because the engine drains only + file-system mounts: `ti fs-vault unmount-vault` (kind `vault`) is never + drained. A locator written before ti recorded the driver is also not gated, + so an upgrade never blocks an existing unmount. +- A failed or pending drain blocks the unmount and keeps the locator, so the + caller can close the writers and retry. +- `--force` skips the gate. The result records `drain.status: "skipped"` with + the reason, so a forced unmount is still attributable. +- `--ignore-absent` keeps its cleanup meaning: a drain that fails because the + mount is already gone does not block, but the result records + `drain.status: "failed"` and a `drain` runtime check instead of pretending the + drain succeeded. +- The result reports the recorded driver and the drain record. Text output adds + the `DRAIN` column and the pending counters. +- `--dry-run` reports the gate in a `pre_unmount_drain` check without running + the drain. + +The `--force` flag previously promised to "kill the mount process" but no code +read it; it now means "accept the undrained writes". + +## Scope + +In scope (this repository): + +- `internal/fs`: drain report parsing and rendering, the unmount gate, driver + recording in the mount locator, dry-run checks, result types. +- `internal/cli`: `--force` help text. +- `internal/fs/mountlocator`: the non-secret `driver` field. +- README, AGENTS.md, unit tests, fake-companion e2e, and live e2e. + +Out of scope (Drive9 repository, tracked there): + +- `drive9 umount` reporting and gating on pending work itself, so a non-ti + caller gets the same protection. +- The SIGTERM handler surfacing `flushOpenHandles`/`chain.Drain` failures + instead of discarding them. +- `ti fs-vault unmount-vault`, which forwards no durability tier and is not + gated: the engine drains only file-system mounts. + +ti can only refuse to detach; it cannot make the engine's shutdown path report +its own drain failure. Until Drive9 changes that, a `kill -TERM` of the mount +process (or a supervisor restart) can still lose staged writes, and ti's gate +remains the client-side mitigation. + +## Verification + +- Unit tests cover: a healthy drain report in the result and text output, a + pending drain error with counters and codes, the gate blocking and preserving + the locator, `--force` skipping it, WebDAV and legacy-locator skips, + `--ignore-absent` still reporting the failure, and the driver recorded at + mount time. +- `make e2e` drives a pending drain through `ti fs drain` and `ti fs unmount` + against the fake companion: both fail with the counters, `--force` unmounts + and reports `skipped`, a WebDAV unmount issues no drain call, and the drain + result carries `response.pending`. +- `make live-e2e-fs` asserts the real drain report fields on a live FUSE mount; + the live unmount now drains first. + +The English PingCAP pages under `docs/pingcap-docs/docs/` are a submodule that +was not checked out while this spec was implemented; the drain/unmount page must +document the gate and `--force` when that submodule is available. diff --git a/e2e/cli_test.go b/e2e/cli_test.go index 3b04e137..609bcbf5 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -1259,23 +1259,61 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { ) webdavMount.wantExitCode(0) webdavMount.wantStdoutContains(`"durability": "auto"`) - for _, path := range []string{strictMountPath, webdavMountPath} { - strictUnmount := runTIWithInput(t, bin, "", baseEnv, - "fs", "unmount-file-system", - "--mount-path", path, - ) - strictUnmount.wantExitCode(0) - } + webdavUnmount := runTIWithInput(t, bin, "", baseEnv, + "fs", "unmount-file-system", + "--mount-path", webdavMountPath, + ) + webdavUnmount.wantExitCode(0) drain := runTIWithInput(t, bin, "", baseEnv, "fs", "drain-file-system", "--mount-path", mountPath, ) drain.wantExitCode(0) + // The drain result must carry the engine's report instead of dropping the + // companion's --json output, and text output must show the counters. + drain.wantStdoutContains(`"response"`) + drain.wantStdoutContains(`"pending"`) + drainText := runTIWithInput(t, bin, "", baseEnv, + "fs", "drain-file-system", + "--mount-path", mountPath, + "--output", "text", + ) + drainText.wantExitCode(0) + drainText.wantStdoutContains("pending:") + drainText.wantStdoutContains("open_handles=") unmount := runTIWithInput(t, bin, "", baseEnv, "fs", "unmount-file-system", "--mount-path", mountPath, ) unmount.wantExitCode(0) + unmount.wantStdoutContains(`"drain"`) + unmount.wantStdoutContains(`"status": "drained"`) + + // A pending drain blocks the unmount, keeps the locator for retry, and + // --force is the explicit way to accept the loss. + pendingDrainEnv := append(append([]string{}, baseEnv...), "FAKE_DRIVE9_DRAIN_PENDING=1") + pendingDrain := runTIWithInput(t, bin, "", pendingDrainEnv, + "fs", "drain-file-system", + "--mount-path", strictMountPath, + ) + pendingDrain.wantExitCode(1) + pendingDrain.wantStderrContains("error_kind=pending_work_remaining") + pendingDrain.wantStderrContains("commit_queue_pending=2") + pendingUnmount := runTIWithInput(t, bin, "", pendingDrainEnv, + "fs", "unmount-file-system", + "--mount-path", strictMountPath, + ) + pendingUnmount.wantExitCode(1) + pendingUnmount.wantStderrContains("error_kind=pending_work_remaining") + pendingUnmount.wantStderrContains("unmount with --force") + forcedUnmount := runTIWithInput(t, bin, "", pendingDrainEnv, + "fs", "unmount-file-system", + "--mount-path", strictMountPath, + "--force", + ) + forcedUnmount.wantExitCode(0) + forcedUnmount.wantStdoutContains(`"status": "skipped"`) + forcedUnmount.wantStdoutContains("--force skips the pre-unmount drain") for _, path := range []string{ filepath.Join(home, ".tdc", "config"), @@ -1313,6 +1351,13 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { assertFakeDrive9FlagValue(t, calls, "mount", mountPath, "--durability", "interactive") assertFakeDrive9FlagValue(t, calls, "mount", strictMountPath, "--durability", "close-sync") assertFakeDrive9FlagValue(t, calls, "mount", webdavMountPath, "--durability", "") + // A FUSE unmount drains before detaching; a WebDAV unmount has no control + // socket and must not try. + for _, call := range calls { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" && call.Args[len(call.Args)-1] == webdavMountPath { + t.Fatalf("webdav unmount drained: %#v", call.Args) + } + } } func TestRuntimeExecCommandSurfaceAndFailureContract(t *testing.T) { diff --git a/e2e/live_test.go b/e2e/live_test.go index 492f05fb..650482e4 100644 --- a/e2e/live_test.go +++ b/e2e/live_test.go @@ -1587,6 +1587,11 @@ func TestLiveFSConfigurationFreeAccess(t *testing.T) { if strings.Contains(mount.stdout, `"driver": "fuse"`) { drain := runTIWithInput(t, bin, "", locatorEnv, "fs", "drain-file-system", "--mount-path", mountPath, "--timeout", "30s") drain.wantExitCode(0) + // The drain report must be surfaced: the pending counters are how an + // operator tells "fully uploaded" from "the upload never finished". + drain.wantStdoutContains(`"response"`) + drain.wantStdoutContains(`"pending"`) + drain.wantStdoutContains(`"open_handles"`) } unmount := runTIWithInput(t, bin, "", locatorEnv, "fs", "unmount-file-system", "--mount-path", mountPath) unmount.wantExitCode(0) diff --git a/e2e/testdata/fake-drive9.go b/e2e/testdata/fake-drive9.go index 09e20b98..9990c3d8 100644 --- a/e2e/testdata/fake-drive9.go +++ b/e2e/testdata/fake-drive9.go @@ -232,6 +232,24 @@ func main() { fmt.Println(`{"status":"unpacked"}`) return } + if hasPrefix(args, "mount", "drain") { + ok := os.Getenv("FAKE_DRIVE9_DRAIN_PENDING") != "1" + report := map[string]any{ + "ok": ok, "mount_point": args[len(args)-1], "duration_ms": 5, + "native_syncfs_supported": false, "pending": map[string]any{}, + } + if !ok { + report["error"] = "writeback uploader: 2 entries remain" + report["error_kind"] = "pending_work_remaining" + report["pending"] = map[string]any{"commit_queue_pending": 2, "uploader_queued": 2} + } + encoded, _ := json.Marshal(report) + fmt.Println(string(encoded)) + if !ok { + os.Exit(1) + } + return + } if hasPrefix(args, "mount") { fmt.Fprintln(os.Stderr, "drive9: mount mode: "+mountMode(args)) if mountPath := args[len(args)-1]; len(args) >= 2 && len(mountPath) > 0 && mountPath[0] != '-' { diff --git a/internal/cli/commands.go b/internal/cli/commands.go index b41d0de9..13456f7f 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -2675,7 +2675,7 @@ func newFSUnmountFileSystemCommand(info version.Info) *cobra.Command { }, info) cmd.Flags().String("mount-path", "", "The local mounted path.") cmd.Flags().Duration("timeout", 30*time.Second, "Time to wait for the mount process to exit.") - cmd.Flags().Bool("force", false, "Kill the mount process if graceful unmount times out.") + cmd.Flags().Bool("force", false, "Skip the pre-unmount drain gate and unmount even when local writes are still pending.") cmd.Flags().Bool("ignore-absent", false, "Return success when no file system mount state exists for the specified path.") cmd.Flags().String("pack-archive-path", "", "Pack archive to write after unmount") cmd.Flags().Bool("no-auto-pack", false, "Skip the portable mount profile's default auto-pack action.") diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index b959d30b..bc58b530 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -1271,7 +1271,7 @@ func (s Service) drive9MountVault(ctx context.Context, opts VaultMountOptions) ( if _, err := s.drive9Runner().Run(ctx, fswrap.RunOptions{Profile: opts.Profile, Args: args, IncludeFSAPIKey: true, VaultToken: opts.VaultToken}); err != nil { return MountResult{}, err } - if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "vault"); err != nil { + if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "vault", "fuse"); err != nil { if umountErr := s.umountMount(ctx, MountFileSystemOptions{Profile: opts.Profile, MountPath: opts.MountPath}); umountErr != nil { // Compose an apperr error: the CLI renders apperr.MessageFor, which // reads only the embedded apperr message, so a fmt.Errorf wrapper would @@ -1359,29 +1359,36 @@ func (s Service) drive9UnpackFileSystem(ctx context.Context, opts UnpackFileSyst } func (s Service) drive9DrainFileSystem(ctx context.Context, opts DrainFileSystemOptions) (DrainResult, error) { - profile, err := s.drive9MountLocatorProfile(opts.Profile, opts.MountPath) + locator, profile, err := s.mountLocatorProfile(opts.Profile, opts.MountPath) if err != nil { return DrainResult{}, err } - args := []string{"mount", "drain", "--json"} - if opts.Timeout > 0 { - args = append(args, "--timeout", opts.Timeout.String()) - } - args = append(args, opts.MountPath) - if _, err := s.drive9Run(ctx, profile, args, true); err != nil { + // The companion's report is the only place the undrained work is counted, + // so keep it in the result instead of discarding it. + drain, err := s.drainMount(ctx, profile, opts.MountPath, opts.Timeout) + if err != nil { return DrainResult{}, err } - return DrainResult{Status: "drained", MountPath: opts.MountPath}, nil + return DrainResult{Status: drain.Status, MountPath: opts.MountPath, Driver: locator.Driver, Response: drain.Response}, nil } func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSystemOptions) (UnmountResult, error) { - profile, err := s.drive9MountLocatorProfile(opts.Profile, opts.MountPath) + locator, profile, err := s.mountLocatorProfile(opts.Profile, opts.MountPath) if err != nil { if opts.IgnoreAbsent && apperr.CodeFor(err) == "fs.mount_locator_not_found" { return UnmountResult{Status: "absent", MountPath: opts.MountPath}, nil } return UnmountResult{}, err } + // Detaching the kernel mount stops the client from noticing that local + // write-back work never reached the remote, so drain first and refuse to + // detach while writes are still pending. --force skips the gate, and + // --ignore-absent tolerates a drain that fails because the mount is already + // gone. + drain, drainErr := s.drainBeforeUnmount(ctx, profile, locator, opts) + if drainErr != nil && !opts.IgnoreAbsent { + return UnmountResult{}, drainErr + } args := []string{"umount"} if opts.Timeout > 0 { args = append(args, "--timeout", opts.Timeout.String()) @@ -1402,7 +1409,7 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy if removeErr := mountlocator.Remove(homeDir, opts.MountPath); removeErr != nil { return UnmountResult{}, apperr.Wrap("fs.remove_mount_locator", "runtime", 1, "remove stale ti fs mount locator", removeErr) } - return UnmountResult{Status: "absent", MountPath: opts.MountPath}, nil + return UnmountResult{Status: "absent", MountPath: opts.MountPath, Driver: locator.Driver, Drain: &drain}, nil } return UnmountResult{}, err } @@ -1413,10 +1420,16 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy if err := mountlocator.Remove(homeDir, opts.MountPath); err != nil { return UnmountResult{}, apperr.Wrap("fs.remove_mount_locator", "runtime", 1, "remove ti fs mount locator", err) } - return UnmountResult{Status: "unmounted", MountPath: opts.MountPath}, nil + result := UnmountResult{Status: "unmounted", MountPath: opts.MountPath, Driver: locator.Driver, Drain: &drain} + if drainErr != nil { + // --ignore-absent asked ti to continue past a drain it could not + // complete, so the outcome must still say the writes may be lost. + result.Checks = append(result.Checks, MountRuntimeCheck{Name: "drain", Status: "failed", Message: apperr.MessageFor(drainErr)}) + } + return result, nil } -func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kind string, mountOpts ...MountFileSystemOptions) error { +func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kind, driver string, mountOpts ...MountFileSystemOptions) error { if profile == nil { return apperr.New("fs.missing_profile", "config", 2, "active profile is required") } @@ -1432,7 +1445,7 @@ func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kin if err != nil { return apperr.Wrap("fs.write_mount_locator", "runtime", 1, "construct ti fs mount locator", err) } - locator = locator.WithTokenCorrelation(profile.FSTenantID, profile.FSTokenID, fsTokenFingerprint(profile.FSAPIKey)).WithServiceAddress(profile.FSServiceAddress) + locator = locator.WithTokenCorrelation(profile.FSTenantID, profile.FSTokenID, fsTokenFingerprint(profile.FSAPIKey)).WithServiceAddress(profile.FSServiceAddress).WithDriver(driver) if len(mountOpts) > 0 { opts := mountOpts[0] locator = locator.WithLayerView(opts.RemotePath, opts.LayerRef, opts.CheckpointID, opts.ReadOnly) @@ -1444,27 +1457,35 @@ func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kin } func (s Service) drive9MountLocatorProfile(base *config.Profile, mountPath string) (*config.Profile, error) { + _, profile, err := s.mountLocatorProfile(base, mountPath) + return profile, err +} + +// mountLocatorProfile resolves the ti-owned mount locator for a mount path and +// rebuilds the companion profile from it. Callers that also need locator facts +// (the recorded driver, for example) use this instead of reading it twice. +func (s Service) mountLocatorProfile(base *config.Profile, mountPath string) (mountlocator.Locator, *config.Profile, error) { homeDir, err := s.homeDir() if err != nil { - return nil, err + return mountlocator.Locator{}, nil, err } locator, _, err := mountlocator.Read(homeDir, mountPath) if err != nil { if os.IsNotExist(err) { - return nil, apperr.New("fs.mount_locator_not_found", "runtime", 1, fmt.Sprintf("no ti fs mount locator found for %q", mountPath)) + return mountlocator.Locator{}, nil, apperr.New("fs.mount_locator_not_found", "runtime", 1, fmt.Sprintf("no ti fs mount locator found for %q", mountPath)) } - return nil, apperr.Wrap("fs.read_mount_locator", "runtime", 1, fmt.Sprintf("read ti fs mount locator for %q", mountPath), err) + return mountlocator.Locator{}, nil, apperr.Wrap("fs.read_mount_locator", "runtime", 1, fmt.Sprintf("read ti fs mount locator for %q", mountPath), err) } expectedHome, err := fscred.CompanionHome(homeDir, locator.Profile, locator.FileSystemName) if err != nil { - return nil, err + return mountlocator.Locator{}, nil, err } if filepath.Clean(expectedHome) != filepath.Clean(locator.CompanionHome) { - return nil, apperr.New("fs.mount_locator_invalid", "config", 2, "ti fs mount locator companion home does not match its profile and file system") + return mountlocator.Locator{}, nil, apperr.New("fs.mount_locator_invalid", "config", 2, "ti fs mount locator companion home does not match its profile and file system") } placement, err := region.ParseCanonicalCode(locator.RegionCode) if err != nil { - return nil, apperr.Wrap("fs.mount_locator_invalid", "config", 2, "ti fs mount locator has an invalid region", err) + return mountlocator.Locator{}, nil, apperr.Wrap("fs.mount_locator_invalid", "config", 2, "ti fs mount locator has an invalid region", err) } profile := config.Profile{HomeDir: homeDir} if base != nil { @@ -1488,7 +1509,7 @@ func (s Service) drive9MountLocatorProfile(base *config.Profile, mountPath strin profile.FSServiceAddress = credential.ServiceAddress } } - return &profile, nil + return locator, &profile, nil } func drive9CopyArgs(opts CopyFileOptions) ([]string, string, string, error) { diff --git a/internal/fs/drive9_fake_companion_test.go b/internal/fs/drive9_fake_companion_test.go index f7f81aff..d7692c31 100644 --- a/internal/fs/drive9_fake_companion_test.go +++ b/internal/fs/drive9_fake_companion_test.go @@ -98,6 +98,21 @@ func main() { "region_code": os.Getenv("DRIVE9_REGION_CODE"), "server": os.Getenv("DRIVE9_SERVER"), }) + case len(args) >= 2 && args[0] == "mount" && args[1] == "drain": + pending := os.Getenv("TI_FAKE_DRIVE9_DRAIN_PENDING") == "1" + report := map[string]any{ + "ok": !pending, "mount_point": args[len(args)-1], "duration_ms": 7, + "native_syncfs_supported": false, "pending": map[string]any{}, + } + if pending { + report["error"] = "writeback uploader: 2 entries remain" + report["error_kind"] = "pending_work_remaining" + report["pending"] = map[string]any{"commit_queue_pending": 2, "uploader_queued": 2} + } + _ = json.NewEncoder(os.Stdout).Encode(report) + if pending { + os.Exit(1) + } case args[0] == "umount": if os.Getenv("TI_FAKE_UMOUNT_FAIL") == "1" { fmt.Fprintln(os.Stderr, "umount: mount is busy") diff --git a/internal/fs/drive9_mount_audit.go b/internal/fs/drive9_mount_audit.go index fbfcd9a9..3f6381da 100644 --- a/internal/fs/drive9_mount_audit.go +++ b/internal/fs/drive9_mount_audit.go @@ -126,7 +126,10 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem } return MountResult{}, runErr } - if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "fs", opts); err != nil { + // Record the driver the companion actually mounted with, so the matching + // unmount knows whether the mount supports the drain gate. + mountedDriver := drive9MountedDriver(result.Stderr, opts.Driver) + if err := s.writeDrive9MountLocator(opts.Profile, opts.MountPath, "fs", mountedDriver, opts); err != nil { // A partially completed unpack can leave an escaping link behind, and a // later mount would record it as pre-existing state, so audit before // deciding whether the failed mount can be torn down. @@ -161,7 +164,7 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem return MountResult{}, readyErr } endpoint, _ := s.resolveFS(opts.Profile) - driver := drive9MountedDriver(result.Stderr, opts.Driver) + driver := mountedDriver if driver == "" { driver = "auto" } diff --git a/internal/fs/drive9_mount_drain.go b/internal/fs/drive9_mount_drain.go new file mode 100644 index 00000000..1b48c249 --- /dev/null +++ b/internal/fs/drive9_mount_drain.go @@ -0,0 +1,181 @@ +package fs + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/tidbcloud/ti-cli/internal/apperr" + "github.com/tidbcloud/ti-cli/internal/config" + "github.com/tidbcloud/ti-cli/internal/fs/mountcontrol" + "github.com/tidbcloud/ti-cli/internal/fs/mountlocator" +) + +// decodeDrive9DrainResponse parses the `mount drain --json` report. The +// companion prints the report before it exits, including when the drain failed, +// so a caller can report the undrained work instead of only an exit status. +func decodeDrive9DrainResponse(stdout []byte) (*mountcontrol.DrainResponse, error) { + trimmed := bytes.TrimSpace(stdout) + if len(trimmed) == 0 { + return nil, apperr.New("fs.drain_report_missing", "runtime", 1, "the ti fs companion printed no drain report; update ti-drive9 so `mount drain --json` reports pending work") + } + var response mountcontrol.DrainResponse + if err := json.Unmarshal(trimmed, &response); err != nil { + return nil, apperr.Wrap("fs.drain_report_invalid", "runtime", 1, "decode the ti-drive9 drain report", err) + } + return &response, nil +} + +// drainPendingSummary renders the undrained counters the engine reports, so an +// operator can tell "nothing was pending" from "the upload of eight files never +// finished" without reading the raw JSON. +func drainPendingSummary(pending mountcontrol.DrainPending) string { + counters := []struct { + name string + value int64 + }{ + {"open_handles", int64(pending.OpenHandles)}, + {"dirty_handles", int64(pending.DirtyHandles)}, + {"commit_queue_pending", int64(pending.CommitQueuePending)}, + {"commit_queue_bytes", pending.CommitQueueBytes}, + {"commit_queue_in_flight", int64(pending.CommitQueueInFlight)}, + {"commit_queue_delayed", int64(pending.CommitQueueDelayed)}, + {"commit_queue_conflicts", int64(pending.CommitQueueConflicts)}, + {"uploader_queued", int64(pending.UploaderQueued)}, + {"uploader_in_flight", int64(pending.UploaderInFlight)}, + {"uploader_cached", int64(pending.UploaderCached)}, + {"uploader_cached_bytes", pending.UploaderCachedBytes}, + } + rendered := make([]string, 0, len(counters)) + for _, counter := range counters { + if counter.value == 0 { + continue + } + rendered = append(rendered, fmt.Sprintf("%s=%d", counter.name, counter.value)) + } + if len(rendered) == 0 { + return "none" + } + return strings.Join(rendered, " ") +} + +// describeDrainFailure renders the engine's drain report for a user-facing +// error: what failed, which path, and what work is still undrained. +func describeDrainFailure(response *mountcontrol.DrainResponse) string { + if response == nil { + return "the companion reported no drain detail" + } + parts := make([]string, 0, 5) + if response.MountPoint != "" { + parts = append(parts, fmt.Sprintf("mount %q", response.MountPoint)) + } + if response.ErrorKind != "" { + parts = append(parts, "error_kind="+response.ErrorKind) + } + if response.Error != "" { + parts = append(parts, "error="+response.Error) + } + parts = append(parts, "pending: "+drainPendingSummary(response.Pending)) + return strings.Join(parts, "; ") +} + +// drainErrorCode classifies a drain failure. Undrained write-back work is the +// dangerous case (the data is still only local), so it gets its own code. +func drainErrorCode(response *mountcontrol.DrainResponse) string { + if response != nil && response.ErrorKind == "pending_work_remaining" { + return "fs.drain_pending_work" + } + return "fs.drain_failed" +} + +// mountKindFileSystem marks a ti fs mount in the mount locator. Vault mounts +// record their own kind and are never drained: the engine drains only FUSE +// file-system mounts, and a vault mount has its own control plane. +const mountKindFileSystem = "fs" + +// mountDriverSupportsDrain reports whether a recorded mount driver can be +// drained. A FUSE mount exposes a control socket; WebDAV keeps the engine's +// native write behavior and has no drain. An unknown driver (a locator written +// before the driver was recorded) is not gated, so an upgrade never blocks an +// existing mount. +func mountDriverSupportsDrain(driver string) bool { + switch strings.TrimSpace(driver) { + case "": + return false + case "webdav": + return false + case "fuse": + return true + default: + return effectiveMountMode(driver) == "fuse" + } +} + +// mountDrainGate reports whether an unmount of this locator should drain first, +// and why it will not. Only a ti fs FUSE mount has a drain gate. +func mountDrainGate(locator mountlocator.Locator, force bool) (bool, string) { + if force { + return false, "unmount --force skips the pre-unmount drain" + } + switch strings.TrimSpace(locator.Kind) { + case mountKindFileSystem: + case "": + return false, "the mount locator records no mount kind, so ti cannot confirm the mount supports drain" + default: + return false, fmt.Sprintf("%s mounts have no drain gate; the companion stops them as they are", strings.TrimSpace(locator.Kind)) + } + if !mountDriverSupportsDrain(locator.Driver) { + return false, mountDrainSkipReason(locator.Driver) + } + return true, "" +} + +// mountDrainSkipReason explains why a driver cannot be drained. +func mountDrainSkipReason(driver string) string { + if strings.TrimSpace(driver) == "" { + return "the mount locator records no driver, so ti cannot confirm the mount supports drain" + } + return fmt.Sprintf("%s mounts have no mount control socket to drain; the companion flushes them when it unmounts", strings.TrimSpace(driver)) +} + +// drainMount runs the companion's drain and reports its result. It returns the +// drain record even when the drain failed, so a caller that continues anyway +// (unmount --force or --ignore-absent) can say what was left undrained. +func (s Service) drainMount(ctx context.Context, profile *config.Profile, mountPath string, timeout time.Duration) (MountDrain, error) { + args := []string{"mount", "drain", "--json"} + if timeout > 0 { + args = append(args, "--timeout", timeout.String()) + } + args = append(args, mountPath) + result, runErr := s.drive9Run(ctx, profile, args, true) + response, decodeErr := decodeDrive9DrainResponse(result.Stdout) + if decodeErr != nil { + if runErr != nil { + return MountDrain{Status: "failed", Message: apperr.MessageFor(runErr)}, runErr + } + return MountDrain{Status: "failed", Message: apperr.MessageFor(decodeErr)}, decodeErr + } + if runErr != nil || !response.OK { + record := MountDrain{Status: "failed", Message: describeDrainFailure(response), Response: response} + return record, apperr.New(drainErrorCode(response), "runtime", 1, fmt.Sprintf("drain %s did not finish: %s; close the writers or let the background upload finish, then retry ti fs drain-file-system --mount-path %q, or unmount with --force to accept losing the undrained writes", mountPath, describeDrainFailure(response), mountPath)) + } + return MountDrain{Status: "drained", Response: response}, nil +} + +// drainBeforeUnmount is the unmount gate: an ordinary unmount drains first and +// refuses to detach while local writes are still pending. --force skips the +// gate explicitly, and a mount that cannot be drained (WebDAV, or a locator +// written before ti recorded the driver) is not gated. +func (s Service) drainBeforeUnmount(ctx context.Context, profile *config.Profile, locator mountlocator.Locator, opts UnmountFileSystemOptions) (MountDrain, error) { + if drainable, reason := mountDrainGate(locator, opts.Force); !drainable { + return MountDrain{Status: "skipped", Message: reason}, nil + } + timeout := opts.Timeout + if timeout <= 0 { + timeout = mountcontrol.DefaultDrainTimeout + } + return s.drainMount(ctx, profile, opts.MountPath, timeout) +} diff --git a/internal/fs/drive9_mount_drain_test.go b/internal/fs/drive9_mount_drain_test.go new file mode 100644 index 00000000..59ba862f --- /dev/null +++ b/internal/fs/drive9_mount_drain_test.go @@ -0,0 +1,340 @@ +package fs + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/tidbcloud/ti-cli/internal/apperr" + "github.com/tidbcloud/ti-cli/internal/fs/fscred" + "github.com/tidbcloud/ti-cli/internal/fs/mountlocator" +) + +// TestDrive9DrainReportsPendingCounters keeps the engine's drain report in the +// result instead of discarding the `--json` output. +func TestDrive9DrainReportsPendingCounters(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + + result, err := service.DrainFileSystem(context.Background(), DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("drain failed: %v", err) + } + if result.Status != "drained" || result.Response == nil || !result.Response.OK { + t.Fatalf("unexpected drain result: %#v", result) + } + if result.Driver != "fuse" { + t.Fatalf("drain driver = %q, want fuse", result.Driver) + } + if result.Response.DurationMS != 7 { + t.Fatalf("drain duration = %d, want the companion's report", result.Response.DurationMS) + } + human := result.Human() + if !strings.Contains(human, "pending:") || !strings.Contains(human, "open_handles=0") { + t.Fatalf("text output dropped the pending counters:\n%s", human) + } +} + +// TestDrive9DrainFailureNamesPendingWork turns an undrained mount into an +// actionable error with the counters that explain it. +func TestDrive9DrainFailureNamesPendingWork(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + + _, err := service.DrainFileSystem(context.Background(), DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if apperr.CodeFor(err) != "fs.drain_pending_work" { + t.Fatalf("drain error = %v, want fs.drain_pending_work", err) + } + message := apperr.MessageFor(err) + for _, want := range []string{"pending_work_remaining", "commit_queue_pending=2", "uploader_queued=2", "ti fs drain-file-system"} { + if !strings.Contains(message, want) { + t.Fatalf("drain error %q missing %q", message, want) + } + } +} + +// TestUnmountDrainsBeforeDetach is the core fix: unmount must not detach while +// local writes are still pending. +func TestUnmountDrainsBeforeDetach(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("unmount failed: %v", err) + } + if result.Status != "unmounted" || result.Drain == nil || result.Drain.Status != "drained" { + t.Fatalf("unexpected unmount result: %#v", result) + } + drainIndex, umountIndex := -1, -1 + for i, call := range readFakeDrive9Calls(t, recordPath) { + switch { + case len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain": + drainIndex = i + case len(call.Args) >= 1 && call.Args[0] == "umount": + umountIndex = i + } + } + if drainIndex < 0 || umountIndex < 0 || drainIndex > umountIndex { + t.Fatalf("drain did not run before umount: drain=%d umount=%d", drainIndex, umountIndex) + } + if !strings.Contains(result.Human(), "drain") { + t.Fatalf("text output dropped the drain outcome:\n%s", result.Human()) + } +} + +// TestUnmountRefusesToDetachWithPendingWork blocks the unmount and keeps the +// locator so the write loss is explicit and retryable. +func TestUnmountRefusesToDetachWithPendingWork(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + + _, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if apperr.CodeFor(err) != "fs.drain_pending_work" { + t.Fatalf("unmount error = %v, want fs.drain_pending_work", err) + } + if _, _, locErr := mountlocator.Read(home, mountPath); locErr != nil { + t.Fatalf("blocked unmount must preserve the locator: %v", locErr) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) > 0 && call.Args[0] == "umount" { + t.Fatalf("unmount ran despite pending writes: %#v", call.Args) + } + } +} + +// TestUnmountForceSkipsDrainGate gives the operator an explicit way to accept +// the loss, and records that it was skipped. +func TestUnmountForceSkipsDrainGate(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, Force: true}) + if err != nil { + t.Fatalf("forced unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "skipped" || !strings.Contains(result.Drain.Message, "--force") { + t.Fatalf("forced unmount did not record the skipped gate: %#v", result.Drain) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("--force still drained: %#v", call.Args) + } + } +} + +// TestUnmountSkipsDrainForWebDAV keeps WebDAV unmount working: there is no +// control socket to drain. +func TestUnmountSkipsDrainForWebDAV(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "webdav", + }); err != nil { + t.Fatalf("webdav mount failed: %v", err) + } + locator, _, err := mountlocator.Read(home, mountPath) + if err != nil { + t.Fatalf("read locator: %v", err) + } + if locator.Driver != "webdav" { + t.Fatalf("locator driver = %q, want webdav", locator.Driver) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("webdav unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "skipped" || !strings.Contains(result.Drain.Message, "webdav") { + t.Fatalf("webdav unmount drain record = %#v", result.Drain) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("webdav unmount drained: %#v", call.Args) + } + } +} + +// TestUnmountSkipsDrainForVaultMount keeps `ti fs-vault unmount-vault` working: +// a vault mount has no file-system drain, and the engine rejects one. +func TestUnmountSkipsDrainForVaultMount(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + mountPath := filepath.Join(t.TempDir(), "vault") + companionHome, err := fscred.CompanionHome(home, "default", "tenant-1") + if err != nil { + t.Fatalf("resolve companion home: %v", err) + } + locator, err := mountlocator.New("default", "tenant-1", "aws-us-east-1", companionHome, mountPath, "vault") + if err != nil { + t.Fatalf("construct locator: %v", err) + } + if _, err := mountlocator.Write(home, locator.WithDriver("fuse")); err != nil { + t.Fatalf("write locator: %v", err) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("vault unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "skipped" || !strings.Contains(result.Drain.Message, "vault") { + t.Fatalf("vault drain record = %#v", result.Drain) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("vault unmount drained: %#v", call.Args) + } + } +} + +// TestUnmountSkipsDrainForLegacyLocator never gates a mount recorded before the +// driver was written, so an upgrade cannot block an existing unmount. +func TestUnmountSkipsDrainForLegacyLocator(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + profile := dataProfile() + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := mountlocator.Write(home, legacyTestLocator(t, home, mountPath)); err != nil { + t.Fatalf("write locator: %v", err) + } + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: profile, MountPath: mountPath}) + if err != nil { + t.Fatalf("legacy unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "skipped" || !strings.Contains(result.Drain.Message, "no driver") { + t.Fatalf("legacy locator drain record = %#v", result.Drain) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("legacy locator drained: %#v", call.Args) + } + } +} + +// TestUnmountIgnoreAbsentReportsDrainFailureStill lets a cleanup path continue, +// but the outcome has to say the writes may be lost. +func TestUnmountIgnoreAbsentReportsDrainFailureStill(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if err != nil { + t.Fatalf("ignore-absent unmount failed: %v", err) + } + if result.Drain == nil || result.Drain.Status != "failed" { + t.Fatalf("drain record = %#v, want failed", result.Drain) + } + found := false + for _, check := range result.Checks { + if check.Name == "drain" && check.Status == "failed" { + found = true + } + } + if !found { + t.Fatalf("unmount did not report the failed drain: %#v", result.Checks) + } +} + +// TestMountRecordsDriverInLocator records what the companion actually mounted, +// because the unmount gate depends on it. +func TestMountRecordsDriverInLocator(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + locator, _, err := mountlocator.Read(home, mountPath) + if err != nil { + t.Fatalf("read locator: %v", err) + } + if locator.Driver != "fuse" { + t.Fatalf("locator driver = %q, want fuse", locator.Driver) + } +} + +func legacyTestLocator(t *testing.T, home, mountPath string) mountlocator.Locator { + t.Helper() + companionHome, err := fscred.CompanionHome(home, "default", "tenant-1") + if err != nil { + t.Fatalf("resolve companion home: %v", err) + } + locator, err := mountlocator.New("default", "tenant-1", "aws-us-east-1", companionHome, mountPath, "fs") + if err != nil { + t.Fatalf("construct locator: %v", err) + } + return locator +} diff --git a/internal/fs/mount.go b/internal/fs/mount.go index c6735e7e..1cbf0695 100644 --- a/internal/fs/mount.go +++ b/internal/fs/mount.go @@ -20,6 +20,7 @@ import ( "github.com/tidbcloud/ti-cli/internal/dryrun" "github.com/tidbcloud/ti-cli/internal/fs/mountcontrol" "github.com/tidbcloud/ti-cli/internal/fs/mountdriver" + "github.com/tidbcloud/ti-cli/internal/fs/mountlocator" "github.com/tidbcloud/ti-cli/internal/fs/mountstate" ) @@ -127,6 +128,19 @@ type UnmountResult struct { PID int `json:"pid,omitempty"` StateFile string `json:"state_file,omitempty"` Checks []MountRuntimeCheck `json:"checks,omitempty"` + // Drain records the pre-unmount drain pass: drained, failed, or skipped + // with the reason. A failed drain is only reported when the caller asked to + // continue anyway (--force or --ignore-absent). + Drain *MountDrain `json:"drain,omitempty"` +} + +// MountDrain is the pre-unmount drain outcome. Response carries the engine's +// own drain report, including the pending counters, so a lost write is +// attributable instead of silent. +type MountDrain struct { + Status string `json:"status"` + Message string `json:"message,omitempty"` + Response *mountcontrol.DrainResponse `json:"response,omitempty"` } type DrainResult struct { @@ -324,7 +338,7 @@ func (s Service) DrainFileSystem(ctx context.Context, opts DrainFileSystemOption } func (s Service) DryRunDrainFileSystem(ctx context.Context, commandPath string, opts DrainFileSystemOptions) (dryrun.Result, error) { - profile, err := s.drive9MountLocatorProfile(opts.Profile, opts.MountPath) + locator, profile, err := s.mountLocatorProfile(opts.Profile, opts.MountPath) if err != nil { return dryrun.Result{}, err } @@ -339,11 +353,12 @@ func (s Service) DryRunDrainFileSystem(ctx context.Context, commandPath string, dryrun.Check{Name: "mount_locator", Status: "passed", Message: opts.MountPath}, dryrun.Check{Name: "file_system_id", Status: "passed", Message: profile.FSTenantID}, dryrun.Check{Name: "region", Status: "passed", Message: profile.FSPlacementRegionCode}, + dryrun.Check{Name: "drain_support", Status: "passed", Message: drainSupportDryRunMessage(locator)}, ), nil } func (s Service) DryRunUnmountFileSystem(ctx context.Context, commandPath string, opts UnmountFileSystemOptions) (dryrun.Result, error) { - profile, err := s.drive9MountLocatorProfile(opts.Profile, opts.MountPath) + locator, profile, err := s.mountLocatorProfile(opts.Profile, opts.MountPath) if err != nil { if opts.IgnoreAbsent && apperr.CodeFor(err) == "fs.mount_locator_not_found" { return dryrun.New( @@ -366,9 +381,27 @@ func (s Service) DryRunUnmountFileSystem(ctx context.Context, commandPath string dryrun.Check{Name: "mount_locator", Status: "passed", Message: opts.MountPath}, dryrun.Check{Name: "file_system_id", Status: "passed", Message: profile.FSTenantID}, dryrun.Check{Name: "region", Status: "passed", Message: profile.FSPlacementRegionCode}, + dryrun.Check{Name: "pre_unmount_drain", Status: "passed", Message: describeDryRunDrainGate(locator, opts)}, ), nil } +// drainSupportDryRunMessage says whether the mount can be drained at all. +func drainSupportDryRunMessage(locator mountlocator.Locator) string { + if drainable, reason := mountDrainGate(locator, false); !drainable { + return "not applicable: " + reason + } + return "FUSE mount with a drain control socket" +} + +// describeDryRunDrainGate reports what the pre-unmount drain would do without +// running it. +func describeDryRunDrainGate(locator mountlocator.Locator, opts UnmountFileSystemOptions) string { + if drainable, reason := mountDrainGate(locator, opts.Force); !drainable { + return "skipped: " + reason + } + return "drain pending writes before detaching the mount" +} + func (s Service) readDrainMountState(mountPathInput string) (mountstate.State, string, string, []MountRuntimeCheck, error) { homeDir, err := s.homeDir() if err != nil { @@ -774,21 +807,33 @@ func (r MountResult) Human() string { func (r UnmountResult) Human() string { var out strings.Builder writer := tabwriter.NewWriter(&out, 0, 0, 2, ' ', 0) - _, _ = fmt.Fprintln(writer, "STATUS\tMOUNT_PATH\tDRIVER\tPID") - _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%d\n", r.Status, r.MountPath, r.Driver, r.PID) + _, _ = fmt.Fprintln(writer, "STATUS\tMOUNT_PATH\tDRIVER\tDRAIN") + drain := "" + if r.Drain != nil { + drain = r.Drain.Status + } + _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%s\n", r.Status, r.MountPath, r.Driver, drain) _ = writer.Flush() + if r.Drain != nil { + if r.Drain.Message != "" { + _, _ = fmt.Fprintf(&out, "drain: %s\n", r.Drain.Message) + } + if r.Drain.Response != nil { + _, _ = fmt.Fprintf(&out, "pending: %s\n", drainPendingSummary(r.Drain.Response.Pending)) + } + } return strings.TrimRight(out.String(), "\n") } func (r DrainResult) Human() string { var out strings.Builder writer := tabwriter.NewWriter(&out, 0, 0, 2, ' ', 0) - _, _ = fmt.Fprintln(writer, "STATUS\tMOUNT_PATH\tDRIVER\tPID\tDURATION_MS") + _, _ = fmt.Fprintln(writer, "STATUS\tMOUNT_PATH\tDRIVER\tDURATION_MS") duration := int64(0) if r.Response != nil { duration = r.Response.DurationMS } - _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%d\t%d\n", r.Status, r.MountPath, r.Driver, r.PID, duration) + _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%d\n", r.Status, r.MountPath, r.Driver, duration) _ = writer.Flush() if r.Response != nil { p := r.Response.Pending diff --git a/internal/fs/mountlocator/locator.go b/internal/fs/mountlocator/locator.go index 6fe529d0..593c1a5a 100644 --- a/internal/fs/mountlocator/locator.go +++ b/internal/fs/mountlocator/locator.go @@ -21,10 +21,15 @@ type Locator struct { // was created. It is a stable, non-secret placement property that lets drain // and unmount rebuild the companion environment without fetching the region // catalog. Older locators may omit it. - ServiceAddress string `json:"service_address,omitempty"` - CompanionHome string `json:"companion_home"` - MountPath string `json:"mount_path"` - Kind string `json:"kind,omitempty"` + ServiceAddress string `json:"service_address,omitempty"` + CompanionHome string `json:"companion_home"` + MountPath string `json:"mount_path"` + Kind string `json:"kind,omitempty"` + // Driver is the mount driver the companion reported when the mount was + // created. Unmount uses it to decide whether the mount supports drain: + // WebDAV mounts have no control socket and no durability tiers. Older + // locators may omit it, and an omitted driver means "unknown, do not gate". + Driver string `json:"driver,omitempty"` FileSystemID string `json:"file_system_id,omitempty"` TokenID string `json:"token_id,omitempty"` TokenFingerprint string `json:"token_fingerprint,omitempty"` @@ -68,6 +73,12 @@ func New(profile, fileSystemName, regionCode, companionHome, mountPath, kind str } // WithServiceAddress records the resolved regional service address. +// WithDriver records the mount driver reported by the companion. +func (l Locator) WithDriver(driver string) Locator { + l.Driver = strings.TrimSpace(driver) + return l +} + func (l Locator) WithServiceAddress(address string) Locator { l.ServiceAddress = strings.TrimSpace(address) return l From 41dc94f5879fb265a6ea6701ee0210eb34409e91 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 19 Nov 2026 12:06:02 +0800 Subject: [PATCH 3/7] fix(fs): block pending drains under --ignore-absent and keep e2e host-neutral Addresses the review on #97. - `--ignore-absent` is the cleanup idempotency flag, so it must not accept undrained writes: `fs.drain_pending_work` now blocks the unmount even with it, the blocked-drain error says so, and only a drain that cannot reach the mount may continue (still reported as `drain.status: "failed"`). `--force` remains the only loss acceptance. - `ti fs drain-file-system` refuses mounts the engine cannot drain (WebDAV and any non-`fs` locator kind) with `fs.drain_not_applicable`, matching what the dry-run already reported instead of forwarding a request that fails raw. - The e2e mount expectations pin `--driver fuse` wherever a forwarded tier is asserted and branch the automatic-mode case on the host platform, so `make e2e` is green on macOS/Windows too. Both fake companions now mirror drive9's `--mode auto` resolution so the fixtures cannot disagree with the companion. - Consolidate the automatic-mode rule in `companionMountMode` with the reason it cannot use the prerequisite-aware `mountdriver.Resolve`, and reconcile the reported `durability` with the driver the companion reported. - Nits: reject an explicitly empty `--durability`, share one pending-counter formatter between the drain and unmount renderers, print the effective tier in the mount text output, use `drainTimeout`, drop the unused `drive9MountLocatorProfile` wrapper, and restore the locator comments. The published English pages are tracked in issue #98, referenced from both specs, because the documentation submodule is not checked out here. --- AGENTS.md | 31 +++++--- README.md | 2 +- docs/spec/done/0038-fs-mount-durability.md | 26 ++++-- docs/spec/done/0039-fs-unmount-drain-gate.md | 35 +++++++-- e2e/cli_test.go | 75 ++++++++++++++++-- e2e/testdata/fake-drive9.go | 14 +++- internal/fs/drive9_companion.go | 18 +++-- internal/fs/drive9_fake_companion_test.go | 23 ++++-- internal/fs/drive9_mount_audit.go | 10 ++- internal/fs/drive9_mount_audit_test.go | 11 ++- internal/fs/drive9_mount_drain.go | 25 ++++-- internal/fs/drive9_mount_drain_test.go | 83 +++++++++++++++++++- internal/fs/mount.go | 17 +--- internal/fs/mount_durability.go | 28 +++++-- internal/fs/mount_durability_test.go | 13 ++- internal/fs/mountlocator/locator.go | 2 +- 16 files changed, 327 insertions(+), 86 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 569c93d6..9b717872 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1154,17 +1154,26 @@ Successful unmount removes the locator; failed unmount preserves it for retry. `ti fs unmount-file-system` drains before it detaches a FUSE mount and must never report success while write-back work is still local. Parse the companion's `mount drain --json` report instead of discarding it: `ti fs drain-file-system` -returns it as `response`, prints the pending counters in text output, and fails -with `fs.drain_pending_work` when work remains. The unmount records the driver -the companion actually mounted in the non-secret mount locator, gates only FUSE -mounts on the drain (and never a `vault` locator: the engine drains only -file-system mounts), blocks with the locator preserved while work is pending, -and treats `--force` as the explicit "accept the loss" escape hatch (reported as -`drain.status: "skipped"`). `--ignore-absent` may continue past a drain failure -but must still report `drain.status: "failed"`. A locator written before the -driver was recorded is never gated, so an upgrade cannot block an existing -unmount. Do not drop these outcomes from the result, and do not reimplement the -drain in ti: the mount control socket belongs to the companion. +returns it as `response`, prints the pending counters in text output with one +shared formatter, fails with `fs.drain_pending_work` when work remains, and +refuses mounts the dry-run calls "not applicable" (WebDAV, and any non-`fs` +locator kind such as `vault`) with `fs.drain_not_applicable` instead of a raw +companion error. The unmount records the driver the companion actually mounted +in the non-secret mount locator, gates only FUSE file-system mounts on the +drain, blocks with the locator preserved while work is pending, and treats +`--force` as the only "accept the loss" path (reported as `drain.status: +"skipped"`). `--ignore-absent` is a cleanup idempotency flag: it may continue +past a drain that cannot reach the mount, but it must never swallow +`fs.drain_pending_work`, and the outcome must still report `drain.status: +"failed"`. A locator written before the driver was recorded is never gated, so +an upgrade cannot block an existing unmount. The reported `durability` must +match the driver the companion reported, and `--durability` must reject an +explicitly empty value instead of treating it as the default. Do not drop these +outcomes from the result, and do not reimplement the drain in ti: the mount +control socket belongs to the companion. The companion's `--mode auto` rule is +mirrored in exactly one place (`companionMountMode`), because +`mountdriver.Resolve` is prerequisite-aware and would disagree with the +companion; see `docs/spec/done/0038-fs-mount-durability.md`. `ti fs mount-file-system` forwards the selected write-durability tier to the companion as `mount --durability`. The flag accepts `auto`, `interactive`, diff --git a/README.md b/README.md index 1e4cfbbb..4294ef5c 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ ti fs drain-file-system --mount-path ~/my-workspace ti fs unmount-file-system --mount-path ~/my-workspace ``` -`ti fs drain-file-system` reports the companion's drain result, including the pending counters (`open_handles`, `commit_queue_*`, `uploader_*`), so a caller can tell "fully uploaded" from "the background upload never finished". `ti fs unmount-file-system` runs that drain itself before detaching a FUSE mount and fails with `fs.drain_pending_work` while writes are still pending, keeping the mount locator so the command can be retried; use `--force` to accept the undrained writes anyway, and note that the result then records `drain.status: "skipped"`. `--ignore-absent` continues past a drain that fails because the mount is already gone but still reports `drain.status: "failed"`. WebDAV mounts keep their native write behavior and are not gated, and `ti fs-vault unmount-vault` is not gated either because the engine drains only file-system mounts. +`ti fs drain-file-system` reports the companion's drain result, including the pending counters (`open_handles`, `commit_queue_*`, `uploader_*`), so a caller can tell "fully uploaded" from "the background upload never finished"; it also refuses WebDAV and vault mounts, which have no drain, instead of failing with a raw companion error. `ti fs unmount-file-system` runs that drain itself before detaching a FUSE file-system mount and fails with `fs.drain_pending_work` while writes are still pending, keeping the mount locator so the command can be retried. `--force` is the only way to accept the undrained writes (the result then records `drain.status: "skipped"`); `--ignore-absent` stays a cleanup idempotency flag and continues only past a drain that cannot reach the mount (`drain.status: "failed"`), never past pending work. WebDAV mounts keep their native write behavior and are not gated, and `ti fs-vault unmount-vault` is not gated either because the engine drains only file-system mounts. Run a command against a remote Filesystem (Runtime is available only in AWS `us-east-1` and `us-west-2`): diff --git a/docs/spec/done/0038-fs-mount-durability.md b/docs/spec/done/0038-fs-mount-durability.md index f25c86fc..c5e8a2f2 100644 --- a/docs/spec/done/0038-fs-mount-durability.md +++ b/docs/spec/done/0038-fs-mount-durability.md @@ -48,7 +48,9 @@ instead of assuming: - `--durability `, default `interactive`, one of `auto`, `interactive`, `fsync`, `close-sync`, `write-sync`. The value is trimmed and case-folded before validation; anything else fails with `fs.invalid_mount_durability` and - exit code 2 before the companion runs. + exit code 2 before the companion runs. An explicitly empty value fails with + `fs.missing_mount_durability`, matching the CLI's rule for `--region` and + `--profile`, instead of being silently treated as the default. `--write-back-cache` stays as a legacy alias so existing scripts keep working: @@ -61,7 +63,12 @@ instead of assuming: ti forwards the tier as `mount --durability ` only when the mount resolves to FUSE, mirroring the companion's own `--mode auto` rule (WebDAV on macOS and -Windows, FUSE elsewhere): +Windows, FUSE elsewhere). That rule lives in one place, +`internal/fs/mount_durability.go`'s `companionMountMode`. It deliberately does +not use `mountdriver.Resolve`, which is prerequisite-aware (macFUSE on darwin) +and would forward a tier to a WebDAV mount the companion chose. The mount result +reconciles the reported `durability` with the driver the companion reports, so +the result never claims a tier that is not in effect: - Explicit `--driver fuse`, and `--driver auto` on a platform whose automatic choice is FUSE, forward the tier. An ordinary mount therefore keeps the @@ -103,15 +110,18 @@ Out of scope: ## Verification - Unit tests pin the tier table, the legacy alias mapping, the conflict error, - the WebDAV and checkpoint rejections, the local mirror of the companion's - `--mode auto` rule, the forwarded argv for every tier, the interactive - default, and the dry-run body. + the explicit-empty rejection, the WebDAV and checkpoint rejections, the local + mirror of the companion's `--mode auto` rule, the forwarded argv for every + tier, the interactive default, and the dry-run body. - `make e2e` covers the default and a strict tier end to end through the fake - companion, plus the invalid-value and WebDAV rejections. + companion, plus the invalid-value and WebDAV rejections. Mounts that assert a + forwarded tier pin `--driver fuse`, and the automatic-mode case branches on + the host platform, so the gate is green on macOS and Windows too. - `make live-e2e-fs` and `make live-e2e-fs-git` mount with an explicit tier (`close-sync`) and assert the reported value, and the live WebDAV mount asserts `auto`. The English PingCAP documentation pages under `docs/pingcap-docs/docs/` are a -submodule that was not checked out while this spec was implemented. The mount -page must gain the `--durability` section when that submodule is available. +submodule that was not checked out while this spec was implemented. The +published mount page must gain the `--durability` section; the tracked follow-up +is issue #98 in this repository and must land before the next release cut. diff --git a/docs/spec/done/0039-fs-unmount-drain-gate.md b/docs/spec/done/0039-fs-unmount-drain-gate.md index e20c8630..6e4812ca 100644 --- a/docs/spec/done/0039-fs-unmount-drain-gate.md +++ b/docs/spec/done/0039-fs-unmount-drain-gate.md @@ -81,16 +81,30 @@ A pending unmount fails with `fs.drain_pending_work`, names the engine's - A failed or pending drain blocks the unmount and keeps the locator, so the caller can close the writers and retry. - `--force` skips the gate. The result records `drain.status: "skipped"` with - the reason, so a forced unmount is still attributable. -- `--ignore-absent` keeps its cleanup meaning: a drain that fails because the - mount is already gone does not block, but the result records - `drain.status: "failed"` and a `drain` runtime check instead of pretending the - drain succeeded. + the reason, so a forced unmount is still attributable. It is the only way to + accept undrained writes. +- `--ignore-absent` keeps its cleanup idempotency meaning, not loss acceptance: + a drain that fails for a reason other than pending work (typically a mount + that is already gone) does not block, and the result records + `drain.status: "failed"` plus a `drain` runtime check. A + `fs.drain_pending_work` failure blocks even under `--ignore-absent`, and the + blocked-drain error names both flags so the remedy is unambiguous. Cleanup + scripts pass `--ignore-absent` routinely, so letting it bypass the pending + check would recreate the silent write loss this spec closes. - The result reports the recorded driver and the drain record. Text output adds the `DRAIN` column and the pending counters. - `--dry-run` reports the gate in a `pre_unmount_drain` check without running the drain. +### Drain applicability + +`ti fs drain-file-system` runs only for mounts the engine can drain. A non-`fs` +locator kind (for example a `vault` mount) and a WebDAV driver fail with +`fs.drain_not_applicable` and an explanation, matching what the drain dry-run +already reported, instead of forwarding a request the companion rejects with a +raw error. A locator that records no driver stays drainable: the command itself +asks for a drain, and only the unmount gate needs the stricter driver check. + The `--force` flag previously promised to "kill the mount process" but no code read it; it now means "accept the undrained writes". @@ -131,7 +145,14 @@ remains the client-side mitigation. result carries `response.pending`. - `make live-e2e-fs` asserts the real drain report fields on a live FUSE mount; the live unmount now drains first. +- The e2e mount expectations pin `--driver fuse` wherever a forwarded tier is + asserted and branch the automatic-mode case on the host platform, so the + mandatory `make e2e` gate stays green on macOS and Windows as well as Linux. +- The `--ignore-absent` cases are covered both ways: pending work still blocks + (unit and e2e), and a drain that cannot reach the mount continues with + `drain.status: "failed"`. The English PingCAP pages under `docs/pingcap-docs/docs/` are a submodule that -was not checked out while this spec was implemented; the drain/unmount page must -document the gate and `--force` when that submodule is available. +was not checked out while this spec was implemented. The published mount and +unmount pages therefore still describe the pre-change CLI; the tracked follow-up +is issue #98 in this repository and must land before the next release cut. diff --git a/e2e/cli_test.go b/e2e/cli_test.go index 609bcbf5..b3d6f5f2 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -1221,17 +1221,49 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { ) mixed.wantExitCode(0) + // FUSE is pinned wherever the test asserts a forwarded tier, so the + // expectations do not depend on the host's automatic mode (auto is WebDAV on + // macOS/Windows and FUSE elsewhere). The fake companion never touches real + // FUSE, so this costs no coverage. mountPath := filepath.Join(home, "mount") mount := runTIWithInput(t, bin, "", authEnv, "fs", "mount-file-system", "--mount-path", mountPath, + "--driver", "fuse", ) mount.wantExitCode(0) mount.wantStdoutContains(`"durability": "interactive"`) + mount.wantStdoutContains(`"driver": "fuse"`) + mountText := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", filepath.Join(home, "mount-text"), + "--driver", "fuse", + "--output", "text", + ) + mountText.wantExitCode(0) + mountText.wantStdoutContains("Durability: interactive") + autoMountPath := filepath.Join(home, "mount-auto") + autoMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", autoMountPath, + ) + autoMount.wantExitCode(0) + // auto resolves to WebDAV on macOS/Windows and FUSE elsewhere. + wantAutoDurability := `"durability": "interactive"` + wantAutoDriver := `"driver": "fuse"` + wantAutoForwarded := "interactive" + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + wantAutoDurability = `"durability": "auto"` + wantAutoDriver = `"driver": "webdav"` + wantAutoForwarded = "" + } + autoMount.wantStdoutContains(wantAutoDurability) + autoMount.wantStdoutContains(wantAutoDriver) strictMountPath := filepath.Join(home, "mount-close-sync") strictMount := runTIWithInput(t, bin, "", authEnv, "fs", "mount-file-system", "--mount-path", strictMountPath, + "--driver", "fuse", "--durability", "close-sync", ) strictMount.wantExitCode(0) @@ -1259,11 +1291,13 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { ) webdavMount.wantExitCode(0) webdavMount.wantStdoutContains(`"durability": "auto"`) - webdavUnmount := runTIWithInput(t, bin, "", baseEnv, - "fs", "unmount-file-system", - "--mount-path", webdavMountPath, - ) - webdavUnmount.wantExitCode(0) + for _, path := range []string{webdavMountPath, filepath.Join(home, "mount-text"), autoMountPath} { + extraUnmount := runTIWithInput(t, bin, "", baseEnv, + "fs", "unmount-file-system", + "--mount-path", path, + ) + extraUnmount.wantExitCode(0) + } drain := runTIWithInput(t, bin, "", baseEnv, "fs", "drain-file-system", "--mount-path", mountPath, @@ -1279,8 +1313,7 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { "--output", "text", ) drainText.wantExitCode(0) - drainText.wantStdoutContains("pending:") - drainText.wantStdoutContains("open_handles=") + drainText.wantStdoutContains("pending: none") unmount := runTIWithInput(t, bin, "", baseEnv, "fs", "unmount-file-system", "--mount-path", mountPath, @@ -1306,6 +1339,15 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { pendingUnmount.wantExitCode(1) pendingUnmount.wantStderrContains("error_kind=pending_work_remaining") pendingUnmount.wantStderrContains("unmount with --force") + // --ignore-absent is the cleanup idempotency flag, not loss acceptance: it + // must not swallow pending writes either. + pendingIgnoreAbsent := runTIWithInput(t, bin, "", pendingDrainEnv, + "fs", "unmount-file-system", + "--mount-path", strictMountPath, + "--ignore-absent", + ) + pendingIgnoreAbsent.wantExitCode(1) + pendingIgnoreAbsent.wantStderrContains("--ignore-absent does not skip this check") forcedUnmount := runTIWithInput(t, bin, "", pendingDrainEnv, "fs", "unmount-file-system", "--mount-path", strictMountPath, @@ -1315,6 +1357,24 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { forcedUnmount.wantStdoutContains(`"status": "skipped"`) forcedUnmount.wantStdoutContains("--force skips the pre-unmount drain") + // A drain that cannot reach the mount is not pending work, so the cleanup + // flag may continue, but the outcome still reports the failed drain. + goneMountPath := filepath.Join(home, "mount-gone") + goneMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", goneMountPath, + "--driver", "fuse", + ) + goneMount.wantExitCode(0) + goneEnv := append(append([]string{}, baseEnv...), "FAKE_DRIVE9_DRAIN_FAIL=1") + goneUnmount := runTIWithInput(t, bin, "", goneEnv, + "fs", "unmount-file-system", + "--mount-path", goneMountPath, + "--ignore-absent", + ) + goneUnmount.wantExitCode(0) + goneUnmount.wantStdoutContains(`"status": "failed"`) + for _, path := range []string{ filepath.Join(home, ".tdc", "config"), filepath.Join(home, ".tdc", "credentials"), @@ -1351,6 +1411,7 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { assertFakeDrive9FlagValue(t, calls, "mount", mountPath, "--durability", "interactive") assertFakeDrive9FlagValue(t, calls, "mount", strictMountPath, "--durability", "close-sync") assertFakeDrive9FlagValue(t, calls, "mount", webdavMountPath, "--durability", "") + assertFakeDrive9FlagValue(t, calls, "mount", autoMountPath, "--durability", wantAutoForwarded) // A FUSE unmount drains before detaching; a WebDAV unmount has no control // socket and must not try. for _, call := range calls { diff --git a/e2e/testdata/fake-drive9.go b/e2e/testdata/fake-drive9.go index 9990c3d8..dd7db61e 100644 --- a/e2e/testdata/fake-drive9.go +++ b/e2e/testdata/fake-drive9.go @@ -7,6 +7,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "sort" "strings" "syscall" @@ -233,6 +234,10 @@ func main() { return } if hasPrefix(args, "mount", "drain") { + if os.Getenv("FAKE_DRIVE9_DRAIN_FAIL") == "1" { + fmt.Fprintln(os.Stderr, "drive9 mount drain: read mount state: no such file or directory") + os.Exit(1) + } ok := os.Getenv("FAKE_DRIVE9_DRAIN_PENDING") != "1" report := map[string]any{ "ok": ok, "mount_point": args[len(args)-1], "duration_ms": 5, @@ -261,10 +266,15 @@ func main() { } func mountMode(args []string) string { - if value := flagValue(args, "--mode"); value != "" { + if value := flagValue(args, "--mode"); value != "" && value != "auto" { return value } - return "webdav" + // Mirror drive9's ResolveMountMode: auto is webdav on macOS/Windows and + // fuse everywhere else. + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return "webdav" + } + return "fuse" } func hasPrefix(args []string, want ...string) bool { diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index bc58b530..2d7ae8a8 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -1365,6 +1365,9 @@ func (s Service) drive9DrainFileSystem(ctx context.Context, opts DrainFileSystem } // The companion's report is the only place the undrained work is counted, // so keep it in the result instead of discarding it. + if drainable, reason := drainApplicability(locator); !drainable { + return DrainResult{}, apperr.New("fs.drain_not_applicable", "usage", 2, fmt.Sprintf("ti fs drain %s: %s", opts.MountPath, reason)) + } drain, err := s.drainMount(ctx, profile, opts.MountPath, opts.Timeout) if err != nil { return DrainResult{}, err @@ -1386,8 +1389,14 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy // --ignore-absent tolerates a drain that fails because the mount is already // gone. drain, drainErr := s.drainBeforeUnmount(ctx, profile, locator, opts) - if drainErr != nil && !opts.IgnoreAbsent { - return UnmountResult{}, drainErr + if drainErr != nil { + // --ignore-absent is the idempotency flag cleanup scripts pass, not a + // licence to drop writes: only a drain that failed for another reason + // (typically a mount that is already gone) may be tolerated. Accepting + // the loss stays --force's job alone, so a pending drain always blocks. + if apperr.CodeFor(drainErr) == "fs.drain_pending_work" || !opts.IgnoreAbsent { + return UnmountResult{}, drainErr + } } args := []string{"umount"} if opts.Timeout > 0 { @@ -1456,11 +1465,6 @@ func (s Service) writeDrive9MountLocator(profile *config.Profile, mountPath, kin return nil } -func (s Service) drive9MountLocatorProfile(base *config.Profile, mountPath string) (*config.Profile, error) { - _, profile, err := s.mountLocatorProfile(base, mountPath) - return profile, err -} - // mountLocatorProfile resolves the ti-owned mount locator for a mount path and // rebuilds the companion profile from it. Callers that also need locator facts // (the recorded driver, for example) use this instead of reading it twice. diff --git a/internal/fs/drive9_fake_companion_test.go b/internal/fs/drive9_fake_companion_test.go index d7692c31..de36e9f6 100644 --- a/internal/fs/drive9_fake_companion_test.go +++ b/internal/fs/drive9_fake_companion_test.go @@ -28,6 +28,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strconv" "time" ) @@ -99,6 +100,10 @@ func main() { "server": os.Getenv("DRIVE9_SERVER"), }) case len(args) >= 2 && args[0] == "mount" && args[1] == "drain": + if os.Getenv("TI_FAKE_DRIVE9_DRAIN_FAIL") == "1" { + fmt.Fprintln(os.Stderr, "drive9 mount drain: read mount state: no such file or directory") + os.Exit(1) + } pending := os.Getenv("TI_FAKE_DRIVE9_DRAIN_PENDING") == "1" report := map[string]any{ "ok": !pending, "mount_point": args[len(args)-1], "duration_ms": 7, @@ -120,11 +125,7 @@ func main() { } case args[0] == "mount" && (len(args) == 1 || args[1] != "drain"): fmt.Fprintln(os.Stderr, "component: drive9 mount") - if flagValue(args, "--mode") == "fuse" { - fmt.Fprintln(os.Stderr, "drive9: mount mode: fuse") - } else { - fmt.Fprintln(os.Stderr, "drive9: mount mode: webdav") - } + fmt.Fprintln(os.Stderr, "drive9: mount mode: "+fakeMountMode(args)) if os.Getenv("TI_FAKE_DRIVE9_MOUNT_FAIL") == "1" { fmt.Fprintln(os.Stderr, "mount: drive9 mount: background mount exited before becoming ready") os.Exit(1) @@ -283,6 +284,18 @@ func record(args []string) { _ = json.NewEncoder(f).Encode(callRecord{Args: args, Env: env}) } +// fakeMountMode mirrors drive9's ResolveMountMode: an explicit mode wins, and +// auto means webdav on macOS/Windows and fuse elsewhere. +func fakeMountMode(args []string) string { + if mode := flagValue(args, "--mode"); mode != "" && mode != "auto" { + return mode + } + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return "webdav" + } + return "fuse" +} + func flagValue(args []string, name string) string { for i := 0; i+1 < len(args); i++ { if args[i] == name { diff --git a/internal/fs/drive9_mount_audit.go b/internal/fs/drive9_mount_audit.go index 3f6381da..6b2b7745 100644 --- a/internal/fs/drive9_mount_audit.go +++ b/internal/fs/drive9_mount_audit.go @@ -168,7 +168,15 @@ func (s Service) drive9MountFileSystem(ctx context.Context, opts MountFileSystem if driver == "" { driver = "auto" } - return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: opts.Profile.FSResourceName, MountPath: opts.MountPath, RemotePath: remotePath, Driver: driver, Endpoint: &endpoint, MountProfile: opts.MountProfile, LocalRoot: auditRoot, PackPaths: opts.PackPaths, Durability: opts.Durability, LayerRef: opts.LayerRef, CheckpointID: opts.CheckpointID, ReadOnly: opts.ReadOnly}, nil + // Report the tier that is actually in effect. ti forwards --durability only + // for mounts its companion-mode mirror calls FUSE; if the companion still + // mounted WebDAV, no tier applies and reporting "interactive" would claim a + // guarantee the mount does not have. + durability := opts.Durability + if driver == "webdav" { + durability = MountDurabilityAuto + } + return MountResult{Status: "mounted", Profile: profileName(opts.Profile), FileSystemName: opts.Profile.FSResourceName, MountPath: opts.MountPath, RemotePath: remotePath, Driver: driver, Endpoint: &endpoint, MountProfile: opts.MountProfile, LocalRoot: auditRoot, PackPaths: opts.PackPaths, Durability: durability, LayerRef: opts.LayerRef, CheckpointID: opts.CheckpointID, ReadOnly: opts.ReadOnly}, nil } // stopMountAfterUnsafeAudit tears down a mount whose overlay failed the audit. It diff --git a/internal/fs/drive9_mount_audit_test.go b/internal/fs/drive9_mount_audit_test.go index bb760e05..8f957dcf 100644 --- a/internal/fs/drive9_mount_audit_test.go +++ b/internal/fs/drive9_mount_audit_test.go @@ -26,11 +26,14 @@ func TestDrive9MountLocatorRoutesDrainAndUnmountWithoutCredentials(t *testing.T) mountPath := filepath.Join(t.TempDir(), "workspace") profile := dataProfile() + // An explicit FUSE driver keeps this routing test independent of the host's + // automatic mode: on macOS/Windows `auto` is WebDAV, which has no drain. if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ Profile: profile, FileSystemName: "workspace", MountPath: mountPath, RemotePath: "/", + Driver: "fuse", }); err != nil { t.Fatalf("MountFileSystem failed: %v", err) } @@ -707,7 +710,13 @@ func TestDrive9MountSuppressesCompanionSuccessChatter(t *testing.T) { if stdout.Len() != 0 || stderr.Len() != 0 { t.Fatalf("companion mount output leaked to ti output: stdout=%q stderr=%q", stdout.String(), stderr.String()) } - if result.Driver != "webdav" || result.Status != "mounted" { + // auto is webdav on macOS/Windows and fuse elsewhere; the point is that the + // driver comes from the companion's own report, not from opts.Driver. + wantDriver := "fuse" + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + wantDriver = "webdav" + } + if result.Driver != wantDriver || result.Status != "mounted" { t.Fatalf("mount result did not retain captured companion state: %#v", result) } } diff --git a/internal/fs/drive9_mount_drain.go b/internal/fs/drive9_mount_drain.go index 1b48c249..f320614f 100644 --- a/internal/fs/drive9_mount_drain.go +++ b/internal/fs/drive9_mount_drain.go @@ -110,7 +110,7 @@ func mountDriverSupportsDrain(driver string) bool { case "fuse": return true default: - return effectiveMountMode(driver) == "fuse" + return companionMountMode(driver) == "fuse" } } @@ -133,6 +133,21 @@ func mountDrainGate(locator mountlocator.Locator, force bool) (bool, string) { return true, "" } +// drainApplicability reports whether `ti fs drain-file-system` can run at all. +// It rejects the mounts whose dry-run already says "not applicable" so the +// command fails with the same explanation instead of a raw companion error. An +// unrecorded driver stays drainable: the command itself asks for a drain, and +// only the unmount gate needs the stricter driver check. +func drainApplicability(locator mountlocator.Locator) (bool, string) { + if kind := strings.TrimSpace(locator.Kind); kind != "" && kind != mountKindFileSystem { + return false, fmt.Sprintf("%s mounts have no drain; the engine drains only file-system mounts", kind) + } + if strings.TrimSpace(locator.Driver) == "webdav" { + return false, "webdav mounts have no mount control socket to drain; the companion flushes them when it unmounts" + } + return true, "" +} + // mountDrainSkipReason explains why a driver cannot be drained. func mountDrainSkipReason(driver string) string { if strings.TrimSpace(driver) == "" { @@ -160,7 +175,7 @@ func (s Service) drainMount(ctx context.Context, profile *config.Profile, mountP } if runErr != nil || !response.OK { record := MountDrain{Status: "failed", Message: describeDrainFailure(response), Response: response} - return record, apperr.New(drainErrorCode(response), "runtime", 1, fmt.Sprintf("drain %s did not finish: %s; close the writers or let the background upload finish, then retry ti fs drain-file-system --mount-path %q, or unmount with --force to accept losing the undrained writes", mountPath, describeDrainFailure(response), mountPath)) + return record, apperr.New(drainErrorCode(response), "runtime", 1, fmt.Sprintf("drain %s did not finish: %s; close the writers or let the background upload finish, then retry ti fs drain-file-system --mount-path %q, or unmount with --force to accept losing the undrained writes (--ignore-absent does not skip this check)", mountPath, describeDrainFailure(response), mountPath)) } return MountDrain{Status: "drained", Response: response}, nil } @@ -173,9 +188,5 @@ func (s Service) drainBeforeUnmount(ctx context.Context, profile *config.Profile if drainable, reason := mountDrainGate(locator, opts.Force); !drainable { return MountDrain{Status: "skipped", Message: reason}, nil } - timeout := opts.Timeout - if timeout <= 0 { - timeout = mountcontrol.DefaultDrainTimeout - } - return s.drainMount(ctx, profile, opts.MountPath, timeout) + return s.drainMount(ctx, profile, opts.MountPath, drainTimeout(opts.Timeout)) } diff --git a/internal/fs/drive9_mount_drain_test.go b/internal/fs/drive9_mount_drain_test.go index 59ba862f..9a2f3c44 100644 --- a/internal/fs/drive9_mount_drain_test.go +++ b/internal/fs/drive9_mount_drain_test.go @@ -2,6 +2,7 @@ package fs import ( "context" + "os" "path/filepath" "strings" "testing" @@ -40,9 +41,13 @@ func TestDrive9DrainReportsPendingCounters(t *testing.T) { t.Fatalf("drain duration = %d, want the companion's report", result.Response.DurationMS) } human := result.Human() - if !strings.Contains(human, "pending:") || !strings.Contains(human, "open_handles=0") { + if !strings.Contains(human, "pending: none") { t.Fatalf("text output dropped the pending counters:\n%s", human) } + mountHuman := MountResult{Status: "mounted", MountPath: "/mnt", Driver: "fuse", Durability: MountDurabilityCloseSync}.Human() + if !strings.Contains(mountHuman, "Durability: close-sync") { + t.Fatalf("mount text output dropped the effective tier:\n%s", mountHuman) + } } // TestDrive9DrainFailureNamesPendingWork turns an undrained mount into an @@ -269,9 +274,10 @@ func TestUnmountSkipsDrainForLegacyLocator(t *testing.T) { } } -// TestUnmountIgnoreAbsentReportsDrainFailureStill lets a cleanup path continue, -// but the outcome has to say the writes may be lost. -func TestUnmountIgnoreAbsentReportsDrainFailureStill(t *testing.T) { +// TestUnmountIgnoreAbsentStillBlocksPendingWork keeps --ignore-absent from +// becoming a silent loss-acceptance flag: it is the idempotency flag cleanup +// scripts pass, so pending writes must still block the detach. +func TestUnmountIgnoreAbsentStillBlocksPendingWork(t *testing.T) { home := t.TempDir() companion, recordPath := buildFakeDrive9(t) t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) @@ -285,6 +291,39 @@ func TestUnmountIgnoreAbsentReportsDrainFailureStill(t *testing.T) { } t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING", "1") + _, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if apperr.CodeFor(err) != "fs.drain_pending_work" { + t.Fatalf("ignore-absent unmount error = %v, want fs.drain_pending_work", err) + } + if !strings.Contains(apperr.MessageFor(err), "--force") || !strings.Contains(apperr.MessageFor(err), "--ignore-absent does not skip") { + t.Fatalf("blocked drain error must name both flags: %v", apperr.MessageFor(err)) + } + if _, _, locErr := mountlocator.Read(home, mountPath); locErr != nil { + t.Fatalf("blocked unmount must preserve the locator: %v", locErr) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) > 0 && call.Args[0] == "umount" { + t.Fatalf("unmount ran despite pending writes: %#v", call.Args) + } + } +} + +// TestUnmountIgnoreAbsentToleratesGoneMount keeps cleanup idempotent: a drain +// that cannot reach the mount (not pending work) may continue. +func TestUnmountIgnoreAbsentToleratesGoneMount(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_FAIL", "1") + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) if err != nil { t.Fatalf("ignore-absent unmount failed: %v", err) @@ -303,6 +342,29 @@ func TestUnmountIgnoreAbsentReportsDrainFailureStill(t *testing.T) { } } +// TestDrainRejectsNonFileSystemMount keeps the drain command from reaching the +// companion with a mount the engine cannot drain. +func TestDrainRejectsNonFileSystemMount(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + mountPath := filepath.Join(t.TempDir(), "vault") + if _, err := mountlocator.Write(home, legacyTestLocatorWithKind(t, home, mountPath, "vault", "fuse")); err != nil { + t.Fatalf("write locator: %v", err) + } + if _, err := service.DrainFileSystem(context.Background(), DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}); apperr.CodeFor(err) != "fs.drain_not_applicable" { + t.Fatalf("vault drain error = %v, want fs.drain_not_applicable", err) + } + if _, statErr := os.Stat(recordPath); statErr == nil { + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) >= 2 && call.Args[0] == "mount" && call.Args[1] == "drain" { + t.Fatalf("vault drain reached the companion: %#v", call.Args) + } + } + } +} + // TestMountRecordsDriverInLocator records what the companion actually mounted, // because the unmount gate depends on it. func TestMountRecordsDriverInLocator(t *testing.T) { @@ -326,6 +388,19 @@ func TestMountRecordsDriverInLocator(t *testing.T) { } } +func legacyTestLocatorWithKind(t *testing.T, home, mountPath, kind, driver string) mountlocator.Locator { + t.Helper() + companionHome, err := fscred.CompanionHome(home, "default", "tenant-1") + if err != nil { + t.Fatalf("resolve companion home: %v", err) + } + locator, err := mountlocator.New("default", "tenant-1", "aws-us-east-1", companionHome, mountPath, kind) + if err != nil { + t.Fatalf("construct locator: %v", err) + } + return locator.WithDriver(driver) +} + func legacyTestLocator(t *testing.T, home, mountPath string) mountlocator.Locator { t.Helper() companionHome, err := fscred.CompanionHome(home, "default", "tenant-1") diff --git a/internal/fs/mount.go b/internal/fs/mount.go index 1cbf0695..4d4fb459 100644 --- a/internal/fs/mount.go +++ b/internal/fs/mount.go @@ -780,6 +780,9 @@ func (r MountResult) Human() string { "Remote path: " + r.RemotePath, "Driver: " + r.Driver, } + if r.Durability != "" { + lines = append(lines, "Durability: "+r.Durability) + } if r.LayerRef != "" { lines = append(lines, "Layer: "+r.LayerRef) } @@ -836,19 +839,7 @@ func (r DrainResult) Human() string { _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%d\n", r.Status, r.MountPath, r.Driver, duration) _ = writer.Flush() if r.Response != nil { - p := r.Response.Pending - _, _ = fmt.Fprintf(&out, "pending: open_handles=%d dirty_handles=%d commit_queue=%d commit_bytes=%d commit_in_flight=%d commit_delayed=%d commit_conflicts=%d uploader_queued=%d uploader_in_flight=%d uploader_cached=%d\n", - p.OpenHandles, - p.DirtyHandles, - p.CommitQueuePending, - p.CommitQueueBytes, - p.CommitQueueInFlight, - p.CommitQueueDelayed, - p.CommitQueueConflicts, - p.UploaderQueued, - p.UploaderInFlight, - p.UploaderCached, - ) + _, _ = fmt.Fprintf(&out, "pending: %s\n", drainPendingSummary(r.Response.Pending)) } return strings.TrimRight(out.String(), "\n") } diff --git a/internal/fs/mount_durability.go b/internal/fs/mount_durability.go index b1465ff1..0bc7f3e5 100644 --- a/internal/fs/mount_durability.go +++ b/internal/fs/mount_durability.go @@ -78,6 +78,13 @@ func mountDurabilityUsesWriteBack(value string) bool { func normalizeMountDurability(opts MountFileSystemOptions) (MountFileSystemOptions, error) { value := strings.ToLower(strings.TrimSpace(opts.Durability)) if value == "" { + // An explicit empty value is a mistake, not a request for the default: + // the rest of the CLI rejects explicit empty --region/--profile values + // for the same reason, and treating "" as both "unset" and "explicit" + // makes --durability "" --write-back-cache=false look like a conflict. + if opts.DurabilitySet { + return MountFileSystemOptions{}, apperr.New("fs.missing_mount_durability", "usage", 2, fmt.Sprintf("--durability must not be empty; supported values: %s", MountDurabilityHelp())) + } value = DefaultMountDurability } if !validMountDurability(value) { @@ -106,7 +113,7 @@ func normalizeMountDurability(opts MountFileSystemOptions) (MountFileSystemOptio // usage error instead of a silent downgrade, so a caller that asked for // close-sync or write-sync never believes it got it. func rejectUnsupportedMountDurability(opts MountFileSystemOptions) (MountFileSystemOptions, error) { - if effectiveMountMode(opts.Driver) != "webdav" { + if companionMountMode(opts.Driver) != "webdav" { return opts, nil } if opts.DurabilityExplicit && opts.Durability != MountDurabilityAuto { @@ -116,12 +123,19 @@ func rejectUnsupportedMountDurability(opts MountFileSystemOptions) (MountFileSys return opts, nil } -// effectiveMountMode mirrors the companion's own `--mode auto` resolution +// companionMountMode mirrors the companion's own `--mode auto` resolution // (WebDAV on macOS and Windows, FUSE everywhere else) so ti's durability -// decision matches the mode the companion will actually mount with. It -// duplicates the companion rule on purpose: resolving it locally keeps the -// flag decision deterministic instead of depending on host prerequisites. -func effectiveMountMode(driver string) string { +// decision matches the mode the companion will actually mount with. +// +// It deliberately does not use mountdriver.Resolve: that resolver is +// prerequisite-aware and prefers FUSE whenever it is installable (macFUSE on +// darwin), while the companion's `--mode auto` is GOOS-only. Forwarding a tier +// based on ti's optimistic resolution would hand a durability flag to a WebDAV +// mount the companion chose, which the companion rejects. This function is the +// companion's rule, kept in one place and pinned by +// TestCompanionMountModeMatchesCompanionAutoResolution; the mount result +// reconciles the reported tier with the driver the companion reports. +func companionMountMode(driver string) string { switch strings.TrimSpace(driver) { case "", "auto": if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { @@ -140,5 +154,5 @@ func effectiveMountMode(driver string) string { // an ordinary mount keeps the documented interactive default instead of the // engine's auto default. Checkpoint mounts are read-only and forward nothing. func mountDurabilityForwarded(opts MountFileSystemOptions) bool { - return opts.CheckpointID == "" && effectiveMountMode(opts.Driver) == "fuse" + return opts.CheckpointID == "" && companionMountMode(opts.Driver) == "fuse" } diff --git a/internal/fs/mount_durability_test.go b/internal/fs/mount_durability_test.go index c5d02bbd..1fe6644a 100644 --- a/internal/fs/mount_durability_test.go +++ b/internal/fs/mount_durability_test.go @@ -95,6 +95,11 @@ func TestNormalizeMountDurability(t *testing.T) { writeBack: true, explicitly: true, }, + { + name: "explicit empty value is refused", + opts: MountFileSystemOptions{Driver: "fuse", Durability: " ", DurabilitySet: true}, + wantCode: "fs.missing_mount_durability", + }, { name: "unknown tier is refused", opts: MountFileSystemOptions{Driver: "fuse", Durability: "turbo", DurabilitySet: true}, @@ -201,9 +206,9 @@ func TestNormalizeMountDurabilityCheckpoint(t *testing.T) { } } -// TestEffectiveMountModeMatchesCompanionAutoResolution pins the local mirror of +// TestCompanionMountModeMatchesCompanionAutoResolution pins the local mirror of // the companion's --mode auto rule, which decides whether a tier is forwarded. -func TestEffectiveMountModeMatchesCompanionAutoResolution(t *testing.T) { +func TestCompanionMountModeMatchesCompanionAutoResolution(t *testing.T) { wantAuto := "fuse" if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { wantAuto = "webdav" @@ -214,8 +219,8 @@ func TestEffectiveMountModeMatchesCompanionAutoResolution(t *testing.T) { {driver: " fuse ", want: "fuse"}, {driver: "webdav", want: "webdav"}, } { - if got := effectiveMountMode(tc.driver); got != tc.want { - t.Fatalf("effectiveMountMode(%q) = %q, want %q", tc.driver, got, tc.want) + if got := companionMountMode(tc.driver); got != tc.want { + t.Fatalf("companionMountMode(%q) = %q, want %q", tc.driver, got, tc.want) } } } diff --git a/internal/fs/mountlocator/locator.go b/internal/fs/mountlocator/locator.go index 593c1a5a..40ff3c88 100644 --- a/internal/fs/mountlocator/locator.go +++ b/internal/fs/mountlocator/locator.go @@ -72,13 +72,13 @@ func New(profile, fileSystemName, regionCode, companionHome, mountPath, kind str }, nil } -// WithServiceAddress records the resolved regional service address. // WithDriver records the mount driver reported by the companion. func (l Locator) WithDriver(driver string) Locator { l.Driver = strings.TrimSpace(driver) return l } +// WithServiceAddress records the resolved regional service address. func (l Locator) WithServiceAddress(address string) Locator { l.ServiceAddress = strings.TrimSpace(address) return l From f1b3afcca54d8d5315a4a409e9099d211cd99dc1 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 19 Nov 2026 12:48:59 +0800 Subject: [PATCH 4/7] fix(fs): block on the parsed pending counters, not the drain classification Addresses the second review on #97. - `--ignore-absent` no longer decides from the engine's `error_kind`: the unmount blocks whenever the parsed drain report shows undrained work, so a drain that times out against a write-back backlog (non-pending classification, queued counters) can no longer detach with exit 0. Only a drain that produced no report at all stays continuable, which is exactly the already-gone-mount carve-out. - The drain error tail is built from the classification: pending work points at `--force` and says `--ignore-absent` will not help, while a non-pending or report-less failure points at `--ignore-absent` for a gone mount and at `--force` for accepting the loss. A drain with no report at all is now a structured `fs.drain_failed`/`fs.drain_report_invalid` instead of a raw companion error. - The drain `--dry-run` predicts the drain command's own rule (`drainApplicability`), so a WebDAV mount reports `drain_support: failed` while an unrecorded-driver locator stays passed. - The tolerated drain failure is recorded on both the `unmounted` and `absent` outcomes, and the shared "no mount control socket" sentence is reused. - Drop the never-populated `PID` fields from `UnmountResult`/`DrainResult`. - New pins: parsed-pending block under `--ignore-absent` (unit and e2e), non-pending remedy text, absent-path drain check, WebDAV and unrecorded-driver `drainApplicability`, and the drain dry-run check in both directions. --- AGENTS.md | 8 +- docs/spec/done/0039-fs-unmount-drain-gate.md | 42 ++++++---- e2e/cli_test.go | 28 ++++++- e2e/testdata/fake-drive9.go | 7 ++ internal/fs/drive9_companion.go | 18 +++-- internal/fs/drive9_fake_companion_test.go | 6 ++ internal/fs/drive9_mount_drain.go | 41 ++++++++-- internal/fs/drive9_mount_drain_test.go | 83 ++++++++++++++++++++ internal/fs/mount.go | 16 ++-- 9 files changed, 213 insertions(+), 36 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9b717872..b67e2ba6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1163,9 +1163,11 @@ in the non-secret mount locator, gates only FUSE file-system mounts on the drain, blocks with the locator preserved while work is pending, and treats `--force` as the only "accept the loss" path (reported as `drain.status: "skipped"`). `--ignore-absent` is a cleanup idempotency flag: it may continue -past a drain that cannot reach the mount, but it must never swallow -`fs.drain_pending_work`, and the outcome must still report `drain.status: -"failed"`. A locator written before the driver was recorded is never gated, so +past a drain that produced no report (a mount that is already gone), but the +block condition is the parsed pending counters, never the engine's `error_kind` +alone, and the outcome must still report `drain.status: "failed"` on both the +`unmounted` and `absent` paths. A non-pending classification with queued work +still blocks. A locator written before the driver was recorded is never gated, so an upgrade cannot block an existing unmount. The reported `durability` must match the driver the companion reported, and `--durability` must reject an explicitly empty value instead of treating it as the default. Do not drop these diff --git a/docs/spec/done/0039-fs-unmount-drain-gate.md b/docs/spec/done/0039-fs-unmount-drain-gate.md index 6e4812ca..69a599dd 100644 --- a/docs/spec/done/0039-fs-unmount-drain-gate.md +++ b/docs/spec/done/0039-fs-unmount-drain-gate.md @@ -84,13 +84,22 @@ A pending unmount fails with `fs.drain_pending_work`, names the engine's the reason, so a forced unmount is still attributable. It is the only way to accept undrained writes. - `--ignore-absent` keeps its cleanup idempotency meaning, not loss acceptance: - a drain that fails for a reason other than pending work (typically a mount - that is already gone) does not block, and the result records - `drain.status: "failed"` plus a `drain` runtime check. A - `fs.drain_pending_work` failure blocks even under `--ignore-absent`, and the - blocked-drain error names both flags so the remedy is unambiguous. Cleanup - scripts pass `--ignore-absent` routinely, so letting it bypass the pending - check would recreate the silent write loss this spec closes. + a drain that produced no report (typically a mount that is already gone) does + not block, and the result records `drain.status: "failed"` plus a `drain` + runtime check on both the `unmounted` and `absent` outcomes. The block + condition is the **parsed pending counters**, not the engine's classification: + a drain that hits its timeout against a write-back backlog reports a + non-pending `error_kind` while still listing queued work, so trusting the + classification alone would detach with exit 0 and print the counters it is + discarding. Cleanup scripts pass `--ignore-absent` routinely, so letting it + bypass the pending check would recreate the silent write loss this spec + closes. `--force` remains the only loss acceptance. +- The blocked-drain error tail is built from the classification: pending work + can only be bypassed with `--force` (and says `--ignore-absent` will not + help), while a non-pending or report-less failure points at + `--ignore-absent` for a mount that may be gone and at `--force` for accepting + the loss. A drain that produces no report at all is a structured + `fs.drain_failed` with the same guidance instead of a raw companion error. - The result reports the recorded driver and the drain record. Text output adds the `DRAIN` column and the pending counters. - `--dry-run` reports the gate in a `pre_unmount_drain` check without running @@ -100,10 +109,12 @@ A pending unmount fails with `fs.drain_pending_work`, names the engine's `ti fs drain-file-system` runs only for mounts the engine can drain. A non-`fs` locator kind (for example a `vault` mount) and a WebDAV driver fail with -`fs.drain_not_applicable` and an explanation, matching what the drain dry-run -already reported, instead of forwarding a request the companion rejects with a -raw error. A locator that records no driver stays drainable: the command itself -asks for a drain, and only the unmount gate needs the stricter driver check. +`fs.drain_not_applicable` and an explanation instead of forwarding a request the +companion rejects with a raw error. A locator that records no driver stays +drainable: the command itself asks for a drain, and only the unmount gate needs +the stricter driver check. The drain `--dry-run` reports the same rule in its +`drain_support` check, with `status: "failed"` when the command would refuse to +run, so the dry run predicts the execution instead of contradicting it. The `--force` flag previously promised to "kill the mount process" but no code read it; it now means "accept the undrained writes". @@ -148,9 +159,12 @@ remains the client-side mitigation. - The e2e mount expectations pin `--driver fuse` wherever a forwarded tier is asserted and branch the automatic-mode case on the host platform, so the mandatory `make e2e` gate stays green on macOS and Windows as well as Linux. -- The `--ignore-absent` cases are covered both ways: pending work still blocks - (unit and e2e), and a drain that cannot reach the mount continues with - `drain.status: "failed"`. +- The `--ignore-absent` cases are covered both ways: pending work blocks even + when the engine classifies the failure as something else (unit and e2e), and a + drain that produced no report continues with `drain.status: "failed"` on both + the `unmounted` and `absent` outcomes. +- The drain `--dry-run` check is pinned for both directions (WebDAV reports + `failed`, an unrecorded driver stays `passed`). The English PingCAP pages under `docs/pingcap-docs/docs/` are a submodule that was not checked out while this spec was implemented. The published mount and diff --git a/e2e/cli_test.go b/e2e/cli_test.go index b3d6f5f2..f30e6fc8 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -1291,6 +1291,22 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { ) webdavMount.wantExitCode(0) webdavMount.wantStdoutContains(`"durability": "auto"`) + // The drain dry-run predicts the drain command's own rule: a WebDAV mount + // cannot be drained, so the check fails instead of claiming success. + webdavDrainDryRun := runTIWithInput(t, bin, "", baseEnv, + "fs", "drain-file-system", + "--mount-path", webdavMountPath, + "--dry-run", + ) + webdavDrainDryRun.wantExitCode(0) + webdavDrainDryRun.wantStdoutContains(`"name": "drain_support"`) + webdavDrainDryRun.wantStdoutContains(`"status": "failed"`) + webdavDrain := runTIWithInput(t, bin, "", baseEnv, + "fs", "drain-file-system", + "--mount-path", webdavMountPath, + ) + webdavDrain.wantExitCode(2) + webdavDrain.wantStderrContains("no mount control socket") for _, path := range []string{webdavMountPath, filepath.Join(home, "mount-text"), autoMountPath} { extraUnmount := runTIWithInput(t, bin, "", baseEnv, "fs", "unmount-file-system", @@ -1338,7 +1354,7 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { ) pendingUnmount.wantExitCode(1) pendingUnmount.wantStderrContains("error_kind=pending_work_remaining") - pendingUnmount.wantStderrContains("unmount with --force") + pendingUnmount.wantStderrContains("--force is the only way past this on unmount") // --ignore-absent is the cleanup idempotency flag, not loss acceptance: it // must not swallow pending writes either. pendingIgnoreAbsent := runTIWithInput(t, bin, "", pendingDrainEnv, @@ -1348,6 +1364,16 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { ) pendingIgnoreAbsent.wantExitCode(1) pendingIgnoreAbsent.wantStderrContains("--ignore-absent does not skip this check") + // A non-pending classification with queued work is still pending work. + pendingKindEnv := append(append([]string{}, baseEnv...), "FAKE_DRIVE9_DRAIN_PENDING_KIND=deadline_exceeded") + pendingKind := runTIWithInput(t, bin, "", pendingKindEnv, + "fs", "unmount-file-system", + "--mount-path", strictMountPath, + "--ignore-absent", + ) + pendingKind.wantExitCode(1) + pendingKind.wantStderrContains("error_kind=deadline_exceeded") + pendingKind.wantStderrContains("commit_queue_pending=3") forcedUnmount := runTIWithInput(t, bin, "", pendingDrainEnv, "fs", "unmount-file-system", "--mount-path", strictMountPath, diff --git a/e2e/testdata/fake-drive9.go b/e2e/testdata/fake-drive9.go index dd7db61e..61b57519 100644 --- a/e2e/testdata/fake-drive9.go +++ b/e2e/testdata/fake-drive9.go @@ -248,6 +248,13 @@ func main() { report["error_kind"] = "pending_work_remaining" report["pending"] = map[string]any{"commit_queue_pending": 2, "uploader_queued": 2} } + if kind := os.Getenv("FAKE_DRIVE9_DRAIN_PENDING_KIND"); kind != "" { + ok = false + report["ok"] = false + report["error"] = "drain deadline exceeded with queued work" + report["error_kind"] = kind + report["pending"] = map[string]any{"commit_queue_pending": 3, "uploader_queued": 4} + } encoded, _ := json.Marshal(report) fmt.Println(string(encoded)) if !ok { diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index 2d7ae8a8..ebbccc00 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -1391,10 +1391,14 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy drain, drainErr := s.drainBeforeUnmount(ctx, profile, locator, opts) if drainErr != nil { // --ignore-absent is the idempotency flag cleanup scripts pass, not a - // licence to drop writes: only a drain that failed for another reason - // (typically a mount that is already gone) may be tolerated. Accepting - // the loss stays --force's job alone, so a pending drain always blocks. - if apperr.CodeFor(drainErr) == "fs.drain_pending_work" || !opts.IgnoreAbsent { + // licence to drop writes. Block whenever the engine's own report shows + // undrained work, regardless of how it classified the failure: a drain + // that times out against a write-back backlog is exactly the loss this + // gate exists to prevent. Only a failure that produced no report at all + // (typically a mount that is already gone) may be tolerated, and + // accepting the loss stays --force's job alone. + pending := drain.Response != nil && hasPendingWork(drain.Response.Pending) + if pending || apperr.CodeFor(drainErr) == "fs.drain_pending_work" || !opts.IgnoreAbsent { return UnmountResult{}, drainErr } } @@ -1418,7 +1422,11 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy if removeErr := mountlocator.Remove(homeDir, opts.MountPath); removeErr != nil { return UnmountResult{}, apperr.Wrap("fs.remove_mount_locator", "runtime", 1, "remove stale ti fs mount locator", removeErr) } - return UnmountResult{Status: "absent", MountPath: opts.MountPath, Driver: locator.Driver, Drain: &drain}, nil + absent := UnmountResult{Status: "absent", MountPath: opts.MountPath, Driver: locator.Driver, Drain: &drain} + if drainErr != nil { + absent.Checks = append(absent.Checks, MountRuntimeCheck{Name: "drain", Status: "failed", Message: apperr.MessageFor(drainErr)}) + } + return absent, nil } return UnmountResult{}, err } diff --git a/internal/fs/drive9_fake_companion_test.go b/internal/fs/drive9_fake_companion_test.go index de36e9f6..1db58423 100644 --- a/internal/fs/drive9_fake_companion_test.go +++ b/internal/fs/drive9_fake_companion_test.go @@ -114,6 +114,12 @@ func main() { report["error_kind"] = "pending_work_remaining" report["pending"] = map[string]any{"commit_queue_pending": 2, "uploader_queued": 2} } + if kind := os.Getenv("TI_FAKE_DRIVE9_DRAIN_PENDING_KIND"); kind != "" { + report["ok"] = false + report["error"] = "drain deadline exceeded with queued work" + report["error_kind"] = kind + report["pending"] = map[string]any{"commit_queue_pending": 3, "uploader_queued": 4} + } _ = json.NewEncoder(os.Stdout).Encode(report) if pending { os.Exit(1) diff --git a/internal/fs/drive9_mount_drain.go b/internal/fs/drive9_mount_drain.go index f320614f..998e5c40 100644 --- a/internal/fs/drive9_mount_drain.go +++ b/internal/fs/drive9_mount_drain.go @@ -82,10 +82,19 @@ func describeDrainFailure(response *mountcontrol.DrainResponse) string { return strings.Join(parts, "; ") } +// hasPendingWork reports whether the engine's report shows undrained write-side +// work. It is the authoritative signal for "the writes are still only local", +// independent of how the engine classified the failure: a drain that hits its +// timeout with a write-back backlog reports a non-pending error_kind but is +// exactly the loss this gate exists to prevent. +func hasPendingWork(pending mountcontrol.DrainPending) bool { + return drainPendingSummary(pending) != "none" +} + // drainErrorCode classifies a drain failure. Undrained write-back work is the // dangerous case (the data is still only local), so it gets its own code. func drainErrorCode(response *mountcontrol.DrainResponse) string { - if response != nil && response.ErrorKind == "pending_work_remaining" { + if response != nil && (response.ErrorKind == "pending_work_remaining" || hasPendingWork(response.Pending)) { return "fs.drain_pending_work" } return "fs.drain_failed" @@ -143,7 +152,7 @@ func drainApplicability(locator mountlocator.Locator) (bool, string) { return false, fmt.Sprintf("%s mounts have no drain; the engine drains only file-system mounts", kind) } if strings.TrimSpace(locator.Driver) == "webdav" { - return false, "webdav mounts have no mount control socket to drain; the companion flushes them when it unmounts" + return false, mountDrainSkipReason(locator.Driver) } return true, "" } @@ -156,6 +165,17 @@ func mountDrainSkipReason(driver string) string { return fmt.Sprintf("%s mounts have no mount control socket to drain; the companion flushes them when it unmounts", strings.TrimSpace(driver)) } +// drainRemedy builds the actionable tail of a drain error from its +// classification. Pending work can only be skipped with --force; a failure that +// produced no report is typically a mount that is already gone, which +// --ignore-absent tolerates on unmount. +func drainRemedy(code, mountPath string) string { + if code == "fs.drain_pending_work" { + return fmt.Sprintf("close the writers or let the background upload finish, then retry ti fs drain-file-system --mount-path %q; --force is the only way past this on unmount (--ignore-absent does not skip this check)", mountPath) + } + return fmt.Sprintf("if the mount is already gone, ti fs unmount-file-system --mount-path %q --ignore-absent continues; --force accepts losing any undrained writes", mountPath) +} + // drainMount runs the companion's drain and reports its result. It returns the // drain record even when the drain failed, so a caller that continues anyway // (unmount --force or --ignore-absent) can say what was left undrained. @@ -168,14 +188,25 @@ func (s Service) drainMount(ctx context.Context, profile *config.Profile, mountP result, runErr := s.drive9Run(ctx, profile, args, true) response, decodeErr := decodeDrive9DrainResponse(result.Stdout) if decodeErr != nil { + // No usable report: the mount is most likely gone, or the companion + // predates the report. That is a drain failure, not pending work, so + // the error has to carry the same unmount escape hatches as a classified + // failure instead of surfacing a raw companion error. + record := MountDrain{Status: "failed", Message: apperr.MessageFor(decodeErr)} + reason := apperr.MessageFor(decodeErr) if runErr != nil { - return MountDrain{Status: "failed", Message: apperr.MessageFor(runErr)}, runErr + reason = fmt.Sprintf("%v", runErr) + } + code := "fs.drain_failed" + if apperr.CodeFor(decodeErr) == "fs.drain_report_invalid" { + code = "fs.drain_report_invalid" } - return MountDrain{Status: "failed", Message: apperr.MessageFor(decodeErr)}, decodeErr + return record, apperr.New(code, "runtime", 1, fmt.Sprintf("drain %s produced no report: %s; %s", mountPath, reason, drainRemedy(code, mountPath))) } if runErr != nil || !response.OK { + code := drainErrorCode(response) record := MountDrain{Status: "failed", Message: describeDrainFailure(response), Response: response} - return record, apperr.New(drainErrorCode(response), "runtime", 1, fmt.Sprintf("drain %s did not finish: %s; close the writers or let the background upload finish, then retry ti fs drain-file-system --mount-path %q, or unmount with --force to accept losing the undrained writes (--ignore-absent does not skip this check)", mountPath, describeDrainFailure(response), mountPath)) + return record, apperr.New(code, "runtime", 1, fmt.Sprintf("drain %s did not finish: %s; %s", mountPath, describeDrainFailure(response), drainRemedy(code, mountPath))) } return MountDrain{Status: "drained", Response: response}, nil } diff --git a/internal/fs/drive9_mount_drain_test.go b/internal/fs/drive9_mount_drain_test.go index 9a2f3c44..9d5eec7c 100644 --- a/internal/fs/drive9_mount_drain_test.go +++ b/internal/fs/drive9_mount_drain_test.go @@ -342,6 +342,83 @@ func TestUnmountIgnoreAbsentToleratesGoneMount(t *testing.T) { } } +// TestUnmountIgnoreAbsentAbsentPathStillReportsDrainFailure keeps the two +// success shapes aligned: when the unmount itself reports the mount absent, the +// tolerated drain failure is still recorded. +func TestUnmountIgnoreAbsentAbsentPathStillReportsDrainFailure(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_FAIL", "1") + t.Setenv("TI_FAKE_UMOUNT_FAIL", "1") + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if err != nil { + t.Fatalf("ignore-absent unmount failed: %v", err) + } + if result.Status != "absent" || result.Drain == nil || result.Drain.Status != "failed" { + t.Fatalf("unexpected absent result: %#v", result) + } + found := false + for _, check := range result.Checks { + if check.Name == "drain" && check.Status == "failed" { + found = true + } + } + if !found { + t.Fatalf("absent path did not report the tolerated drain failure: %#v", result.Checks) + } +} + +// TestUnmountIgnoreAbsentStillBlocksParsedPendingWork covers the classification +// hole: a drain that fails with a non-pending error_kind while the report shows +// queued work must still block. +func TestUnmountIgnoreAbsentStillBlocksParsedPendingWork(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_PENDING_KIND", "deadline_exceeded") + + _, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if apperr.CodeFor(err) != "fs.drain_pending_work" { + t.Fatalf("ignore-absent unmount error = %v, want fs.drain_pending_work", err) + } + for _, call := range readFakeDrive9Calls(t, recordPath) { + if len(call.Args) > 0 && call.Args[0] == "umount" { + t.Fatalf("unmount ran with queued work: %#v", call.Args) + } + } +} + +// TestDrainRemedyMatchesClassification keeps the error guidance accurate: only +// pending work is unbailable by --ignore-absent. +func TestDrainRemedyMatchesClassification(t *testing.T) { + pending := drainRemedy("fs.drain_pending_work", "/mnt") + if !strings.Contains(pending, "--force") || !strings.Contains(pending, "--ignore-absent does not skip") { + t.Fatalf("pending remedy = %q", pending) + } + other := drainRemedy("fs.drain_failed", "/mnt") + if strings.Contains(other, "does not skip") || !strings.Contains(other, "--ignore-absent") || !strings.Contains(other, "--force") { + t.Fatalf("non-pending remedy = %q", other) + } +} + // TestDrainRejectsNonFileSystemMount keeps the drain command from reaching the // companion with a mount the engine cannot drain. func TestDrainRejectsNonFileSystemMount(t *testing.T) { @@ -353,6 +430,12 @@ func TestDrainRejectsNonFileSystemMount(t *testing.T) { if _, err := mountlocator.Write(home, legacyTestLocatorWithKind(t, home, mountPath, "vault", "fuse")); err != nil { t.Fatalf("write locator: %v", err) } + if drainable, _ := drainApplicability(mountlocator.Locator{Kind: mountKindFileSystem, Driver: "webdav"}); drainable { + t.Fatal("webdav locator reported drainable") + } + if drainable, _ := drainApplicability(mountlocator.Locator{Kind: mountKindFileSystem}); !drainable { + t.Fatal("locator with an unrecorded driver must stay drainable") + } if _, err := service.DrainFileSystem(context.Background(), DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}); apperr.CodeFor(err) != "fs.drain_not_applicable" { t.Fatalf("vault drain error = %v, want fs.drain_not_applicable", err) } diff --git a/internal/fs/mount.go b/internal/fs/mount.go index 4d4fb459..7f299ac8 100644 --- a/internal/fs/mount.go +++ b/internal/fs/mount.go @@ -125,7 +125,6 @@ type UnmountResult struct { Status string `json:"status"` MountPath string `json:"mount_path"` Driver string `json:"driver,omitempty"` - PID int `json:"pid,omitempty"` StateFile string `json:"state_file,omitempty"` Checks []MountRuntimeCheck `json:"checks,omitempty"` // Drain records the pre-unmount drain pass: drained, failed, or skipped @@ -147,7 +146,6 @@ type DrainResult struct { Status string `json:"status"` MountPath string `json:"mount_path"` Driver string `json:"driver,omitempty"` - PID int `json:"pid,omitempty"` StateFile string `json:"state_file,omitempty"` ControlSocket string `json:"control_socket,omitempty"` Response *mountcontrol.DrainResponse `json:"response,omitempty"` @@ -353,7 +351,7 @@ func (s Service) DryRunDrainFileSystem(ctx context.Context, commandPath string, dryrun.Check{Name: "mount_locator", Status: "passed", Message: opts.MountPath}, dryrun.Check{Name: "file_system_id", Status: "passed", Message: profile.FSTenantID}, dryrun.Check{Name: "region", Status: "passed", Message: profile.FSPlacementRegionCode}, - dryrun.Check{Name: "drain_support", Status: "passed", Message: drainSupportDryRunMessage(locator)}, + drainSupportCheck(locator), ), nil } @@ -385,12 +383,14 @@ func (s Service) DryRunUnmountFileSystem(ctx context.Context, commandPath string ), nil } -// drainSupportDryRunMessage says whether the mount can be drained at all. -func drainSupportDryRunMessage(locator mountlocator.Locator) string { - if drainable, reason := mountDrainGate(locator, false); !drainable { - return "not applicable: " + reason +// drainSupportCheck predicts whether `ti fs drain-file-system` will run for +// this mount. It uses the drain command's own rule, so a dry run cannot claim a +// drain that normal execution deterministically refuses. +func drainSupportCheck(locator mountlocator.Locator) dryrun.Check { + if drainable, reason := drainApplicability(locator); !drainable { + return dryrun.Check{Name: "drain_support", Status: "failed", Message: reason} } - return "FUSE mount with a drain control socket" + return dryrun.Check{Name: "drain_support", Status: "passed", Message: "file-system mount with a drain control socket"} } // describeDryRunDrainGate reports what the pre-unmount drain would do without From 33823dad02004d2474c845e784029a25795bda51 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 19 Nov 2026 13:11:19 +0800 Subject: [PATCH 5/7] docs(fs): describe the exact drain failures --ignore-absent tolerates qiffang's approval noted that the comment and specs claimed only a report-less drain failure may be tolerated, while the code also tolerates a parsed report with zero counters and a non-pending classification. The behavior is the narrower one we want, so only the wording changes: the comment, both specs, README, and AGENTS now say that --ignore-absent tolerates a drain failure that reports no pending work (no report at all, or zero counters with a non-pending classification) and that the block condition stays the parsed counters. --- AGENTS.md | 11 ++++++----- README.md | 2 +- docs/spec/done/0039-fs-unmount-drain-gate.md | 11 +++++++---- internal/fs/drive9_companion.go | 8 +++++--- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b67e2ba6..25936701 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1163,11 +1163,12 @@ in the non-secret mount locator, gates only FUSE file-system mounts on the drain, blocks with the locator preserved while work is pending, and treats `--force` as the only "accept the loss" path (reported as `drain.status: "skipped"`). `--ignore-absent` is a cleanup idempotency flag: it may continue -past a drain that produced no report (a mount that is already gone), but the -block condition is the parsed pending counters, never the engine's `error_kind` -alone, and the outcome must still report `drain.status: "failed"` on both the -`unmounted` and `absent` paths. A non-pending classification with queued work -still blocks. A locator written before the driver was recorded is never gated, so +only past a drain failure that reports no pending work (no report at all, or a +parsed report whose counters are all zero with a non-pending classification), +but the block condition is the parsed pending counters, never the engine's +`error_kind` alone, and the outcome must still report `drain.status: "failed"` +on both the `unmounted` and `absent` paths. A non-pending classification with +queued work still blocks. A locator written before the driver was recorded is never gated, so an upgrade cannot block an existing unmount. The reported `durability` must match the driver the companion reported, and `--durability` must reject an explicitly empty value instead of treating it as the default. Do not drop these diff --git a/README.md b/README.md index 4294ef5c..c960f7df 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ ti fs drain-file-system --mount-path ~/my-workspace ti fs unmount-file-system --mount-path ~/my-workspace ``` -`ti fs drain-file-system` reports the companion's drain result, including the pending counters (`open_handles`, `commit_queue_*`, `uploader_*`), so a caller can tell "fully uploaded" from "the background upload never finished"; it also refuses WebDAV and vault mounts, which have no drain, instead of failing with a raw companion error. `ti fs unmount-file-system` runs that drain itself before detaching a FUSE file-system mount and fails with `fs.drain_pending_work` while writes are still pending, keeping the mount locator so the command can be retried. `--force` is the only way to accept the undrained writes (the result then records `drain.status: "skipped"`); `--ignore-absent` stays a cleanup idempotency flag and continues only past a drain that cannot reach the mount (`drain.status: "failed"`), never past pending work. WebDAV mounts keep their native write behavior and are not gated, and `ti fs-vault unmount-vault` is not gated either because the engine drains only file-system mounts. +`ti fs drain-file-system` reports the companion's drain result, including the pending counters (`open_handles`, `commit_queue_*`, `uploader_*`), so a caller can tell "fully uploaded" from "the background upload never finished"; it also refuses WebDAV and vault mounts, which have no drain, instead of failing with a raw companion error. `ti fs unmount-file-system` runs that drain itself before detaching a FUSE file-system mount and fails with `fs.drain_pending_work` while writes are still pending, keeping the mount locator so the command can be retried. `--force` is the only way to accept the undrained writes (the result then records `drain.status: "skipped"`); `--ignore-absent` stays a cleanup idempotency flag and continues only past a drain failure that reports no pending work - no report at all, or a zero-counter report with a non-pending classification (`drain.status: "failed"`) - never past pending work. WebDAV mounts keep their native write behavior and are not gated, and `ti fs-vault unmount-vault` is not gated either because the engine drains only file-system mounts. Run a command against a remote Filesystem (Runtime is available only in AWS `us-east-1` and `us-west-2`): diff --git a/docs/spec/done/0039-fs-unmount-drain-gate.md b/docs/spec/done/0039-fs-unmount-drain-gate.md index 69a599dd..0a6c97a7 100644 --- a/docs/spec/done/0039-fs-unmount-drain-gate.md +++ b/docs/spec/done/0039-fs-unmount-drain-gate.md @@ -84,9 +84,11 @@ A pending unmount fails with `fs.drain_pending_work`, names the engine's the reason, so a forced unmount is still attributable. It is the only way to accept undrained writes. - `--ignore-absent` keeps its cleanup idempotency meaning, not loss acceptance: - a drain that produced no report (typically a mount that is already gone) does - not block, and the result records `drain.status: "failed"` plus a `drain` - runtime check on both the `unmounted` and `absent` outcomes. The block + it may tolerate only a drain failure that reports **no pending work** — no + report at all (typically a mount that is already gone, or a companion that + predates the report), or a parsed report whose counters are all zero with a + non-pending classification. The result records `drain.status: "failed"` plus a + `drain` runtime check on both the `unmounted` and `absent` outcomes. The block condition is the **parsed pending counters**, not the engine's classification: a drain that hits its timeout against a write-back backlog reports a non-pending `error_kind` while still listing queued work, so trusting the @@ -161,7 +163,8 @@ remains the client-side mitigation. mandatory `make e2e` gate stays green on macOS and Windows as well as Linux. - The `--ignore-absent` cases are covered both ways: pending work blocks even when the engine classifies the failure as something else (unit and e2e), and a - drain that produced no report continues with `drain.status: "failed"` on both + drain that reports no pending work (no report, or zero counters with a + non-pending classification) continues with `drain.status: "failed"` on both the `unmounted` and `absent` outcomes. - The drain `--dry-run` check is pinned for both directions (WebDAV reports `failed`, an unrecorded driver stays `passed`). diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index ebbccc00..41d8764a 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -1394,9 +1394,11 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy // licence to drop writes. Block whenever the engine's own report shows // undrained work, regardless of how it classified the failure: a drain // that times out against a write-back backlog is exactly the loss this - // gate exists to prevent. Only a failure that produced no report at all - // (typically a mount that is already gone) may be tolerated, and - // accepting the loss stays --force's job alone. + // gate exists to prevent. --ignore-absent may only tolerate a drain + // failure that reports no pending work: no report at all (typically a + // mount that is already gone, or a companion that predates the report) + // or a parsed report whose counters are all zero with a non-pending + // classification. Accepting actual loss stays --force's job alone. pending := drain.Response != nil && hasPendingWork(drain.Response.Pending) if pending || apperr.CodeFor(drainErr) == "fs.drain_pending_work" || !opts.IgnoreAbsent { return UnmountResult{}, drainErr From dcf8d97206671f04b6b61048059eaaca03e882f8 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 19 Nov 2026 13:34:08 +0800 Subject: [PATCH 6/7] refactor(fs): make the pending-work predicate explicit and one block authority Addresses mornyx's remaining hardening note and nits. - `DrainPending.HasPendingWork()` is now the single data-safety predicate, an explicit enumeration of the counters instead of a parse of the operator-facing summary, so rewording the text output can no longer change a safety decision. `drainPendingSummary` returns "none" only when the predicate is false. Spec 0039 records the engine contract behind it: the drain-failure exit record counts dirty_handles + commit_queue_pending + uploader_queued as pending files and commit_queue_bytes + uploader_cached_bytes as pending bytes, and ti keeps the conservative over-block for the informational counters until Drive9 publishes a loss-carrying split. - The unmount gate has one authority: `fs.drain_pending_work` already folds in the parsed counters and the engine's classification, so the second `hasPendingWork(...)` term is gone and the terms cannot drift. - `fs.drain_report_missing` now propagates like `fs.drain_report_invalid` instead of being collapsed into `fs.drain_failed`, and both are named in the spec. - `drain_support`'s passed message is neutral, so it no longer claims a control socket for driver-less legacy locators. - New pins: the predicate table independent of rendering, the tolerated zero-counter failure shape (unit and e2e fixtures), and a drain dry-run against a driver-less locator. --- AGENTS.md | 6 +- docs/spec/done/0039-fs-unmount-drain-gate.md | 31 ++++++- e2e/cli_test.go | 18 ++++ e2e/testdata/fake-drive9.go | 6 ++ internal/fs/drive9_companion.go | 20 ++--- internal/fs/drive9_fake_companion_test.go | 5 ++ internal/fs/drive9_mount_drain.go | 22 ++--- internal/fs/drive9_mount_drain_test.go | 87 ++++++++++++++++++++ internal/fs/mount.go | 2 +- internal/fs/mountcontrol/control.go | 25 ++++++ 10 files changed, 196 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 25936701..8eda6554 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1166,8 +1166,10 @@ drain, blocks with the locator preserved while work is pending, and treats only past a drain failure that reports no pending work (no report at all, or a parsed report whose counters are all zero with a non-pending classification), but the block condition is the parsed pending counters, never the engine's -`error_kind` alone, and the outcome must still report `drain.status: "failed"` -on both the `unmounted` and `absent` paths. A non-pending classification with +`error_kind` alone and never the rendered summary (`DrainPending.HasPendingWork` +is the single predicate, with one block authority: the error code that already +folds the counters in), and the outcome must still report `drain.status: +"failed"` on both the `unmounted` and `absent` paths. A non-pending classification with queued work still blocks. A locator written before the driver was recorded is never gated, so an upgrade cannot block an existing unmount. The reported `durability` must match the driver the companion reported, and `--durability` must reject an diff --git a/docs/spec/done/0039-fs-unmount-drain-gate.md b/docs/spec/done/0039-fs-unmount-drain-gate.md index 0a6c97a7..99e0a0b3 100644 --- a/docs/spec/done/0039-fs-unmount-drain-gate.md +++ b/docs/spec/done/0039-fs-unmount-drain-gate.md @@ -100,8 +100,28 @@ A pending unmount fails with `fs.drain_pending_work`, names the engine's can only be bypassed with `--force` (and says `--ignore-absent` will not help), while a non-pending or report-less failure points at `--ignore-absent` for a mount that may be gone and at `--force` for accepting - the loss. A drain that produces no report at all is a structured - `fs.drain_failed` with the same guidance instead of a raw companion error. + the loss. The gate itself has one authority, the error code: the + `fs.drain_pending_work` classification already folds in both the engine's + `error_kind` and the parsed counters, so the block condition cannot drift from + the message. A drain that produced no report is a structured + `fs.drain_report_missing` (no stdout at all) or `fs.drain_report_invalid` + (unparseable stdout) with the same guidance, instead of a raw companion error. + +### The pending predicate + +`DrainPending.HasPendingWork()` is the single data-safety predicate. It is an +explicit enumeration of the counters, never a parse of the operator-facing +summary, so rewording the text output cannot change a safety decision. + +The engine's own drain-failure exit record counts +`dirty_handles + commit_queue_pending + uploader_queued` as pending files and +`commit_queue_bytes + uploader_cached_bytes` as pending bytes, so those counters +are undrained work by construction. The remaining counters are informational, +but they are non-zero only while the mount still has unfinished work, so ti +treats any non-zero counter as pending. That is a deliberate conservative +over-block: refusing to detach is recoverable, silently detaching from queued +write-back is not. If Drive9 later publishes a loss-carrying versus +informational split, tighten the predicate and this section together. - The result reports the recorded driver and the drain record. Text output adds the `DRAIN` column and the pending counters. - `--dry-run` reports the gate in a `pre_unmount_drain` check without running @@ -167,7 +187,12 @@ remains the client-side mitigation. non-pending classification) continues with `drain.status: "failed"` on both the `unmounted` and `absent` outcomes. - The drain `--dry-run` check is pinned for both directions (WebDAV reports - `failed`, an unrecorded driver stays `passed`). + `failed`, and a driver-less legacy locator reports `passed` because the drain + command still accepts it). +- Both tolerated shapes have their own fixture: a drain that produces no report + (`fs.drain_report_missing`, tolerated) and a parsed report with a non-pending + classification and zero counters (tolerated, `drain.status: "failed"`), plus + the pending shape that blocks under `--ignore-absent`. The English PingCAP pages under `docs/pingcap-docs/docs/` are a submodule that was not checked out while this spec was implemented. The published mount and diff --git a/e2e/cli_test.go b/e2e/cli_test.go index f30e6fc8..939b737c 100644 --- a/e2e/cli_test.go +++ b/e2e/cli_test.go @@ -1383,6 +1383,24 @@ func TestFSDataPlaneUsesRecordedCredentialWithoutConfiguration(t *testing.T) { forcedUnmount.wantStdoutContains(`"status": "skipped"`) forcedUnmount.wantStdoutContains("--force skips the pre-unmount drain") + // A parsed failure with a non-pending classification and no queued counters + // reports the failure but lets the cleanup flag continue. + zeroMountPath := filepath.Join(home, "mount-zero") + zeroMount := runTIWithInput(t, bin, "", authEnv, + "fs", "mount-file-system", + "--mount-path", zeroMountPath, + "--driver", "fuse", + ) + zeroMount.wantExitCode(0) + zeroKindEnv := append(append([]string{}, baseEnv...), "FAKE_DRIVE9_DRAIN_ZERO_KIND=deadline_exceeded") + zeroKind := runTIWithInput(t, bin, "", zeroKindEnv, + "fs", "unmount-file-system", + "--mount-path", zeroMountPath, + "--ignore-absent", + ) + zeroKind.wantExitCode(0) + zeroKind.wantStdoutContains(`"status": "failed"`) + // A drain that cannot reach the mount is not pending work, so the cleanup // flag may continue, but the outcome still reports the failed drain. goneMountPath := filepath.Join(home, "mount-gone") diff --git a/e2e/testdata/fake-drive9.go b/e2e/testdata/fake-drive9.go index 61b57519..f437f404 100644 --- a/e2e/testdata/fake-drive9.go +++ b/e2e/testdata/fake-drive9.go @@ -248,6 +248,12 @@ func main() { report["error_kind"] = "pending_work_remaining" report["pending"] = map[string]any{"commit_queue_pending": 2, "uploader_queued": 2} } + if kind := os.Getenv("FAKE_DRIVE9_DRAIN_ZERO_KIND"); kind != "" { + ok = false + report["ok"] = false + report["error"] = "drain deadline exceeded with nothing queued" + report["error_kind"] = kind + } if kind := os.Getenv("FAKE_DRIVE9_DRAIN_PENDING_KIND"); kind != "" { ok = false report["ok"] = false diff --git a/internal/fs/drive9_companion.go b/internal/fs/drive9_companion.go index 41d8764a..6314011f 100644 --- a/internal/fs/drive9_companion.go +++ b/internal/fs/drive9_companion.go @@ -1391,16 +1391,16 @@ func (s Service) drive9UnmountFileSystem(ctx context.Context, opts UnmountFileSy drain, drainErr := s.drainBeforeUnmount(ctx, profile, locator, opts) if drainErr != nil { // --ignore-absent is the idempotency flag cleanup scripts pass, not a - // licence to drop writes. Block whenever the engine's own report shows - // undrained work, regardless of how it classified the failure: a drain - // that times out against a write-back backlog is exactly the loss this - // gate exists to prevent. --ignore-absent may only tolerate a drain - // failure that reports no pending work: no report at all (typically a - // mount that is already gone, or a companion that predates the report) - // or a parsed report whose counters are all zero with a non-pending - // classification. Accepting actual loss stays --force's job alone. - pending := drain.Response != nil && hasPendingWork(drain.Response.Pending) - if pending || apperr.CodeFor(drainErr) == "fs.drain_pending_work" || !opts.IgnoreAbsent { + // licence to drop writes. The error code is the single authority here: + // fs.drain_pending_work folds in both the engine's classification and + // the parsed pending counters, so a drain that times out against a + // write-back backlog still blocks. --ignore-absent may only tolerate a + // drain failure that reports no pending work: no report at all + // (typically a mount that is already gone, or a companion that predates + // the report) or a parsed report whose counters are all zero with a + // non-pending classification. Accepting actual loss stays --force's job + // alone. + if apperr.CodeFor(drainErr) == "fs.drain_pending_work" || !opts.IgnoreAbsent { return UnmountResult{}, drainErr } } diff --git a/internal/fs/drive9_fake_companion_test.go b/internal/fs/drive9_fake_companion_test.go index 1db58423..fac7dd04 100644 --- a/internal/fs/drive9_fake_companion_test.go +++ b/internal/fs/drive9_fake_companion_test.go @@ -114,6 +114,11 @@ func main() { report["error_kind"] = "pending_work_remaining" report["pending"] = map[string]any{"commit_queue_pending": 2, "uploader_queued": 2} } + if kind := os.Getenv("TI_FAKE_DRIVE9_DRAIN_ZERO_KIND"); kind != "" { + report["ok"] = false + report["error"] = "drain deadline exceeded with nothing queued" + report["error_kind"] = kind + } if kind := os.Getenv("TI_FAKE_DRIVE9_DRAIN_PENDING_KIND"); kind != "" { report["ok"] = false report["error"] = "drain deadline exceeded with queued work" diff --git a/internal/fs/drive9_mount_drain.go b/internal/fs/drive9_mount_drain.go index 998e5c40..5896d1b7 100644 --- a/internal/fs/drive9_mount_drain.go +++ b/internal/fs/drive9_mount_drain.go @@ -56,7 +56,7 @@ func drainPendingSummary(pending mountcontrol.DrainPending) string { } rendered = append(rendered, fmt.Sprintf("%s=%d", counter.name, counter.value)) } - if len(rendered) == 0 { + if len(rendered) == 0 || !pending.HasPendingWork() { return "none" } return strings.Join(rendered, " ") @@ -83,12 +83,13 @@ func describeDrainFailure(response *mountcontrol.DrainResponse) string { } // hasPendingWork reports whether the engine's report shows undrained write-side -// work. It is the authoritative signal for "the writes are still only local", -// independent of how the engine classified the failure: a drain that hits its -// timeout with a write-back backlog reports a non-pending error_kind but is -// exactly the loss this gate exists to prevent. +// work. The predicate is an explicit enumeration of the counters, never the +// operator-facing rendering: rewording the summary must not be able to change a +// data-safety decision. It is independent of how the engine classified the +// failure, because a drain that hits its timeout with a write-back backlog +// reports a non-pending error_kind and is exactly the loss this gate prevents. func hasPendingWork(pending mountcontrol.DrainPending) bool { - return drainPendingSummary(pending) != "none" + return pending.HasPendingWork() } // drainErrorCode classifies a drain failure. Undrained write-back work is the @@ -197,10 +198,11 @@ func (s Service) drainMount(ctx context.Context, profile *config.Profile, mountP if runErr != nil { reason = fmt.Sprintf("%v", runErr) } - code := "fs.drain_failed" - if apperr.CodeFor(decodeErr) == "fs.drain_report_invalid" { - code = "fs.drain_report_invalid" - } + // Keep the report-shaped code: "no report" and "unparseable report" + // tell an operator different things (a gone mount or an old companion + // versus a broken companion), and both are tolerated by + // --ignore-absent because neither shows pending work. + code := apperr.CodeFor(decodeErr) return record, apperr.New(code, "runtime", 1, fmt.Sprintf("drain %s produced no report: %s; %s", mountPath, reason, drainRemedy(code, mountPath))) } if runErr != nil || !response.OK { diff --git a/internal/fs/drive9_mount_drain_test.go b/internal/fs/drive9_mount_drain_test.go index 9d5eec7c..e78cdd5b 100644 --- a/internal/fs/drive9_mount_drain_test.go +++ b/internal/fs/drive9_mount_drain_test.go @@ -9,6 +9,7 @@ import ( "github.com/tidbcloud/ti-cli/internal/apperr" "github.com/tidbcloud/ti-cli/internal/fs/fscred" + "github.com/tidbcloud/ti-cli/internal/fs/mountcontrol" "github.com/tidbcloud/ti-cli/internal/fs/mountlocator" ) @@ -406,6 +407,92 @@ func TestUnmountIgnoreAbsentStillBlocksParsedPendingWork(t *testing.T) { } } +// TestUnmountIgnoreAbsentToleratesZeroCounterFailure pins the tolerated shape: +// a parsed report with a non-pending classification and no queued counters +// reports drain.status "failed" but lets the cleanup unmount continue. +func TestUnmountIgnoreAbsentToleratesZeroCounterFailure(t *testing.T) { + home := t.TempDir() + companion, recordPath := buildFakeDrive9(t) + t.Setenv("TI_FAKE_DRIVE9_RECORD", recordPath) + service := testCompanionService(home, companion) + service.mountPointActive = trueMountEvidence + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := service.MountFileSystem(context.Background(), MountFileSystemOptions{ + Profile: dataProfile(), MountPath: mountPath, Driver: "fuse", + }); err != nil { + t.Fatalf("mount failed: %v", err) + } + t.Setenv("TI_FAKE_DRIVE9_DRAIN_ZERO_KIND", "deadline_exceeded") + + result, err := service.UnmountFileSystem(context.Background(), UnmountFileSystemOptions{Profile: dataProfile(), MountPath: mountPath, IgnoreAbsent: true}) + if err != nil { + t.Fatalf("ignore-absent unmount failed: %v", err) + } + if result.Status != "unmounted" || result.Drain == nil || result.Drain.Status != "failed" { + t.Fatalf("unexpected unmount result: %#v", result) + } + if result.Drain.Response == nil || result.Drain.Response.Pending.HasPendingWork() { + t.Fatalf("fixture was supposed to report no queued work: %#v", result.Drain.Response) + } +} + +// TestDryRunDrainSupportForDriverlessLocator keeps the drain dry run honest for +// mounts whose locator predates the recorded driver: the drain command still +// accepts them, so the check must not claim it cannot run. +func TestDryRunDrainSupportForDriverlessLocator(t *testing.T) { + home := t.TempDir() + companion, _ := buildFakeDrive9(t) + service := testCompanionService(home, companion) + mountPath := filepath.Join(t.TempDir(), "mnt") + if _, err := mountlocator.Write(home, legacyTestLocator(t, home, mountPath)); err != nil { + t.Fatalf("write locator: %v", err) + } + result, err := service.DryRunDrainFileSystem(context.Background(), "ti fs drain-file-system", DrainFileSystemOptions{Profile: dataProfile(), MountPath: mountPath}) + if err != nil { + t.Fatalf("dry run failed: %v", err) + } + for _, check := range result.Checks { + if check.Name != "drain_support" { + continue + } + if check.Status != "passed" { + t.Fatalf("driver-less drain dry run = %#v, want passed", check) + } + return + } + t.Fatal("drain_support check missing") +} + +// TestHasPendingWorkIsIndependentOfRendering keeps the safety predicate from +// ever depending on the operator-facing summary. +func TestHasPendingWorkIsIndependentOfRendering(t *testing.T) { + for _, tc := range []struct { + name string + pending mountcontrol.DrainPending + want bool + }{ + {name: "empty", pending: mountcontrol.DrainPending{}, want: false}, + {name: "open handles", pending: mountcontrol.DrainPending{OpenHandles: 1}, want: true}, + {name: "dirty handles", pending: mountcontrol.DrainPending{DirtyHandles: 1}, want: true}, + {name: "commit queue", pending: mountcontrol.DrainPending{CommitQueuePending: 1}, want: true}, + {name: "commit bytes", pending: mountcontrol.DrainPending{CommitQueueBytes: 1}, want: true}, + {name: "uploader queued", pending: mountcontrol.DrainPending{UploaderQueued: 1}, want: true}, + {name: "uploader cached", pending: mountcontrol.DrainPending{UploaderCached: 1}, want: true}, + {name: "uploader cached bytes", pending: mountcontrol.DrainPending{UploaderCachedBytes: 1}, want: true}, + {name: "conflicts", pending: mountcontrol.DrainPending{CommitQueueConflicts: 1}, want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := hasPendingWork(tc.pending); got != tc.want { + t.Fatalf("hasPendingWork(%+v) = %t, want %t", tc.pending, got, tc.want) + } + summary := drainPendingSummary(tc.pending) + if tc.want != (summary != "none") { + t.Fatalf("summary %q disagrees with the predicate for %+v", summary, tc.pending) + } + }) + } +} + // TestDrainRemedyMatchesClassification keeps the error guidance accurate: only // pending work is unbailable by --ignore-absent. func TestDrainRemedyMatchesClassification(t *testing.T) { diff --git a/internal/fs/mount.go b/internal/fs/mount.go index 7f299ac8..e5cfbca3 100644 --- a/internal/fs/mount.go +++ b/internal/fs/mount.go @@ -390,7 +390,7 @@ func drainSupportCheck(locator mountlocator.Locator) dryrun.Check { if drainable, reason := drainApplicability(locator); !drainable { return dryrun.Check{Name: "drain_support", Status: "failed", Message: reason} } - return dryrun.Check{Name: "drain_support", Status: "passed", Message: "file-system mount with a drain control socket"} + return dryrun.Check{Name: "drain_support", Status: "passed", Message: "file-system mount; the drain command asks the companion"} } // describeDryRunDrainGate reports what the pre-unmount drain would do without diff --git a/internal/fs/mountcontrol/control.go b/internal/fs/mountcontrol/control.go index 532ad723..37f35b1a 100644 --- a/internal/fs/mountcontrol/control.go +++ b/internal/fs/mountcontrol/control.go @@ -52,6 +52,31 @@ type DrainPending struct { UploaderCachedBytes int64 `json:"uploader_cached_bytes"` } +// HasPendingWork reports whether the snapshot lists work the engine has not made +// durable remotely. +// +// The engine's own drain-failure exit record counts exactly +// dirty_handles + commit_queue_pending + uploader_queued as pending files and +// commit_queue_bytes + uploader_cached_bytes as pending bytes, so those counters +// are undrained work by construction. The remaining counters are informational, +// but they are non-zero only while the mount still has unfinished work, so ti +// treats any non-zero counter as pending: over-blocking an unmount is safe, +// silently detaching from queued write-back is not. Keep this conservative until +// the engine publishes a loss-carrying/informational split. +func (p DrainPending) HasPendingWork() bool { + return p.OpenHandles != 0 || + p.DirtyHandles != 0 || + p.CommitQueuePending != 0 || + p.CommitQueueBytes != 0 || + p.CommitQueueInFlight != 0 || + p.CommitQueueDelayed != 0 || + p.CommitQueueConflicts != 0 || + p.UploaderQueued != 0 || + p.UploaderInFlight != 0 || + p.UploaderCached != 0 || + p.UploaderCachedBytes != 0 +} + type DrainPhase struct { Name string `json:"name"` DurationMS int64 `json:"duration_ms"` From 64fa530555561b59c18900198f0b7aefac6c1929 Mon Sep 17 00:00:00 2001 From: srstack Date: Thu, 19 Nov 2026 14:16:31 +0800 Subject: [PATCH 7/7] test(fs): pin every pending counter and finish the predicate cleanup Addresses mornyx's round-4 review. - `TestHasPendingWorkIsIndependentOfRendering` now has a row for all eleven counters, including the three it silently skipped (commit_queue_in_flight, commit_queue_delayed, uploader_in_flight). Deleting any term from the predicate now fails the suite, which matters because spec 0039 schedules a future tightening edit of exactly this predicate. - Drop the vestigial `len(rendered) == 0` term from `drainPendingSummary`; the summary now derives "none" from the predicate alone. - Drop the one-line `hasPendingWork` delegation and call `response.Pending.HasPendingWork()` at the single consuming site, keeping the doc comment on the method. - Decode-failure message says "produced no usable report", which is also true when stdout existed but was unparseable. - Spec 0039: the drain-command taxonomy lists `fs.drain_report_missing` and `fs.drain_report_invalid` alongside `fs.drain_failed`, and the pending predicate section sits after the completed unmount bullet list instead of splitting it. --- docs/spec/done/0039-fs-unmount-drain-gate.md | 17 ++++++++++------- internal/fs/drive9_mount_drain.go | 16 +++------------- internal/fs/drive9_mount_drain_test.go | 18 +++++++++++------- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/docs/spec/done/0039-fs-unmount-drain-gate.md b/docs/spec/done/0039-fs-unmount-drain-gate.md index 99e0a0b3..7a3b6f49 100644 --- a/docs/spec/done/0039-fs-unmount-drain-gate.md +++ b/docs/spec/done/0039-fs-unmount-drain-gate.md @@ -61,9 +61,12 @@ A pending unmount fails with `fs.drain_pending_work`, names the engine's `response` (same shape as `mountcontrol.DrainResponse`) instead of being dropped. - `status` is `drained`. A drain that leaves write-side work fails with - `fs.drain_pending_work`; any other drain failure uses `fs.drain_failed`. Both - errors carry `error_kind`, the engine's error text, and the non-zero pending - counters, plus the retry command. + `fs.drain_pending_work`; a classified failure that reports none uses + `fs.drain_failed`; a failure with no usable report uses + `fs.drain_report_missing` (no stdout) or `fs.drain_report_invalid` + (unparseable stdout). Every error carries `error_kind`, the engine's error + text, and the non-zero pending counters when there are any, plus the remedy + command. - Text output prints the pending counters, so `--output text` is usable in an operator runbook. @@ -106,6 +109,10 @@ A pending unmount fails with `fs.drain_pending_work`, names the engine's the message. A drain that produced no report is a structured `fs.drain_report_missing` (no stdout at all) or `fs.drain_report_invalid` (unparseable stdout) with the same guidance, instead of a raw companion error. +- The result reports the recorded driver and the drain record. Text output adds + the `DRAIN` column and the pending counters. +- `--dry-run` reports the gate in a `pre_unmount_drain` check without running + the drain. ### The pending predicate @@ -122,10 +129,6 @@ treats any non-zero counter as pending. That is a deliberate conservative over-block: refusing to detach is recoverable, silently detaching from queued write-back is not. If Drive9 later publishes a loss-carrying versus informational split, tighten the predicate and this section together. -- The result reports the recorded driver and the drain record. Text output adds - the `DRAIN` column and the pending counters. -- `--dry-run` reports the gate in a `pre_unmount_drain` check without running - the drain. ### Drain applicability diff --git a/internal/fs/drive9_mount_drain.go b/internal/fs/drive9_mount_drain.go index 5896d1b7..b5b2fc05 100644 --- a/internal/fs/drive9_mount_drain.go +++ b/internal/fs/drive9_mount_drain.go @@ -56,7 +56,7 @@ func drainPendingSummary(pending mountcontrol.DrainPending) string { } rendered = append(rendered, fmt.Sprintf("%s=%d", counter.name, counter.value)) } - if len(rendered) == 0 || !pending.HasPendingWork() { + if !pending.HasPendingWork() { return "none" } return strings.Join(rendered, " ") @@ -82,20 +82,10 @@ func describeDrainFailure(response *mountcontrol.DrainResponse) string { return strings.Join(parts, "; ") } -// hasPendingWork reports whether the engine's report shows undrained write-side -// work. The predicate is an explicit enumeration of the counters, never the -// operator-facing rendering: rewording the summary must not be able to change a -// data-safety decision. It is independent of how the engine classified the -// failure, because a drain that hits its timeout with a write-back backlog -// reports a non-pending error_kind and is exactly the loss this gate prevents. -func hasPendingWork(pending mountcontrol.DrainPending) bool { - return pending.HasPendingWork() -} - // drainErrorCode classifies a drain failure. Undrained write-back work is the // dangerous case (the data is still only local), so it gets its own code. func drainErrorCode(response *mountcontrol.DrainResponse) string { - if response != nil && (response.ErrorKind == "pending_work_remaining" || hasPendingWork(response.Pending)) { + if response != nil && (response.ErrorKind == "pending_work_remaining" || response.Pending.HasPendingWork()) { return "fs.drain_pending_work" } return "fs.drain_failed" @@ -203,7 +193,7 @@ func (s Service) drainMount(ctx context.Context, profile *config.Profile, mountP // versus a broken companion), and both are tolerated by // --ignore-absent because neither shows pending work. code := apperr.CodeFor(decodeErr) - return record, apperr.New(code, "runtime", 1, fmt.Sprintf("drain %s produced no report: %s; %s", mountPath, reason, drainRemedy(code, mountPath))) + return record, apperr.New(code, "runtime", 1, fmt.Sprintf("drain %s produced no usable report: %s; %s", mountPath, reason, drainRemedy(code, mountPath))) } if runErr != nil || !response.OK { code := drainErrorCode(response) diff --git a/internal/fs/drive9_mount_drain_test.go b/internal/fs/drive9_mount_drain_test.go index e78cdd5b..592f67ba 100644 --- a/internal/fs/drive9_mount_drain_test.go +++ b/internal/fs/drive9_mount_drain_test.go @@ -463,8 +463,8 @@ func TestDryRunDrainSupportForDriverlessLocator(t *testing.T) { t.Fatal("drain_support check missing") } -// TestHasPendingWorkIsIndependentOfRendering keeps the safety predicate from -// ever depending on the operator-facing summary. +// TestHasPendingWorkIsIndependentOfRendering pins every counter the predicate +// enumerates and keeps it from ever depending on the operator-facing summary. func TestHasPendingWorkIsIndependentOfRendering(t *testing.T) { for _, tc := range []struct { name string @@ -474,16 +474,20 @@ func TestHasPendingWorkIsIndependentOfRendering(t *testing.T) { {name: "empty", pending: mountcontrol.DrainPending{}, want: false}, {name: "open handles", pending: mountcontrol.DrainPending{OpenHandles: 1}, want: true}, {name: "dirty handles", pending: mountcontrol.DrainPending{DirtyHandles: 1}, want: true}, - {name: "commit queue", pending: mountcontrol.DrainPending{CommitQueuePending: 1}, want: true}, - {name: "commit bytes", pending: mountcontrol.DrainPending{CommitQueueBytes: 1}, want: true}, + {name: "commit queue pending", pending: mountcontrol.DrainPending{CommitQueuePending: 1}, want: true}, + {name: "commit queue bytes", pending: mountcontrol.DrainPending{CommitQueueBytes: 1}, want: true}, + {name: "commit queue in flight", pending: mountcontrol.DrainPending{CommitQueueInFlight: 1}, want: true}, + {name: "commit queue delayed", pending: mountcontrol.DrainPending{CommitQueueDelayed: 1}, want: true}, + {name: "commit queue conflicts", pending: mountcontrol.DrainPending{CommitQueueConflicts: 1}, want: true}, {name: "uploader queued", pending: mountcontrol.DrainPending{UploaderQueued: 1}, want: true}, + {name: "uploader in flight", pending: mountcontrol.DrainPending{UploaderInFlight: 1}, want: true}, {name: "uploader cached", pending: mountcontrol.DrainPending{UploaderCached: 1}, want: true}, {name: "uploader cached bytes", pending: mountcontrol.DrainPending{UploaderCachedBytes: 1}, want: true}, - {name: "conflicts", pending: mountcontrol.DrainPending{CommitQueueConflicts: 1}, want: true}, } { t.Run(tc.name, func(t *testing.T) { - if got := hasPendingWork(tc.pending); got != tc.want { - t.Fatalf("hasPendingWork(%+v) = %t, want %t", tc.pending, got, tc.want) + got := tc.pending.HasPendingWork() + if got != tc.want { + t.Fatalf("HasPendingWork(%+v) = %t, want %t", tc.pending, got, tc.want) } summary := drainPendingSummary(tc.pending) if tc.want != (summary != "none") {