From f97058ec476d98b1cd7529c1acfbe852c2f8b219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Efe=20G=C3=B6kdemir?= Date: Tue, 27 Oct 2026 14:45:33 +0300 Subject: [PATCH] feat(backup): support restore modes and cluster integrity --- README.md | 2 + docs/releases/unreleased.md | 2 + internal/client/backup.go | 114 +++++---- internal/client/backup_incomplete_test.go | 26 ++- internal/client/backup_test.go | 12 +- internal/commands/backup.go | 242 ++++++++++++++++---- internal/commands/backup_incomplete_test.go | 49 ++-- internal/commands/backup_test.go | 84 ++++++- 8 files changed, 413 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index deed222..7c8cf67 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,8 @@ arcli logs --level warn --since 6h `backup restore` is destructive and leaves metadata restoration to Arc's defaults: enabled on standalone nodes when present in the backup, skipped on cluster nodes. Use `--with-metadata` to request it explicitly (clusters refuse it), `--data-only` to explicitly skip metadata and config, or `--with-config` to also request config restoration. Stop writers and compaction before restoring. When the server reports staged metadata or a required restart, wait for the restore to complete before restarting. +Use `arcli backup restore --mode merge|replace` to choose how cluster data is restored. If omitted, Arc uses merge for full backups and may use replace for scoped backups on a cluster. `backup list` reports cluster-manifest gaps, while `backup show` includes samples and distinguishes missing data from files excluded or reconciled during the backup. + The command tree also covers API tokens, continuous queries, schedulers, predicate deletes, backups, cluster membership, and compaction. Run `arcli --help` or `arcli --help` for the complete command reference. `license status` reads public `/health` without sending a token or checking authentication. It works on standalone and clustered deployments and supports table and JSON output. It reports the server's tier, status, source, and optional expiry, days remaining, and site-license state. Expiry is normalized to UTC RFC3339 with fractional seconds when provided. diff --git a/docs/releases/unreleased.md b/docs/releases/unreleased.md index b75bbe1..96786f6 100644 --- a/docs/releases/unreleased.md +++ b/docs/releases/unreleased.md @@ -5,3 +5,5 @@ - All successfully read license states exit 0. Configuration, request, decoding, and output failures exit nonzero. No token or database header is sent to `/health`, and no authentication verification is performed. Normal connection selection, timeout, TLS verification, and identity opt-out settings apply. License installation and key/file configuration remain server responsibilities. Existing `ping -o json` license pass-through is unchanged. Closes [arcli#34](https://github.com/Basekick-Labs/arcli/issues/34). + +- Backup commands support explicit `--mode merge|replace` restores and report the effective mode returned by Arc. Backup list/show and restore status now include cluster-manifest counts and samples; reconciled skips are excluded from incomplete counts, while unregistered and mid-run manifest changes remain informational. Part of [arcli#44](https://github.com/Basekick-Labs/arcli/issues/44). diff --git a/internal/client/backup.go b/internal/client/backup.go index 34c46a1..cbb5fd7 100644 --- a/internal/client/backup.go +++ b/internal/client/backup.go @@ -38,22 +38,26 @@ func (e *BackupBusyError) Error() string { // BackupSummary is one entry of GET /api/v1/backup/. // -// The three incompleteness counts arrive from Arc 26.09.3 on and are absent -// (zero) from older servers; each is its own population on the server: -// SkippedFiles are data files inside TotalFiles that were not stored, -// SkippedMetadataFiles are Iceberg metadata or compaction recovery state -// (not inside TotalFiles), UnaddressableFiles were never inventoried because -// no listing can return their key. +// Older Arc servers omit the incompleteness counts. SkippedFiles are data +// files inside TotalFiles that were not stored, SkippedMetadataFiles are +// Iceberg metadata or compaction recovery state (not inside TotalFiles), and +// UnaddressableFiles were never inventoried because no listing can return +// their key. Cluster manifest counts arrive on Arc 27.01.1+. type BackupSummary struct { - BackupID string `json:"backup_id"` - CreatedAt time.Time `json:"created_at"` - BackupType string `json:"backup_type"` - TotalFiles int64 `json:"total_files"` - TotalBytes int64 `json:"total_size_bytes"` - DatabaseCount int `json:"database_count"` - SkippedFiles int64 `json:"skipped_files,omitempty"` - SkippedMetadataFiles int64 `json:"skipped_metadata_files,omitempty"` - UnaddressableFiles int64 `json:"unaddressable_files,omitempty"` + BackupID string `json:"backup_id"` + CreatedAt time.Time `json:"created_at"` + BackupType string `json:"backup_type"` + TotalFiles int64 `json:"total_files"` + TotalBytes int64 `json:"total_size_bytes"` + DatabaseCount int `json:"database_count"` + SkippedFiles int64 `json:"skipped_files,omitempty"` + SkippedMetadataFiles int64 `json:"skipped_metadata_files,omitempty"` + UnaddressableFiles int64 `json:"unaddressable_files,omitempty"` + SkippedReconciled int64 `json:"skipped_reconciled,omitempty"` + UnregisteredSkipped int64 `json:"unregistered_skipped,omitempty"` + ManifestOnlyFiles int64 `json:"manifest_only_files,omitempty"` + LeftManifestDuringRun int64 `json:"left_manifest_during_run,omitempty"` + ClusterManifestChecked *bool `json:"cluster_manifest_checked,omitempty"` } // BackupMeasurement / BackupDatabase / BackupManifest mirror @@ -81,24 +85,32 @@ type BackupIcebergWarehouse struct { } type BackupManifest struct { - Version string `json:"version"` - BackupID string `json:"backup_id"` - CreatedAt time.Time `json:"created_at"` - BackupType string `json:"backup_type"` - Databases []BackupDatabase `json:"databases"` - TotalFiles int64 `json:"total_files"` - TotalSizeBytes int64 `json:"total_size_bytes"` - SkippedFiles int64 `json:"skipped_files,omitempty"` - SkippedMetadataFiles int64 `json:"skipped_metadata_files,omitempty"` - SkippedSample []string `json:"skipped_sample,omitempty"` - SkippedOverlongKeys int64 `json:"skipped_overlong_keys,omitempty"` - UnaddressableFiles int64 `json:"unaddressable_files,omitempty"` - UnaddressableSample []string `json:"unaddressable_sample,omitempty"` - IcebergWarehouse *BackupIcebergWarehouse `json:"iceberg_warehouse,omitempty"` - HasMetadata bool `json:"has_metadata"` - HasIcebergCatalog bool `json:"has_iceberg_catalog,omitempty"` - HasConfig bool `json:"has_config"` - Raw json.RawMessage `json:"-"` + Version string `json:"version"` + BackupID string `json:"backup_id"` + CreatedAt time.Time `json:"created_at"` + BackupType string `json:"backup_type"` + Databases []BackupDatabase `json:"databases"` + TotalFiles int64 `json:"total_files"` + TotalSizeBytes int64 `json:"total_size_bytes"` + SkippedFiles int64 `json:"skipped_files,omitempty"` + SkippedMetadataFiles int64 `json:"skipped_metadata_files,omitempty"` + SkippedSample []string `json:"skipped_sample,omitempty"` + SkippedOverlongKeys int64 `json:"skipped_overlong_keys,omitempty"` + UnaddressableFiles int64 `json:"unaddressable_files,omitempty"` + UnaddressableSample []string `json:"unaddressable_sample,omitempty"` + SkippedReconciled int64 `json:"skipped_reconciled,omitempty"` + UnregisteredSkipped int64 `json:"unregistered_skipped,omitempty"` + UnregisteredSample []string `json:"unregistered_sample,omitempty"` + ManifestOnlyFiles int64 `json:"manifest_only_files,omitempty"` + ManifestOnlySample []string `json:"manifest_only_sample,omitempty"` + LeftManifestDuringRun int64 `json:"left_manifest_during_run,omitempty"` + LeftManifestSample []string `json:"left_manifest_sample,omitempty"` + ClusterManifestChecked *bool `json:"cluster_manifest_checked,omitempty"` + IcebergWarehouse *BackupIcebergWarehouse `json:"iceberg_warehouse,omitempty"` + HasMetadata bool `json:"has_metadata"` + HasIcebergCatalog bool `json:"has_iceberg_catalog,omitempty"` + HasConfig bool `json:"has_config"` + Raw json.RawMessage `json:"-"` } // BackupProgress is GET /api/v1/backup/status when an operation has run @@ -125,10 +137,20 @@ type BackupProgress struct { UnaddressableSample []string `json:"unaddressable_sample,omitempty"` // Restore only: what the restored backup already lacked when it was taken, // and warehouse files left out because this node has no outside-root - // Iceberg warehouse. - BackupSkippedFiles int64 `json:"backup_skipped_files,omitempty"` - BackupUnaddressableFiles int64 `json:"backup_unaddressable_files,omitempty"` - IcebergWarehouseFilesSkipped int64 `json:"iceberg_warehouse_files_skipped,omitempty"` + // Iceberg warehouse. Newer Arc versions also report the selected mode, + // cluster registration outcomes, and files replaced by the restore. + BackupSkippedFiles int64 `json:"backup_skipped_files,omitempty"` + BackupUnaddressableFiles int64 `json:"backup_unaddressable_files,omitempty"` + IcebergWarehouseFilesSkipped int64 `json:"iceberg_warehouse_files_skipped,omitempty"` + Mode string `json:"mode,omitempty"` + FilesRegistered int64 `json:"files_registered,omitempty"` + RegistrationFailed int64 `json:"registration_failed,omitempty"` + RegistrationFailedSample []string `json:"registration_failed_sample,omitempty"` + SidecarMismatches int64 `json:"sidecar_mismatches,omitempty"` + SidecarMismatchSample []string `json:"sidecar_mismatch_sample,omitempty"` + ReplacedFiles int64 `json:"replaced_files,omitempty"` + BackupUnregisteredSkipped int64 `json:"backup_unregistered_skipped,omitempty"` + BackupManifestOnlyFiles int64 `json:"backup_manifest_only_files,omitempty"` } // BackupStatus is the decoded status endpoint: Idle when the server has @@ -147,11 +169,13 @@ type BackupCreateOptions struct { } // RestoreOptions is the body of POST /api/v1/backup/restore. Confirm is -// always sent true by the client; the command layer owns the prompt. +// always sent true by the client; the command layer owns the prompt. A nil +// Mode leaves the effective mode to Arc, including its scoped-backup default. type RestoreOptions struct { RestoreData *bool RestoreMetadata *bool RestoreConfig *bool + Mode *string } // RestoreStarted is the 202 body of POST /api/v1/backup/restore. @@ -160,6 +184,7 @@ type RestoreStarted struct { Message string `json:"message"` BackupID string `json:"backup_id"` Status string `json:"status"` + Mode string `json:"mode,omitempty"` RestartRequired bool `json:"restart_required"` Staged bool `json:"staged"` Raw json.RawMessage `json:"-"` @@ -276,12 +301,13 @@ func (c *Client) RestoreBackup(ctx context.Context, id string, opts RestoreOptio return nil, err } req := struct { - BackupID string `json:"backup_id"` - RestoreData *bool `json:"restore_data,omitempty"` - RestoreMetadata *bool `json:"restore_metadata,omitempty"` - RestoreConfig *bool `json:"restore_config,omitempty"` - Confirm bool `json:"confirm"` - }{id, opts.RestoreData, opts.RestoreMetadata, opts.RestoreConfig, true} + BackupID string `json:"backup_id"` + RestoreData *bool `json:"restore_data,omitempty"` + RestoreMetadata *bool `json:"restore_metadata,omitempty"` + RestoreConfig *bool `json:"restore_config,omitempty"` + Mode *string `json:"mode,omitempty"` + Confirm bool `json:"confirm"` + }{id, opts.RestoreData, opts.RestoreMetadata, opts.RestoreConfig, opts.Mode, true} var out RestoreStarted body, err := c.backupJSON(ctx, http.MethodPost, backupBase+"restore", req, 64<<10, &out) if err != nil { diff --git a/internal/client/backup_incomplete_test.go b/internal/client/backup_incomplete_test.go index 67b1f1f..16f3aa5 100644 --- a/internal/client/backup_incomplete_test.go +++ b/internal/client/backup_incomplete_test.go @@ -15,11 +15,11 @@ func TestBackup_IncompleteShapes(t *testing.T) { cli, _ := newAuthTestClient(t, func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/api/v1/backup/": - _, _ = w.Write([]byte(`{"backups":[{"backup_id":"backup-20260917-000000-aaaaaaaa","created_at":"2026-09-17T00:00:00Z","backup_type":"full","total_files":21,"total_size_bytes":2086,"database_count":1,"skipped_files":3,"skipped_metadata_files":1,"unaddressable_files":2},{"backup_id":"backup-20260916-000000-bbbbbbbb","created_at":"2026-09-16T00:00:00Z","backup_type":"full","total_files":20,"total_size_bytes":2000,"database_count":1}],"count":2}`)) + _, _ = w.Write([]byte(`{"backups":[{"backup_id":"backup-20260917-000000-aaaaaaaa","created_at":"2026-09-17T00:00:00Z","backup_type":"full","total_files":21,"total_size_bytes":2086,"database_count":1,"skipped_files":3,"skipped_metadata_files":1,"unaddressable_files":2,"skipped_reconciled":1,"unregistered_skipped":4,"manifest_only_files":2,"left_manifest_during_run":3,"cluster_manifest_checked":true},{"backup_id":"backup-20260916-000000-bbbbbbbb","created_at":"2026-09-16T00:00:00Z","backup_type":"full","total_files":20,"total_size_bytes":2000,"database_count":1}],"count":2}`)) case "/api/v1/backup/backup-20260917-000000-aaaaaaaa": - _, _ = w.Write([]byte(`{"version":"26.09.3","backup_id":"backup-20260917-000000-aaaaaaaa","created_at":"2026-09-17T00:00:00Z","backup_type":"full","databases":[{"name":"mydb","measurements":[{"name":"cpu","file_count":21,"size_bytes":2086}],"file_count":21,"size_bytes":2086}],"total_files":21,"total_size_bytes":2086,"skipped_files":3,"skipped_metadata_files":1,"skipped_sample":["mydb/cpu/2026/09/17/00/a.parquet","mydb/cpu/2026/09/17/00/b.parquet","` + long + `","mydb/cpu/metadata/00003-5f2c.metadata.json"],"skipped_overlong_keys":1,"unaddressable_files":2,"unaddressable_sample":["mydb/cpu/2026/09/17/00/.hidden.parquet","mydb/cpu/2026/09/17/00/bad key.parquet"],"iceberg_warehouse":{"path":"/srv/wh","file_count":4,"size_bytes":100,"skipped_files":1},"has_metadata":true,"has_config":false}`)) + _, _ = w.Write([]byte(`{"version":"26.09.3","backup_id":"backup-20260917-000000-aaaaaaaa","created_at":"2026-09-17T00:00:00Z","backup_type":"full","databases":[{"name":"mydb","measurements":[{"name":"cpu","file_count":21,"size_bytes":2086}],"file_count":21,"size_bytes":2086}],"total_files":21,"total_size_bytes":2086,"skipped_files":3,"skipped_metadata_files":1,"skipped_sample":["mydb/cpu/2026/09/17/00/a.parquet","mydb/cpu/2026/09/17/00/b.parquet","` + long + `","mydb/cpu/metadata/00003-5f2c.metadata.json"],"skipped_overlong_keys":1,"unaddressable_files":2,"unaddressable_sample":["mydb/cpu/2026/09/17/00/.hidden.parquet","mydb/cpu/2026/09/17/00/bad key.parquet"],"skipped_reconciled":1,"unregistered_skipped":4,"unregistered_sample":["mydb/cpu/unregistered.parquet"],"manifest_only_files":2,"manifest_only_sample":["mydb/cpu/manifest-only.parquet"],"left_manifest_during_run":3,"left_manifest_sample":["mydb/cpu/left-manifest.parquet"],"cluster_manifest_checked":true,"iceberg_warehouse":{"path":"/srv/wh","file_count":4,"size_bytes":100,"skipped_files":1},"has_metadata":true,"has_config":false}`)) case "/api/v1/backup/status": - _, _ = w.Write([]byte(`{"operation":"restore","backup_id":"backup-20260917-000000-aaaaaaaa","status":"failed","total_files":17,"processed_files":16,"skipped_files":1,"unaddressable_files":1,"unaddressable_sample":["backup-20260917-000000-aaaaaaaa/data/mydb/cpu/2026/09/17/00/.hidden.parquet"],"skipped_sample":["backup-20260917-000000-aaaaaaaa/data/mydb/cpu/2026/09/17/00/c.parquet"],"missing_files":1,"backup_skipped_files":3,"backup_unaddressable_files":2,"iceberg_warehouse_files_skipped":4,"total_bytes":2086,"processed_bytes":1600,"started_at":"2026-09-17T01:00:00Z","completed_at":"2026-09-17T01:00:02Z","error":"restore incomplete: 1 objects could not be read from backup storage (skipped_sample)"}`)) + _, _ = w.Write([]byte(`{"operation":"restore","backup_id":"backup-20260917-000000-aaaaaaaa","status":"failed","total_files":17,"processed_files":16,"skipped_files":1,"unaddressable_files":1,"unaddressable_sample":["backup-20260917-000000-aaaaaaaa/data/mydb/cpu/2026/09/17/00/.hidden.parquet"],"skipped_sample":["backup-20260917-000000-aaaaaaaa/data/mydb/cpu/2026/09/17/00/c.parquet"],"missing_files":1,"backup_skipped_files":2,"backup_unaddressable_files":2,"backup_unregistered_skipped":4,"backup_manifest_only_files":2,"mode":"replace","files_registered":15,"registration_failed":1,"registration_failed_sample":["mydb/cpu/unregistered.parquet"],"sidecar_mismatches":2,"sidecar_mismatch_sample":["mydb/cpu/sidecar-mismatch.parquet"],"replaced_files":8,"iceberg_warehouse_files_skipped":4,"total_bytes":2086,"processed_bytes":1600,"started_at":"2026-09-17T01:00:00Z","completed_at":"2026-09-17T01:00:02Z","error":"restore incomplete: 1 objects could not be read from backup storage (skipped_sample)"}`)) } }) ctx := context.Background() @@ -27,22 +27,31 @@ func TestBackup_IncompleteShapes(t *testing.T) { if err != nil || len(list) != 2 { t.Fatalf("list=%+v err=%v", list, err) } - if list[0].SkippedFiles != 3 || list[0].SkippedMetadataFiles != 1 || list[0].UnaddressableFiles != 2 { + if list[0].SkippedFiles != 3 || list[0].SkippedMetadataFiles != 1 || list[0].UnaddressableFiles != 2 || + list[0].SkippedReconciled != 1 || list[0].UnregisteredSkipped != 4 || list[0].ManifestOnlyFiles != 2 || + list[0].LeftManifestDuringRun != 3 || list[0].ClusterManifestChecked == nil || !*list[0].ClusterManifestChecked { t.Errorf("incomplete entry = %+v", list[0]) } - if list[1].SkippedFiles != 0 || list[1].SkippedMetadataFiles != 0 || list[1].UnaddressableFiles != 0 { + if list[1].SkippedFiles != 0 || list[1].SkippedMetadataFiles != 0 || list[1].UnaddressableFiles != 0 || + list[1].SkippedReconciled != 0 || list[1].UnregisteredSkipped != 0 || list[1].ManifestOnlyFiles != 0 || + list[1].LeftManifestDuringRun != 0 || list[1].ClusterManifestChecked != nil { t.Errorf("entry without the keys must read zero: %+v", list[1]) } m, err := cli.GetBackup(ctx, "backup-20260917-000000-aaaaaaaa") if err != nil { t.Fatal(err) } - if m.SkippedFiles != 3 || m.SkippedMetadataFiles != 1 || m.SkippedOverlongKeys != 1 || m.UnaddressableFiles != 2 { + if m.SkippedFiles != 3 || m.SkippedMetadataFiles != 1 || m.SkippedOverlongKeys != 1 || m.UnaddressableFiles != 2 || + m.SkippedReconciled != 1 || m.UnregisteredSkipped != 4 || m.ManifestOnlyFiles != 2 || m.LeftManifestDuringRun != 3 || + m.ClusterManifestChecked == nil || !*m.ClusterManifestChecked { t.Errorf("manifest counts = %+v", m) } if len(m.SkippedSample) != 4 || m.SkippedSample[2] != long || len(m.UnaddressableSample) != 2 { t.Errorf("manifest samples = %v / %v", m.SkippedSample, m.UnaddressableSample) } + if len(m.UnregisteredSample) != 1 || len(m.ManifestOnlySample) != 1 || len(m.LeftManifestSample) != 1 { + t.Errorf("cluster manifest samples = %v / %v / %v", m.UnregisteredSample, m.ManifestOnlySample, m.LeftManifestSample) + } if m.IcebergWarehouse == nil || m.IcebergWarehouse.SkippedFiles != 1 || m.IcebergWarehouse.Path != "/srv/wh" { t.Errorf("iceberg warehouse = %+v", m.IcebergWarehouse) } @@ -52,7 +61,10 @@ func TestBackup_IncompleteShapes(t *testing.T) { } p := st.Progress if len(p.SkippedSample) != 1 || len(p.UnaddressableSample) != 1 || p.UnaddressableFiles != 1 || - p.BackupSkippedFiles != 3 || p.BackupUnaddressableFiles != 2 || p.IcebergWarehouseFilesSkipped != 4 { + p.BackupSkippedFiles != 2 || p.BackupUnaddressableFiles != 2 || p.IcebergWarehouseFilesSkipped != 4 || + p.Mode != "replace" || p.FilesRegistered != 15 || p.RegistrationFailed != 1 || len(p.RegistrationFailedSample) != 1 || + p.SidecarMismatches != 2 || len(p.SidecarMismatchSample) != 1 || p.ReplacedFiles != 8 || + p.BackupUnregisteredSkipped != 4 || p.BackupManifestOnlyFiles != 2 { t.Errorf("restore progress = %+v", p) } } diff --git a/internal/client/backup_test.go b/internal/client/backup_test.go index 55bd7a5..8f4d8f7 100644 --- a/internal/client/backup_test.go +++ b/internal/client/backup_test.go @@ -190,7 +190,7 @@ func TestBackup_StatusIdleAndRestoreBody(t *testing.T) { case r.URL.Path == "/api/v1/backup/restore": _ = json.NewDecoder(r.Body).Decode(&raw) w.WriteHeader(202) - _, _ = w.Write([]byte(`{"backup_id":"backup-20260907-201105-a0f5e600","message":"Restore started","status":"running","restart_required":true,"staged":true}`)) + _, _ = w.Write([]byte(`{"backup_id":"backup-20260907-201105-a0f5e600","message":"Restore started","status":"running","mode":"replace","restart_required":true,"staged":true}`)) } }) ctx := context.Background() @@ -200,7 +200,7 @@ func TestBackup_StatusIdleAndRestoreBody(t *testing.T) { } f := false res, err := cli.RestoreBackup(ctx, "backup-20260907-201105-a0f5e600", RestoreOptions{RestoreMetadata: &f, RestoreConfig: &f}) - if err != nil || !res.RestartRequired || !res.Staged { + if err != nil || !res.RestartRequired || !res.Staged || res.Mode != "replace" { t.Errorf("res=%+v err=%v", res, err) } if raw["confirm"] != true || raw["restore_metadata"] != false || raw["restore_config"] != false { @@ -209,6 +209,14 @@ func TestBackup_StatusIdleAndRestoreBody(t *testing.T) { if _, has := raw["restore_data"]; has { t.Error("nil restore_data must be omitted") } + if _, has := raw["mode"]; has { + t.Error("nil mode must be omitted so the server can apply its default") + } + mode := "replace" + res, err = cli.RestoreBackup(ctx, "backup-20260907-201105-a0f5e600", RestoreOptions{Mode: &mode}) + if err != nil || res.Mode != "replace" || raw["mode"] != "replace" { + t.Errorf("explicit mode: request=%v response=%+v err=%v", raw, res, err) + } } func TestBackup_DisabledServer(t *testing.T) { diff --git a/internal/commands/backup.go b/internal/commands/backup.go index 192ad40..a0b06d1 100644 --- a/internal/commands/backup.go +++ b/internal/commands/backup.go @@ -152,6 +152,7 @@ func describeProgress(w io.Writer, p *client.BackupProgress) { if p.Error != "" { fmt.Fprintf(w, "error: %s\n", clean(p.Error)) } + writeRestoreDetails(w, p) // What the counts above do not say: which files, what the restored backup // already lacked, and warehouse files a restore had nowhere to put. The // server already names the counts of a failed restore in its error, so @@ -171,14 +172,40 @@ func describeProgress(w io.Writer, p *client.BackupProgress) { writeSample(w, "unaddressable", p.UnaddressableSample) } +// writeRestoreDetails prints the additional outcomes Arc reports for a +// restore. Older servers omit these fields, so an otherwise empty line is +// not emitted for them. +func writeRestoreDetails(w io.Writer, p *client.BackupProgress) { + if p.Operation != "restore" { + return + } + if p.Mode != "" { + fmt.Fprintf(w, "mode: %s\n", clean(p.Mode)) + } + if p.Mode != "" || p.FilesRegistered > 0 || p.RegistrationFailed > 0 || p.SidecarMismatches > 0 || p.ReplacedFiles > 0 { + fmt.Fprintf(w, "restore: %d files registered, %d registration failures, %d sidecar mismatches, %d files replaced\n", + p.FilesRegistered, p.RegistrationFailed, p.SidecarMismatches, p.ReplacedFiles) + } + writeSample(w, "registration-failed", p.RegistrationFailedSample) + writeSample(w, "sidecar-mismatch", p.SidecarMismatchSample) +} + // writeRestoreGaps prints what a restore's progress says about gaps that are // not the restore's own doing: what the restored backup already lacked when it // was taken, and warehouse files this node had nowhere to put. Printed for a // failed and a completed restore alike, since the latter is the only place the // warehouse gap shows when Iceberg is off on the node. func writeRestoreGaps(w io.Writer, p *client.BackupProgress) { - if p.BackupSkippedFiles > 0 || p.BackupUnaddressableFiles > 0 { - fmt.Fprintf(w, "backup had: %s when it was taken\n", describeIncomplete(p.BackupSkippedFiles, 0, p.BackupUnaddressableFiles)) + backupGaps := incompleteCounts{ + Skipped: p.BackupSkippedFiles, + Unaddressable: p.BackupUnaddressableFiles, + ManifestOnly: p.BackupManifestOnlyFiles, + } + if gaps := describeIncomplete(backupGaps); gaps != "" { + fmt.Fprintf(w, "backup had: %s when it was taken\n", gaps) + } + if p.BackupUnregisteredSkipped > 0 { + fmt.Fprintf(w, "backup info: %d files were not listed in the cluster manifest when it was taken\n", p.BackupUnregisteredSkipped) } if p.IcebergWarehouseFilesSkipped > 0 { fmt.Fprintf(w, "iceberg: %d warehouse files not restored (this node has no outside-root warehouse)\n", p.IcebergWarehouseFilesSkipped) @@ -202,26 +229,83 @@ func wasWere(n int64) string { return "were" } -// describeIncomplete phrases a backup's incompleteness counts, each in its own -// domain because the server keeps them apart: skipped data files are part of -// the backup's file total, metadata skips (Iceberg metadata, compaction -// recovery state) are not, and unaddressable files were never inventoried at -// all, so the numbers are never summed. Returns "" when all are zero, which is -// also what an Arc older than 26.09.3 reports: none, not necessarily complete. -func describeIncomplete(skipped, metadata, unaddressable int64) string { +type incompleteCounts struct { + Skipped int64 + SkippedReconciled int64 + Metadata int64 + Unaddressable int64 + ManifestOnly int64 +} + +func effectiveSkipped(skipped, reconciled int64) int64 { + if reconciled >= skipped { + return 0 + } + return skipped - reconciled +} + +// describeIncomplete keeps the server's distinct missing-file populations +// separate. Skips later reconciled against the cluster manifest are not +// missing data, and manifest-only files are a separate source of incompleteness. +func describeIncomplete(c incompleteCounts) string { var parts []string - if skipped > 0 { + if skipped := effectiveSkipped(c.Skipped, c.SkippedReconciled); skipped > 0 { parts = append(parts, fmt.Sprintf("%d skipped", skipped)) } - if metadata > 0 { - parts = append(parts, fmt.Sprintf("%d metadata", metadata)) + if c.Metadata > 0 { + parts = append(parts, fmt.Sprintf("%d metadata", c.Metadata)) + } + if c.Unaddressable > 0 { + parts = append(parts, fmt.Sprintf("%d unaddressable", c.Unaddressable)) } - if unaddressable > 0 { - parts = append(parts, fmt.Sprintf("%d unaddressable", unaddressable)) + if c.ManifestOnly > 0 { + parts = append(parts, fmt.Sprintf("%d manifest-only", c.ManifestOnly)) } return strings.Join(parts, ", ") } +func describeClusterBackupInfo(reconciled, unregistered, leftManifest int64, checked *bool) string { + var parts []string + if reconciled > 0 { + parts = append(parts, fmt.Sprintf("%d skipped reconciled", reconciled)) + } + if unregistered > 0 { + parts = append(parts, fmt.Sprintf("%d unregistered", unregistered)) + } + if leftManifest > 0 { + parts = append(parts, fmt.Sprintf("%d left manifest during backup", leftManifest)) + } + if checked != nil { + if *checked { + parts = append(parts, "cluster manifest checked") + } else { + parts = append(parts, "cluster manifest not checked") + } + } + return strings.Join(parts, ", ") +} + +func backupListIncomplete(b client.BackupSummary) string { + incomplete := describeIncomplete(incompleteCounts{ + Skipped: b.SkippedFiles, + SkippedReconciled: b.SkippedReconciled, + Metadata: b.SkippedMetadataFiles, + Unaddressable: b.UnaddressableFiles, + ManifestOnly: b.ManifestOnlyFiles, + }) + info := describeClusterBackupInfo(b.SkippedReconciled, b.UnregisteredSkipped, b.LeftManifestDuringRun, b.ClusterManifestChecked) + if incomplete == "" { + if info == "" { + return "-" + } + return "info: " + info + } + if info != "" { + return incomplete + "; info: " + info + } + return incomplete +} + // writeSample prints one labelled line per path (up to the server's 32). func writeSample(w io.Writer, label string, paths []string) { for _, p := range paths { @@ -379,16 +463,14 @@ func newBackupListCmd() *cobra.Command { Short: "List backups, newest first", Long: `List backups (GET /api/v1/backup/), newest first. -INCOMPLETE says what the backup lacks, as the server counts it (Arc -26.09.3+): "skipped" data files that were inventoried but not stored, -"metadata" files (Iceberg metadata, compaction recovery state) that were -skipped, and "unaddressable" files whose key no listing can return, so -they were never inventoried. "-" means none reported: a complete backup, -or an Arc older than 26.09.3. "backup show" names the files. +INCOMPLETE includes skipped data files that remain absent from the cluster +manifest, manifest-only files, metadata skips, and unaddressable files. +Reconciled skips, unregistered local files, and files removed from the +manifest during backup are shown as information, not missing backup data. +"-" means no gaps were reported; "backup show" names sampled files. -Output formats: table (default) | json | csv. The csv columns -skipped_files, skipped_metadata_files and unaddressable_files follow -total_size_bytes.`, +Output formats: table (default) | json | csv. The csv output includes the +incompleteness and cluster-manifest counts returned by the server.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { if !validListFormat(outputFormat) { @@ -412,12 +494,16 @@ total_size_bytes.`, if outputFormat == output.FormatCSV { cw := csv.NewWriter(w) if !noHeader { - if err := cw.Write([]string{"backup_id", "created_at", "backup_type", "database_count", "total_files", "total_size_bytes", "skipped_files", "skipped_metadata_files", "unaddressable_files"}); err != nil { + if err := cw.Write([]string{"backup_id", "created_at", "backup_type", "database_count", "total_files", "total_size_bytes", "skipped_files", "skipped_metadata_files", "unaddressable_files", "skipped_reconciled", "unregistered_skipped", "manifest_only_files", "left_manifest_during_run", "cluster_manifest_checked"}); err != nil { return err } } for _, b := range list { - if err := cw.Write([]string{b.BackupID, fmtTimeVal(b.CreatedAt, ""), b.BackupType, strconv.Itoa(b.DatabaseCount), strconv.FormatInt(b.TotalFiles, 10), strconv.FormatInt(b.TotalBytes, 10), strconv.FormatInt(b.SkippedFiles, 10), strconv.FormatInt(b.SkippedMetadataFiles, 10), strconv.FormatInt(b.UnaddressableFiles, 10)}); err != nil { + checked := "" + if b.ClusterManifestChecked != nil { + checked = strconv.FormatBool(*b.ClusterManifestChecked) + } + if err := cw.Write([]string{b.BackupID, fmtTimeVal(b.CreatedAt, ""), b.BackupType, strconv.Itoa(b.DatabaseCount), strconv.FormatInt(b.TotalFiles, 10), strconv.FormatInt(b.TotalBytes, 10), strconv.FormatInt(b.SkippedFiles, 10), strconv.FormatInt(b.SkippedMetadataFiles, 10), strconv.FormatInt(b.UnaddressableFiles, 10), strconv.FormatInt(b.SkippedReconciled, 10), strconv.FormatInt(b.UnregisteredSkipped, 10), strconv.FormatInt(b.ManifestOnlyFiles, 10), strconv.FormatInt(b.LeftManifestDuringRun, 10), checked}); err != nil { return err } } @@ -430,11 +516,7 @@ total_size_bytes.`, } rows := make([][]string, 0, len(list)) for _, b := range list { - incomplete := describeIncomplete(b.SkippedFiles, b.SkippedMetadataFiles, b.UnaddressableFiles) - if incomplete == "" { - incomplete = "-" - } - rows = append(rows, []string{clean(b.BackupID), fmtTimeVal(b.CreatedAt, "-"), clean(b.BackupType), strconv.Itoa(b.DatabaseCount), strconv.FormatInt(b.TotalFiles, 10), humanBytes(b.TotalBytes), incomplete}) + rows = append(rows, []string{clean(b.BackupID), fmtTimeVal(b.CreatedAt, "-"), clean(b.BackupType), strconv.Itoa(b.DatabaseCount), strconv.FormatInt(b.TotalFiles, 10), humanBytes(b.TotalBytes), backupListIncomplete(b)}) } headers := []string{"ID", "CREATED", "TYPE", "DATABASES", "FILES", "SIZE", "INCOMPLETE"} if noHeader { @@ -494,9 +576,13 @@ func writeManifest(w io.Writer, m *client.BackupManifest) error { fmt.Fprintf(w, "contents: %d files, %s, metadata %t, config %t\n", m.TotalFiles, humanBytes(m.TotalSizeBytes), m.HasMetadata, m.HasConfig) if line := describeManifestIncomplete(m); line != "" { fmt.Fprintf(w, "INCOMPLETE: %s\n", line) - writeSample(w, "skipped", m.SkippedSample) - writeSample(w, "unaddressable", m.UnaddressableSample) } + writeSample(w, "skipped", m.SkippedSample) + writeSample(w, "unaddressable", m.UnaddressableSample) + writeManifestClusterInfo(w, m) + writeSample(w, "unregistered", m.UnregisteredSample) + writeSample(w, "manifest-only", m.ManifestOnlySample) + writeSample(w, "left-manifest", m.LeftManifestSample) if len(m.Databases) == 0 { _, err := fmt.Fprintln(w, "(no databases)") return err @@ -514,6 +600,25 @@ func writeManifest(w io.Writer, m *client.BackupManifest) error { return output.Table(w, []string{"DATABASE", "MEASUREMENT", "FILES", "SIZE"}, rows) } +func writeManifestClusterInfo(w io.Writer, m *client.BackupManifest) { + if m.SkippedReconciled > 0 { + fmt.Fprintf(w, "reconciled: %s later absent from the manifest; not incomplete\n", plural(m.SkippedReconciled, "skipped file was", "skipped files were")) + } + if m.UnregisteredSkipped > 0 { + fmt.Fprintf(w, "unregistered: %d local data files were not in the cluster manifest and were not backed up\n", m.UnregisteredSkipped) + } + if m.LeftManifestDuringRun > 0 { + fmt.Fprintf(w, "left manifest: %d files were copied then removed as the manifest changed during backup\n", m.LeftManifestDuringRun) + } + if m.ClusterManifestChecked != nil { + if *m.ClusterManifestChecked { + fmt.Fprintln(w, "cluster manifest: checked") + } else { + fmt.Fprintln(w, "cluster manifest: not checked") + } + } +} + // describeManifestIncomplete phrases a manifest's incompleteness, or returns // "" for a backup that reports none. The skipped clause counts data files // against the backup's file total and metadata files apart (they are not in @@ -521,21 +626,22 @@ func writeManifest(w io.Writer, m *client.BackupManifest) error { func describeManifestIncomplete(m *client.BackupManifest) string { var clauses []string var skipped []string - if m.SkippedFiles > 0 { + missingSkipped := effectiveSkipped(m.SkippedFiles, m.SkippedReconciled) + if missingSkipped > 0 { // A manifest written before the data/metadata split folded metadata // skips into skipped_files, so the count can exceed the total; then // "N of M" would read as nonsense. - if m.SkippedFiles > m.TotalFiles { - skipped = append(skipped, plural(m.SkippedFiles, "file", "files")) + if missingSkipped > m.TotalFiles { + skipped = append(skipped, plural(missingSkipped, "file", "files")) } else { - skipped = append(skipped, fmt.Sprintf("%d of %d files", m.SkippedFiles, m.TotalFiles)) + skipped = append(skipped, fmt.Sprintf("%d of %d files", missingSkipped, m.TotalFiles)) } } if m.SkippedMetadataFiles > 0 { skipped = append(skipped, plural(m.SkippedMetadataFiles, "metadata file", "metadata files")) } if len(skipped) > 0 { - clause := strings.Join(skipped, " and ") + " " + wasWere(m.SkippedFiles+m.SkippedMetadataFiles) + " skipped while backing up" + clause := strings.Join(skipped, " and ") + " " + wasWere(missingSkipped+m.SkippedMetadataFiles) + " skipped while backing up" if m.SkippedOverlongKeys > 0 { clause += " (" + overlongClause(m.SkippedOverlongKeys) + ")" } @@ -544,6 +650,9 @@ func describeManifestIncomplete(m *client.BackupManifest) string { if m.UnaddressableFiles > 0 { clauses = append(clauses, plural(m.UnaddressableFiles, "file", "files")+" could not be listed (unaddressable)") } + if m.ManifestOnlyFiles > 0 { + clauses = append(clauses, plural(m.ManifestOnlyFiles, "file was", "files were")+" present only in the cluster manifest") + } if m.IcebergWarehouse != nil && m.IcebergWarehouse.SkippedFiles > 0 { clauses = append(clauses, plural(m.IcebergWarehouse.SkippedFiles, "Iceberg warehouse file was", "Iceberg warehouse files were")+" skipped") } @@ -558,14 +667,15 @@ func describeManifestIncomplete(m *client.BackupManifest) string { func describeRestoreGaps(m *client.BackupManifest) string { var clauses []string var skipped []string - if m.SkippedFiles > 0 { - skipped = append(skipped, plural(m.SkippedFiles, "file", "files")) + missingSkipped := effectiveSkipped(m.SkippedFiles, m.SkippedReconciled) + if missingSkipped > 0 { + skipped = append(skipped, plural(missingSkipped, "file", "files")) } if m.SkippedMetadataFiles > 0 { skipped = append(skipped, plural(m.SkippedMetadataFiles, "metadata file", "metadata files")) } if len(skipped) > 0 { - clause := strings.Join(skipped, " and ") + " " + wasWere(m.SkippedFiles+m.SkippedMetadataFiles) + " skipped when it was taken" + clause := strings.Join(skipped, " and ") + " " + wasWere(missingSkipped+m.SkippedMetadataFiles) + " skipped when it was taken" if m.SkippedOverlongKeys > 0 { clause += ", " + strings.Replace(overlongClause(m.SkippedOverlongKeys), " for ", " of them for ", 1) } @@ -574,6 +684,9 @@ func describeRestoreGaps(m *client.BackupManifest) string { if m.UnaddressableFiles > 0 { clauses = append(clauses, plural(m.UnaddressableFiles, "file", "files")+" could not be listed") } + if m.ManifestOnlyFiles > 0 { + clauses = append(clauses, plural(m.ManifestOnlyFiles, "file was", "files were")+" present only in the cluster manifest") + } if m.IcebergWarehouse != nil && m.IcebergWarehouse.SkippedFiles > 0 { clauses = append(clauses, plural(m.IcebergWarehouse.SkippedFiles, "Iceberg warehouse file was", "Iceberg warehouse files were")+" skipped") } @@ -692,6 +805,7 @@ func newBackupRestoreCmd() *cobra.Command { var ( f connFlags outputFormat string + mode string dataOnly bool withMetadata bool withConfig bool @@ -705,8 +819,8 @@ func newBackupRestoreCmd() *cobra.Command { Data files from the backup are written over the server's storage unconditionally; existing files at the same paths are replaced. Run it -on a quiescent server: stop writers and compaction first. In cluster -mode restored files are not registered in the cluster manifest. +on a quiescent server: stop writers and compaction first. On a cluster, +Arc registers restored files in the cluster manifest and reports failures. By default the metadata store (tokens, policies, continuous queries) is restored on a standalone node if the backup contains it, but skipped on a @@ -717,6 +831,12 @@ also restores the server config file (staged as well). The restore runs in the background; --wait polls until it finishes (--wait-timeout defaults to 2h). +The restore mode is merge by default for a full backup. Arc may default a +scoped cluster backup to replace; when --mode is omitted, arcli leaves that +choice to the server. Use --mode merge or --mode replace to choose explicitly. +Replace is available only on clusters and removes the current manifest entries +for the backup's databases before restoring them. + On a cluster, only the primary writer can restore a backup. A node that is not the primary writer returns HTTP 503 after confirmation, before the restore starts.`, @@ -730,6 +850,9 @@ before the restore starts.`, if waitTimeout <= 0 { return fmt.Errorf("--wait-timeout must be > 0") } + if mode != "merge" && mode != "replace" { + return fmt.Errorf("invalid --mode %q (valid: merge, replace)", mode) + } if dataOnly && withConfig { return fmt.Errorf("--data-only and --with-config are mutually exclusive") } @@ -773,6 +896,11 @@ before the restore starts.`, scope = "the live storage of database(s) " + strings.Join(dbs, ", ") } parts := []string{fmt.Sprintf("Restore backup %s (%s, %d files, %s) over %s? Existing files at the same paths are replaced.", clean(id), fmtTimeVal(m.CreatedAt, "-"), m.TotalFiles, humanBytes(m.TotalSizeBytes), scope)} + if cmd.Flags().Changed("mode") && mode == "replace" { + parts = append(parts, "The current cluster manifest entries for these databases will be removed before restoring.") + } else if !cmd.Flags().Changed("mode") { + parts = append(parts, "The restore mode follows Arc's default: merge for a full backup, or replace for a scoped backup on a cluster.") + } if restoreMeta { parts = append(parts, "The metadata store will be staged and applied at the next server start.") } else if metadataMayRestoreByDefault { @@ -806,6 +934,9 @@ before the restore starts.`, if restoreCfg { opts.RestoreConfig = &t } + if cmd.Flags().Changed("mode") { + opts.Mode = &mode + } ctx, cancel = f.ctx(cmd) res, err := cli.RestoreBackup(ctx, id, opts) cancel() @@ -829,6 +960,8 @@ before the restore starts.`, if err := writeRawJSON(cmd.OutOrStdout(), res.Raw); err != nil { return err } + } else if res.Mode != "" { + fmt.Fprintf(cmd.OutOrStdout(), "Restore of %s accepted (mode: %s)\n", clean(id), clean(res.Mode)) } restartNote() return nil @@ -839,7 +972,15 @@ before the restore starts.`, return err } } else { - fmt.Fprintf(cmd.OutOrStdout(), "Restore of %s started\n", clean(id)) + startedMode := res.Mode + if startedMode == "" && cmd.Flags().Changed("mode") { + startedMode = mode + } + if startedMode == "" { + fmt.Fprintf(cmd.OutOrStdout(), "Restore of %s started\n", clean(id)) + } else { + fmt.Fprintf(cmd.OutOrStdout(), "Restore of %s started (mode: %s)\n", clean(id), clean(startedMode)) + } } fmt.Fprintln(stderr, "It runs in the background; follow it with `arcli backup status` or re-run with --wait.") restartNote() @@ -855,8 +996,20 @@ before the restore starts.`, return err } } else if final.Status == "completed" { - fmt.Fprintf(cmd.OutOrStdout(), "Restore of %s completed: %d files, %s written\n", clean(id), final.ProcessedFiles, humanBytes(final.ProcessedBytes)) + completedMode := final.Mode + if completedMode == "" { + completedMode = res.Mode + } + if completedMode == "" && cmd.Flags().Changed("mode") { + completedMode = mode + } + if completedMode == "" { + fmt.Fprintf(cmd.OutOrStdout(), "Restore of %s completed: %d files, %s written\n", clean(id), final.ProcessedFiles, humanBytes(final.ProcessedBytes)) + } else { + fmt.Fprintf(cmd.OutOrStdout(), "Restore of %s completed: %d files, %s written (mode: %s)\n", clean(id), final.ProcessedFiles, humanBytes(final.ProcessedBytes), clean(completedMode)) + } writeRestoreGaps(cmd.OutOrStdout(), final) + writeRestoreDetails(cmd.OutOrStdout(), final) } else { describeProgress(cmd.OutOrStdout(), final) } @@ -869,6 +1022,7 @@ before the restore starts.`, } f.add(c) c.Flags().StringVarP(&outputFormat, "output", "o", output.FormatTable, "output format: table|json") + c.Flags().StringVar(&mode, "mode", "merge", "restore mode: merge|replace (default: merge)") c.Flags().BoolVar(&dataOnly, "data-only", false, "restore data files only (skip metadata and config)") c.Flags().BoolVar(&withMetadata, "with-metadata", false, "also restore the metadata store (staged; unavailable on clusters)") c.Flags().BoolVar(&withConfig, "with-config", false, "also restore the server config file (staged)") diff --git a/internal/commands/backup_incomplete_test.go b/internal/commands/backup_incomplete_test.go index 8fb2990..6f173a5 100644 --- a/internal/commands/backup_incomplete_test.go +++ b/internal/commands/backup_incomplete_test.go @@ -8,16 +8,23 @@ import ( "testing" ) -// Fixtures in the shapes Arc 26.09.3 (arc#977) produces: a backup with 21 -// data files of which 3 were skipped (1 for an overlong key), 1 metadata file -// skipped, 2 unaddressable files; its completed backup status; and a failed -// restore of it. This file uses only symbols that exist before the arcli -// change, so it compiles against the old renderings and fails there. +// Fixtures cover the Arc 26.09.3 incompleteness response and the Arc 27.01.1 +// cluster-manifest and restore-progress fields. const ( incompleteID = "backup-20260917-000000-aaaaaaaa" metadataOnlyID = "backup-20260915-000000-cccccccc" ) +func TestBackup_ReconciledSkipsAreNotIncomplete(t *testing.T) { + if got := describeIncomplete(incompleteCounts{Skipped: 1, SkippedReconciled: 2}); got != "" { + t.Fatalf("reconciled skips counted as incomplete: %q", got) + } + checked := false + if got := describeClusterBackupInfo(0, 0, 0, &checked); got != "cluster manifest not checked" { + t.Fatalf("unchecked manifest status = %q", got) + } +} + func newIncompleteBackupServer(t *testing.T) *httptest.Server { t.Helper() long := "mydb/cpu/2026/09/17/00/" + strings.Repeat("x", 960) + ".parquet" @@ -30,10 +37,10 @@ func newIncompleteBackupServer(t *testing.T) *httptest.Server { rest := strings.TrimPrefix(r.URL.Path, "/api/v1/backup/") switch { case rest == "" && r.Method == http.MethodGet: - _, _ = w.Write([]byte(`{"backups":[{"backup_id":"` + incompleteID + `","created_at":"2026-09-17T00:00:00Z","backup_type":"full","total_files":21,"total_size_bytes":2086,"database_count":1,"skipped_files":3,"skipped_metadata_files":1,"unaddressable_files":2},{"backup_id":"backup-20260916-000000-bbbbbbbb","created_at":"2026-09-16T00:00:00Z","backup_type":"full","total_files":20,"total_size_bytes":2000,"database_count":1},{"backup_id":"` + metadataOnlyID + `","created_at":"2026-09-15T00:00:00Z","backup_type":"full","total_files":20,"total_size_bytes":2000,"database_count":1,"skipped_metadata_files":1}],"count":3}`)) + _, _ = w.Write([]byte(`{"backups":[{"backup_id":"` + incompleteID + `","created_at":"2026-09-17T00:00:00Z","backup_type":"full","total_files":21,"total_size_bytes":2086,"database_count":1,"skipped_files":3,"skipped_metadata_files":1,"unaddressable_files":2,"skipped_reconciled":1,"unregistered_skipped":4,"manifest_only_files":2,"left_manifest_during_run":3,"cluster_manifest_checked":true},{"backup_id":"backup-20260916-000000-bbbbbbbb","created_at":"2026-09-16T00:00:00Z","backup_type":"full","total_files":20,"total_size_bytes":2000,"database_count":1},{"backup_id":"` + metadataOnlyID + `","created_at":"2026-09-15T00:00:00Z","backup_type":"full","total_files":20,"total_size_bytes":2000,"database_count":1,"skipped_metadata_files":1}],"count":3}`)) case rest == "status": if restored.Load() { - _, _ = w.Write([]byte(`{"operation":"restore","backup_id":"` + incompleteID + `","status":"failed","total_files":17,"processed_files":16,"skipped_files":1,"unaddressable_files":1,"unaddressable_sample":["` + incompleteID + `/data/mydb/cpu/2026/09/17/00/.hidden.parquet"],"skipped_sample":["` + incompleteID + `/data/mydb/cpu/2026/09/17/00/c.parquet"],"missing_files":1,"backup_skipped_files":3,"backup_unaddressable_files":2,"iceberg_warehouse_files_skipped":4,"total_bytes":2086,"processed_bytes":1600,"started_at":"2026-09-17T01:00:00Z","completed_at":"2026-09-17T01:00:02Z","error":"restore incomplete: 1 objects could not be read from backup storage (skipped_sample); 1 data files are in backup storage under names no listing returns (unaddressable_sample; rename them and re-run); 1 data files the backup inventoried are absent from backup storage (missing_files); the files that could be restored are in place"}`)) + _, _ = w.Write([]byte(`{"operation":"restore","backup_id":"` + incompleteID + `","status":"failed","total_files":17,"processed_files":16,"skipped_files":1,"unaddressable_files":1,"unaddressable_sample":["` + incompleteID + `/data/mydb/cpu/2026/09/17/00/.hidden.parquet"],"skipped_sample":["` + incompleteID + `/data/mydb/cpu/2026/09/17/00/c.parquet"],"missing_files":1,"backup_skipped_files":2,"backup_unaddressable_files":2,"backup_unregistered_skipped":4,"backup_manifest_only_files":2,"mode":"replace","files_registered":15,"registration_failed":1,"registration_failed_sample":["mydb/cpu/registration-failed.parquet"],"sidecar_mismatches":2,"sidecar_mismatch_sample":["mydb/cpu/sidecar-mismatch.parquet"],"replaced_files":8,"iceberg_warehouse_files_skipped":4,"total_bytes":2086,"processed_bytes":1600,"started_at":"2026-09-17T01:00:00Z","completed_at":"2026-09-17T01:00:02Z","error":"restore incomplete: 1 objects could not be read from backup storage (skipped_sample); 1 data files are in backup storage under names no listing returns (unaddressable_sample; rename them and re-run); 1 data files the backup inventoried are absent from backup storage (missing_files); the files that could be restored are in place"}`)) return } _, _ = w.Write([]byte(`{"operation":"backup","backup_id":"` + incompleteID + `","status":"completed","total_files":23,"processed_files":19,"skipped_files":4,"unaddressable_files":2,"skipped_sample":["mydb/cpu/2026/09/17/00/a.parquet","mydb/cpu/2026/09/17/00/b.parquet","` + long + `","mydb/cpu/metadata/00003-5f2c.metadata.json"],"total_bytes":2086,"processed_bytes":1790,"started_at":"2026-09-17T00:00:00Z","completed_at":"2026-09-17T00:00:03Z"}`)) @@ -44,7 +51,7 @@ func newIncompleteBackupServer(t *testing.T) *httptest.Server { case rest == metadataOnlyID && r.Method == http.MethodGet: _, _ = w.Write([]byte(`{"version":"26.09.3","backup_id":"` + metadataOnlyID + `","created_at":"2026-09-15T00:00:00Z","backup_type":"full","databases":[{"name":"mydb","measurements":[{"name":"cpu","file_count":20,"size_bytes":2000}],"file_count":20,"size_bytes":2000}],"total_files":20,"total_size_bytes":2000,"skipped_metadata_files":1,"skipped_sample":["mydb/cpu/metadata/00004-aa11.metadata.json"],"skipped_overlong_keys":1,"has_metadata":true,"has_config":false}`)) case rest == incompleteID && r.Method == http.MethodGet: - _, _ = w.Write([]byte(`{"version":"26.09.3","backup_id":"` + incompleteID + `","created_at":"2026-09-17T00:00:00Z","backup_type":"full","databases":[{"name":"mydb","measurements":[{"name":"cpu","file_count":21,"size_bytes":2086}],"file_count":21,"size_bytes":2086}],"total_files":21,"total_size_bytes":2086,"skipped_files":3,"skipped_metadata_files":1,"skipped_sample":["mydb/cpu/2026/09/17/00/a.parquet","mydb/cpu/2026/09/17/00/b.parquet","` + long + `","mydb/cpu/metadata/00003-5f2c.metadata.json"],"skipped_overlong_keys":1,"unaddressable_files":2,"unaddressable_sample":["mydb/cpu/2026/09/17/00/.hidden.parquet","mydb/cpu/2026/09/17/00/bad key.parquet"],"iceberg_warehouse":{"path":"/srv/wh","file_count":4,"size_bytes":100,"skipped_files":1},"has_metadata":true,"has_config":false}`)) + _, _ = w.Write([]byte(`{"version":"26.09.3","backup_id":"` + incompleteID + `","created_at":"2026-09-17T00:00:00Z","backup_type":"full","databases":[{"name":"mydb","measurements":[{"name":"cpu","file_count":21,"size_bytes":2086}],"file_count":21,"size_bytes":2086}],"total_files":21,"total_size_bytes":2086,"skipped_files":3,"skipped_metadata_files":1,"skipped_reconciled":1,"skipped_sample":["mydb/cpu/2026/09/17/00/a.parquet","mydb/cpu/2026/09/17/00/b.parquet","` + long + `","mydb/cpu/metadata/00003-5f2c.metadata.json"],"skipped_overlong_keys":1,"unaddressable_files":2,"unaddressable_sample":["mydb/cpu/2026/09/17/00/.hidden.parquet","mydb/cpu/2026/09/17/00/bad key.parquet"],"unregistered_skipped":4,"unregistered_sample":["mydb/cpu/unregistered.parquet"],"manifest_only_files":2,"manifest_only_sample":["mydb/cpu/manifest-only.parquet"],"left_manifest_during_run":3,"left_manifest_sample":["mydb/cpu/left-manifest.parquet"],"cluster_manifest_checked":true,"iceberg_warehouse":{"path":"/srv/wh","file_count":4,"size_bytes":100,"skipped_files":1},"has_metadata":true,"has_config":false}`)) default: w.WriteHeader(404) _, _ = w.Write([]byte(`{"error":"Backup not found"}`)) @@ -66,7 +73,7 @@ func TestBackup_IncompleteIsVisible(t *testing.T) { if err != nil { t.Fatal(err) } - for _, want := range []string{"INCOMPLETE", "3 skipped, 1 metadata, 2 unaddressable", "│ -"} { + for _, want := range []string{"INCOMPLETE", "2 skipped, 1 metadata, 2 unaddressable, 2 manifest-only; info: 1 skipped reconciled, 4 unregistered, 3 left manifest during backup, cluster manifest checked", "│ -"} { if !strings.Contains(out, want) { t.Errorf("list table lacks %q:\n%s", want, out) } @@ -76,14 +83,14 @@ func TestBackup_IncompleteIsVisible(t *testing.T) { t.Fatal(err) } lines := strings.Split(strings.TrimSpace(out), "\n") - if len(lines) != 4 || lines[0] != "backup_id,created_at,backup_type,database_count,total_files,total_size_bytes,skipped_files,skipped_metadata_files,unaddressable_files" { + if len(lines) != 4 || lines[0] != "backup_id,created_at,backup_type,database_count,total_files,total_size_bytes,skipped_files,skipped_metadata_files,unaddressable_files,skipped_reconciled,unregistered_skipped,manifest_only_files,left_manifest_during_run,cluster_manifest_checked" { t.Errorf("csv header = %q (%d lines)", lines[0], len(lines)) } - if len(lines) == 4 && (!strings.HasSuffix(lines[1], ",21,2086,3,1,2") || !strings.HasSuffix(lines[2], ",20,2000,0,0,0") || !strings.HasSuffix(lines[3], ",20,2000,0,1,0")) { + if len(lines) == 4 && (!strings.HasSuffix(lines[1], ",21,2086,3,1,2,1,4,2,3,true") || !strings.HasSuffix(lines[2], ",20,2000,0,0,0,0,0,0,0,") || !strings.HasSuffix(lines[3], ",20,2000,0,1,0,0,0,0,0,")) { t.Errorf("csv rows = %q", lines[1:]) } out, _, err = execCmd(t, newBackupListCmd(), "-o", "csv", "--no-header") - if err != nil || strings.Contains(out, "backup_id,") || !strings.Contains(out, ",3,1,2") { + if err != nil || strings.Contains(out, "backup_id,") || !strings.Contains(out, ",3,1,2,1,4,2,3,true") { t.Errorf("csv --no-header: err=%v out=%q", err, out) } // json is the server's body, untouched. @@ -109,10 +116,17 @@ func TestBackup_IncompleteIsVisible(t *testing.T) { t.Fatal(err) } for _, want := range []string{ - "INCOMPLETE: 3 of 21 files and 1 metadata file were skipped while backing up (1 for a key too long to store); 2 files could not be listed (unaddressable); 1 Iceberg warehouse file was skipped\n", + "INCOMPLETE: 2 of 21 files and 1 metadata file were skipped while backing up (1 for a key too long to store); 2 files could not be listed (unaddressable); 2 files were present only in the cluster manifest; 1 Iceberg warehouse file was skipped\n", " skipped: mydb/cpu/2026/09/17/00/a.parquet\n", " skipped: mydb/cpu/metadata/00003-5f2c.metadata.json\n", " unaddressable: mydb/cpu/2026/09/17/00/bad key.parquet\n", + "reconciled: 1 skipped file was later absent from the manifest; not incomplete\n", + "unregistered: 4 local data files were not in the cluster manifest and were not backed up\n", + " unregistered: mydb/cpu/unregistered.parquet\n", + " manifest-only: mydb/cpu/manifest-only.parquet\n", + "left manifest: 3 files were copied then removed as the manifest changed during backup\n", + " left-manifest: mydb/cpu/left-manifest.parquet\n", + "cluster manifest: checked\n", } { if !strings.Contains(out, want) { t.Errorf("show lacks %q:\n%s", want, out) @@ -143,7 +157,7 @@ func TestBackup_IncompleteIsVisible(t *testing.T) { if err != nil { t.Fatalf("restore: %v (stderr %s)", err, stderr) } - if !strings.Contains(stderr, "warning: backup "+incompleteID+" is incomplete (3 files and 1 metadata file were skipped when it was taken, 1 of them for a key too long to store; 2 files could not be listed; 1 Iceberg warehouse file was skipped)\n") { + if !strings.Contains(stderr, "warning: backup "+incompleteID+" is incomplete (2 files and 1 metadata file were skipped when it was taken, 1 of them for a key too long to store; 2 files could not be listed; 2 files were present only in the cluster manifest; 1 Iceberg warehouse file was skipped)\n") { t.Errorf("restore warning missing or wrong:\n%s", stderr) } if !strings.Contains(out, "Restore of "+incompleteID+" started") { @@ -160,8 +174,13 @@ func TestBackup_IncompleteIsVisible(t *testing.T) { for _, want := range []string{ "status: failed\n", "error: restore incomplete: 1 objects could not be read", - "backup had: 3 skipped, 2 unaddressable when it was taken\n", + "backup had: 2 skipped, 2 unaddressable, 2 manifest-only when it was taken\n", + "backup info: 4 files were not listed in the cluster manifest when it was taken\n", "iceberg: 4 warehouse files not restored (this node has no outside-root warehouse)\n", + "mode: replace\n", + "restore: 15 files registered, 1 registration failures, 2 sidecar mismatches, 8 files replaced\n", + " registration-failed: mydb/cpu/registration-failed.parquet\n", + " sidecar-mismatch: mydb/cpu/sidecar-mismatch.parquet\n", " skipped: " + incompleteID + "/data/mydb/cpu/2026/09/17/00/c.parquet\n", " unaddressable: " + incompleteID + "/data/mydb/cpu/2026/09/17/00/.hidden.parquet\n", } { diff --git a/internal/commands/backup_test.go b/internal/commands/backup_test.go index 1eb86ef..43ae4d6 100644 --- a/internal/commands/backup_test.go +++ b/internal/commands/backup_test.go @@ -304,10 +304,11 @@ func newFakeBackupServer(t *testing.T) *fakeBackupServer { return } var req struct { - BackupID string `json:"backup_id"` - Confirm bool `json:"confirm"` - RestoreMetadata *bool `json:"restore_metadata"` - RestoreConfig *bool `json:"restore_config"` + BackupID string `json:"backup_id"` + Confirm bool `json:"confirm"` + RestoreMetadata *bool `json:"restore_metadata"` + RestoreConfig *bool `json:"restore_config"` + Mode *string `json:"mode"` } if err := json.Unmarshal(body, &req); err != nil { w.WriteHeader(http.StatusBadRequest) @@ -329,6 +330,11 @@ func newFakeBackupServer(t *testing.T) *fakeBackupServer { _, _ = w.Write([]byte(`{"error":"restore_config is not available on a cluster node: arc.toml holds this node identity"}`)) return } + if !f.clustered && req.Mode != nil && *req.Mode == "replace" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"replace mode is available only on cluster nodes"}`)) + return + } restoreMetadata := !f.clustered if req.RestoreMetadata != nil { restoreMetadata = *req.RestoreMetadata @@ -341,7 +347,11 @@ func newFakeBackupServer(t *testing.T) *fakeBackupServer { restartRequired := staged || restoreConfig && f.hasConfig go publish("restore", req.BackupID) w.WriteHeader(202) - response := map[string]any{"backup_id": req.BackupID, "message": "Restore started", "status": "running"} + mode := "merge" + if req.Mode != nil { + mode = *req.Mode + } + response := map[string]any{"backup_id": req.BackupID, "message": "Restore started", "status": "running", "mode": mode} if restartRequired { response["restart_required"] = true } @@ -404,7 +414,7 @@ func TestBackup_RestoreUsesServerDefaultsUnlessExplicit(t *testing.T) { const id = "backup-20260907-200009-00000009" t.Run("cluster default does not request metadata or config", func(t *testing.T) { f := newRestoreDefaultsServer(t, true, true) - _, stderr, err := execCmd(t, newBackupRestoreCmd(), id, "--yes") + out, stderr, err := execCmd(t, newBackupRestoreCmd(), id, "--yes") if err != nil { t.Fatalf("restore using cluster defaults: %v (stderr %q)", err, stderr) } @@ -418,6 +428,28 @@ func TestBackup_RestoreUsesServerDefaultsUnlessExplicit(t *testing.T) { if strings.Contains(stderr, "metadata store will be staged") { t.Errorf("cluster default skips metadata, but stderr claimed it would be staged: %q", stderr) } + if _, ok := f.lastRestoreRequest()["mode"]; ok { + t.Error("omitted --mode must leave the request mode to Arc") + } + if !strings.Contains(out, "Restore of "+id+" started (mode: merge)") { + t.Errorf("output omitted the server's effective default mode: %q", out) + } + }) + + t.Run("explains the scoped default before confirmation", func(t *testing.T) { + f := newRestoreDefaultsServer(t, true, true) + cmd := newBackupRestoreCmd() + cmd.SetIn(strings.NewReader("n\n")) + _, stderr, err := execCmd(t, cmd, id) + if err == nil || err.Error() != "aborted" { + t.Fatalf("err = %v; want declined confirmation", err) + } + if !strings.Contains(stderr, "restore mode follows Arc's default: merge for a full backup, or replace for a scoped backup on a cluster") { + t.Errorf("confirmation omitted the server's scoped mode default: %q", stderr) + } + if len(f.lastRestoreRequest()) != 0 { + t.Errorf("declined restore reached the server: %v", f.lastRestoreRequest()) + } }) t.Run("data-only explicitly disables metadata and config", func(t *testing.T) { @@ -503,6 +535,46 @@ func TestBackup_RestoreUsesServerDefaultsUnlessExplicit(t *testing.T) { }) } +func TestBackup_RestoreMode(t *testing.T) { + const id = "backup-20260907-200009-00000009" + + t.Run("rejects invalid mode before making a request", func(t *testing.T) { + f := newRestoreDefaultsServer(t, true, true) + _, _, err := execCmd(t, newBackupRestoreCmd(), id, "--mode", "replace-all", "--yes") + if err == nil || err.Error() != `invalid --mode "replace-all" (valid: merge, replace)` { + t.Fatalf("err = %v", err) + } + if len(f.lastRestoreRequest()) != 0 { + t.Errorf("invalid mode reached the server: %v", f.lastRestoreRequest()) + } + }) + + t.Run("surfaces standalone replace refusal", func(t *testing.T) { + f := newRestoreDefaultsServer(t, false, true) + _, _, err := execCmd(t, newBackupRestoreCmd(), id, "--mode", "replace", "--yes") + if err == nil || !strings.Contains(err.Error(), "replace mode is available only on cluster nodes") { + t.Fatalf("err = %v", err) + } + if len(f.restoreRequests) != 1 || string(f.lastRestoreRequest()["mode"]) != `"replace"` { + t.Errorf("replace request = %v", f.lastRestoreRequest()) + } + }) + + t.Run("prints the effective mode returned by the server", func(t *testing.T) { + f := newRestoreDefaultsServer(t, true, true) + out, _, err := execCmd(t, newBackupRestoreCmd(), id, "--mode", "replace", "--yes") + if err != nil { + t.Fatalf("restore: %v", err) + } + if !strings.Contains(out, "Restore of "+id+" started (mode: replace)") { + t.Errorf("restore output does not include the server mode: %q", out) + } + if string(f.lastRestoreRequest()["mode"]) != `"replace"` { + t.Errorf("request mode = %s", f.lastRestoreRequest()["mode"]) + } + }) +} + func TestBackup_MutatingCommandsDocumentPrimaryWriterRequirement(t *testing.T) { backup := newBackupCmd() want := map[string][]string{