diff --git a/.nextchanges/bundles/postgres-map-update-mask.md b/.nextchanges/bundles/postgres-map-update-mask.md new file mode 100644 index 00000000000..6617e62057a --- /dev/null +++ b/.nextchanges/bundles/postgres-map-update-mask.md @@ -0,0 +1 @@ +direct: Fix deploying an update to `postgres_endpoints.settings.pg_settings` ([#6441](https://github.com/databricks/cli/pull/6441)). diff --git a/acceptance/bundle/resources/postgres_endpoints/add_settings/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_endpoints/add_settings/databricks.yml.tmpl new file mode 100644 index 00000000000..cf88ee9dbea --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/add_settings/databricks.yml.tmpl @@ -0,0 +1,27 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Endpoint settings Add" + pg_version: 16 + history_retention_duration: 604800s + + postgres_branches: + main: + parent: ${resources.postgres_projects.my_project.id} + branch_id: main + no_expiry: true + + postgres_endpoints: + my_endpoint: + parent: ${resources.postgres_branches.main.id} + endpoint_id: my-endpoint + endpoint_type: ENDPOINT_TYPE_READ_ONLY + autoscaling_limit_min_cu: 0.5 + autoscaling_limit_max_cu: 8 diff --git a/acceptance/bundle/resources/postgres_endpoints/add_settings/out.requests.direct.json b/acceptance/bundle/resources/postgres_endpoints/add_settings/out.requests.direct.json new file mode 100644 index 00000000000..0693bf14f3b --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/add_settings/out.requests.direct.json @@ -0,0 +1,19 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint", + "q": { + "update_mask": "spec.settings" + }, + "body": { + "spec": { + "autoscaling_limit_max_cu": 8, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "settings": { + "pg_settings": { + "statement_timeout": "8000" + } + } + } + } +} diff --git a/acceptance/bundle/resources/postgres_endpoints/add_settings/out.requests.terraform.json b/acceptance/bundle/resources/postgres_endpoints/add_settings/out.requests.terraform.json new file mode 100644 index 00000000000..56601044e08 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/add_settings/out.requests.terraform.json @@ -0,0 +1,21 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "autoscaling_limit_max_cu": 8, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "settings": { + "pg_settings": { + "statement_timeout": "8000" + } + } + } + } +} diff --git a/acceptance/bundle/resources/postgres_endpoints/add_settings/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/add_settings/out.test.toml new file mode 100644 index 00000000000..9c8db837a9e --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/add_settings/out.test.toml @@ -0,0 +1,5 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_endpoints/add_settings/output.txt b/acceptance/bundle/resources/postgres_endpoints/add_settings/output.txt new file mode 100644 index 00000000000..64e5b4d2d1d --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/add_settings/output.txt @@ -0,0 +1,45 @@ + +=== Initial deployment with no settings block +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created postgres_branches.main +Created postgres_endpoints.my_endpoint +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Add a settings block and re-deploy +>>> [CLI] bundle plan +update postgres_endpoints.my_endpoint + +Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated postgres_endpoints.my_endpoint +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id --sort //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +=== Plan again: the addition must leave no drift behind +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.main + delete resources.postgres_endpoints.my_endpoint + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.main + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/postgres_endpoints/add_settings/script b/acceptance/bundle/resources/postgres_endpoints/add_settings/script new file mode 100644 index 00000000000..ceeea8f6008 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/add_settings/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Initial deployment with no settings block" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "Add a settings block and re-deploy" +# Going from absent to present leaves one change path, settings, with no children, +# so the mask names the message itself. Verified against a real endpoint on +# 2026-08-31 that update_mask=spec.settings is accepted. +cat >> databricks.yml <<'YAML' + settings: + pg_settings: + statement_timeout: 8000 +YAML +trace $CLI bundle plan +trace $CLI bundle deploy + +trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json + +title "Plan again: the addition must leave no drift behind" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/databricks.yml.tmpl new file mode 100644 index 00000000000..57e210af5c7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/databricks.yml.tmpl @@ -0,0 +1,30 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Endpoint pg_settings Update" + pg_version: 16 + history_retention_duration: 604800s + + postgres_branches: + main: + parent: ${resources.postgres_projects.my_project.id} + branch_id: main + no_expiry: true + + postgres_endpoints: + my_endpoint: + parent: ${resources.postgres_branches.main.id} + endpoint_id: my-endpoint + endpoint_type: ENDPOINT_TYPE_READ_ONLY + autoscaling_limit_min_cu: 0.5 + autoscaling_limit_max_cu: 8 + settings: + pg_settings: + statement_timeout: 8000 diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.plan.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.plan.direct.json new file mode 100644 index 00000000000..1cb37958265 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.plan.direct.json @@ -0,0 +1,42 @@ +{ + "action": "update", + "changes": { + "autoscaling_limit_max_cu": { + "action": "skip", + "reason": "spec:input_only", + "old": 8, + "new": 8 + }, + "autoscaling_limit_min_cu": { + "action": "skip", + "reason": "spec:input_only", + "old": 0.5, + "new": 0.5 + }, + "endpoint_type": { + "action": "skip", + "reason": "spec:input_only", + "old": "ENDPOINT_TYPE_READ_ONLY", + "new": "ENDPOINT_TYPE_READ_ONLY", + "remote": "" + }, + "settings": { + "action": "update", + "old": { + "pg_settings": { + "statement_timeout": "8000" + } + }, + "new": { + "pg_settings": { + "statement_timeout": "16000" + } + } + }, + "settings.pg_settings['statement_timeout']": { + "action": "update", + "old": "8000", + "new": "16000" + } + } +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.plan.terraform.json b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.plan.terraform.json new file mode 100644 index 00000000000..fb09ecaf88a --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.plan.terraform.json @@ -0,0 +1,4 @@ +{ + "action": "update", + "changes": null +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.requests.direct.json b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.requests.direct.json new file mode 100644 index 00000000000..8e87c6f1861 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.requests.direct.json @@ -0,0 +1,19 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint", + "q": { + "update_mask": "spec.settings.pg_settings" + }, + "body": { + "spec": { + "autoscaling_limit_max_cu": 8, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "settings": { + "pg_settings": { + "statement_timeout": "16000" + } + } + } + } +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.requests.terraform.json b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.requests.terraform.json new file mode 100644 index 00000000000..ce239694d16 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.requests.terraform.json @@ -0,0 +1,21 @@ +{ + "method": "PATCH", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint", + "q": { + "update_mask": "spec" + }, + "body": { + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main", + "spec": { + "autoscaling_limit_max_cu": 8, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_ONLY", + "settings": { + "pg_settings": { + "statement_timeout": "16000" + } + } + } + } +} diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.test.toml new file mode 100644 index 00000000000..9c8db837a9e --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.test.toml @@ -0,0 +1,5 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/output.txt b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/output.txt new file mode 100644 index 00000000000..8bc4e2561b9 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/output.txt @@ -0,0 +1,49 @@ + +=== Initial deployment with pg_settings.statement_timeout: 8000 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created postgres_branches.main +Created postgres_endpoints.my_endpoint +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Change the value of one pg_settings key +>>> update_file.py databricks.yml statement_timeout: 8000 statement_timeout: 16000 + +>>> [CLI] bundle plan +update postgres_endpoints.my_endpoint + +Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated postgres_endpoints.my_endpoint +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py --del-body project_id,branch_id,endpoint_id --sort //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +=== Plan again: the change must leave no drift behind +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_branches.main + delete resources.postgres_endpoints.my_endpoint + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.main + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script new file mode 100644 index 00000000000..ca616cee3f1 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script @@ -0,0 +1,33 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +endpoint_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/main/endpoints/my-endpoint" + +title "Initial deployment with pg_settings.statement_timeout: 8000" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "Change the value of one pg_settings key" +trace update_file.py databricks.yml "statement_timeout: 8000" "statement_timeout: 16000" + +# pg_settings is a map, and the plan diffs maps entry by entry, so the changed path +# is settings.pg_settings['statement_timeout']. A map or repeated field is +# addressable only as a whole, so it has to be masked as spec.settings.pg_settings. +# Verified against a real workspace on 2026-08-31: spec.settings and +# spec.settings.pg_settings are both accepted, while an indexed path is answered with +# 400 "Unknown field path in update_mask". +trace $CLI bundle plan +trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +rm -f out.requests.txt +trace $CLI bundle deploy + +# The mask each engine sends: direct names the map, terraform replaces the whole spec. +trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json + +title "Plan again: the change must leave no drift behind" +trace $CLI bundle plan diff --git a/bundle/direct/dresources/util.go b/bundle/direct/dresources/util.go index 93148ea95e2..e6d4911ccc4 100644 --- a/bundle/direct/dresources/util.go +++ b/bundle/direct/dresources/util.go @@ -52,9 +52,22 @@ func collectUpdatePathsWithPrefix(changes Changes, prefix string) []string { } } if !hasChild { - paths = append(paths, prefix+path) + paths = append(paths, prefix+maskPath(path)) } } slices.Sort(paths) - return paths + // Truncating subscripts can map two changed entries of the same map onto one path. + return slices.Compact(paths) +} + +// maskPath converts a change path into the path the API accepts in update_mask. A map or +// repeated field is addressable only as a whole, so everything from the first subscript +// onwards is dropped: settings.pg_settings['work_mem'] is masked as settings.pg_settings. +// Verified against a real workspace on 2026-08-31, which answers the indexed form with +// "Unknown field path in update_mask: 'spec.settings.pg_settings['work_mem']'". +func maskPath(path string) string { + if before, _, ok := strings.Cut(path, "["); ok { + return before + } + return path } diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index e9c101a7e20..54f0610a1d4 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -692,7 +692,13 @@ var endpointUpdateMaskPaths = []string{ "spec.group.enable_readable_secondaries", "spec.group.max", "spec.group.min", + + // A message is maskable on its own, which is how a bundle that adds a whole + // settings block reaches the API. Probed on a real endpoint 2026-08-31: both + // spec.settings and spec.settings.pg_settings get past mask validation. + "spec.settings", "spec.settings.pg_settings", + "spec.suspension", }