Mask a Lakebase map field as a whole, not per entry

.nextchanges/bundles/postgres-map-update-mask.md

@@ -0,0 +1 @@
+direct: Fix deploying an update to `postgres_endpoints.settings.pg_settings` ([#6441](https://github.com/databricks/cli/pull/6441)).

acceptance/bundle/resources/postgres_endpoints/add_settings/databricks.yml.tmpl

@@ -0,0 +1,27 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+resources:
+  postgres_projects:
+    my_project:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: "Test Project for Endpoint settings Add"
+      pg_version: 16
+      history_retention_duration: 604800s
+
+  postgres_branches:
+    main:
+      parent: ${resources.postgres_projects.my_project.id}
+      branch_id: main
+      no_expiry: true
+
+  postgres_endpoints:
+    my_endpoint:
+      parent: ${resources.postgres_branches.main.id}
+      endpoint_id: my-endpoint
+      endpoint_type: ENDPOINT_TYPE_READ_ONLY
+      autoscaling_limit_min_cu: 0.5
+      autoscaling_limit_max_cu: 8

acceptance/bundle/resources/postgres_endpoints/add_settings/out.requests.direct.json

@@ -0,0 +1,19 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint",
+  "q": {
+    "update_mask": "spec.settings"
+  },
+  "body": {
+    "spec": {
+      "autoscaling_limit_max_cu": 8,
+      "autoscaling_limit_min_cu": 0.5,
+      "endpoint_type": "ENDPOINT_TYPE_READ_ONLY",
+      "settings": {
+        "pg_settings": {
+          "statement_timeout": "8000"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/postgres_endpoints/add_settings/out.requests.terraform.json

@@ -0,0 +1,21 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint",
+  "q": {
+    "update_mask": "spec"
+  },
+  "body": {
+    "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint",
+    "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main",
+    "spec": {
+      "autoscaling_limit_max_cu": 8,
+      "autoscaling_limit_min_cu": 0.5,
+      "endpoint_type": "ENDPOINT_TYPE_READ_ONLY",
+      "settings": {
+        "pg_settings": {
+          "statement_timeout": "8000"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/postgres_endpoints/add_settings/output.txt

@@ -0,0 +1,45 @@
+
+=== Initial deployment with no settings block
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Created postgres_branches.main
+Created postgres_endpoints.my_endpoint
+Created postgres_projects.my_project
+Files: 0 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Add a settings block and re-deploy
+>>> [CLI] bundle plan
+update postgres_endpoints.my_endpoint
+
+Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Updated postgres_endpoints.my_endpoint
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 2 unchanged
+
+>>> print_requests.py --del-body project_id,branch_id,endpoint_id --sort //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/
+
+=== Plan again: the addition must leave no drift behind
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.postgres_branches.main
+  delete resources.postgres_endpoints.my_endpoint
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase branches.
+All data stored in them will be permanently lost:
+  delete resources.postgres_branches.main
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Destroy: 3 deleted

acceptance/bundle/resources/postgres_endpoints/add_settings/script

@@ -0,0 +1,28 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+   trace $CLI bundle destroy --auto-approve
+   rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "Initial deployment with no settings block"
+trace $CLI bundle deploy
+rm -f out.requests.txt
+
+title "Add a settings block and re-deploy"
+# Going from absent to present leaves one change path, settings, with no children,
+# so the mask names the message itself. Verified against a real endpoint on
+# 2026-08-31 that update_mask=spec.settings is accepted.
+cat >> databricks.yml <<'YAML'
+      settings:
+        pg_settings:
+          statement_timeout: 8000
+YAML
+trace $CLI bundle plan
+trace $CLI bundle deploy
+
+trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json
+
+title "Plan again: the addition must leave no drift behind"
+trace $CLI bundle plan

acceptance/bundle/resources/postgres_endpoints/update_pg_settings/databricks.yml.tmpl

@@ -0,0 +1,30 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+resources:
+  postgres_projects:
+    my_project:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: "Test Project for Endpoint pg_settings Update"
+      pg_version: 16
+      history_retention_duration: 604800s
+
+  postgres_branches:
+    main:
+      parent: ${resources.postgres_projects.my_project.id}
+      branch_id: main
+      no_expiry: true
+
+  postgres_endpoints:
+    my_endpoint:
+      parent: ${resources.postgres_branches.main.id}
+      endpoint_id: my-endpoint
+      endpoint_type: ENDPOINT_TYPE_READ_ONLY
+      autoscaling_limit_min_cu: 0.5
+      autoscaling_limit_max_cu: 8
+      settings:
+        pg_settings:
+          statement_timeout: 8000

acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.plan.direct.json

@@ -0,0 +1,42 @@
+{
+  "action": "update",
+  "changes": {
+    "autoscaling_limit_max_cu": {
+      "action": "skip",
+      "reason": "spec:input_only",
+      "old": 8,
+      "new": 8
+    },
+    "autoscaling_limit_min_cu": {
+      "action": "skip",
+      "reason": "spec:input_only",
+      "old": 0.5,
+      "new": 0.5
+    },
+    "endpoint_type": {
+      "action": "skip",
+      "reason": "spec:input_only",
+      "old": "ENDPOINT_TYPE_READ_ONLY",
+      "new": "ENDPOINT_TYPE_READ_ONLY",
+      "remote": ""
+    },
+    "settings": {
+      "action": "update",
+      "old": {
+        "pg_settings": {
+          "statement_timeout": "8000"
+        }
+      },
+      "new": {
+        "pg_settings": {
+          "statement_timeout": "16000"
+        }
+      }
+    },
+    "settings.pg_settings['statement_timeout']": {
+      "action": "update",
+      "old": "8000",
+      "new": "16000"
+    }
+  }
+}

acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.plan.terraform.json

@@ -0,0 +1,4 @@
+{
+  "action": "update",
+  "changes": null
+}

acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.requests.direct.json

@@ -0,0 +1,19 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint",
+  "q": {
+    "update_mask": "spec.settings.pg_settings"
+  },
+  "body": {
+    "spec": {
+      "autoscaling_limit_max_cu": 8,
+      "autoscaling_limit_min_cu": 0.5,
+      "endpoint_type": "ENDPOINT_TYPE_READ_ONLY",
+      "settings": {
+        "pg_settings": {
+          "statement_timeout": "16000"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/postgres_endpoints/update_pg_settings/out.requests.terraform.json

@@ -0,0 +1,21 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint",
+  "q": {
+    "update_mask": "spec"
+  },
+  "body": {
+    "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints/my-endpoint",
+    "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/main",
+    "spec": {
+      "autoscaling_limit_max_cu": 8,
+      "autoscaling_limit_min_cu": 0.5,
+      "endpoint_type": "ENDPOINT_TYPE_READ_ONLY",
+      "settings": {
+        "pg_settings": {
+          "statement_timeout": "16000"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/postgres_endpoints/update_pg_settings/output.txt

@@ -0,0 +1,49 @@
+
+=== Initial deployment with pg_settings.statement_timeout: 8000
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Created postgres_branches.main
+Created postgres_endpoints.my_endpoint
+Created postgres_projects.my_project
+Files: 0 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Change the value of one pg_settings key
+>>> update_file.py databricks.yml statement_timeout: 8000 statement_timeout: 16000
+
+>>> [CLI] bundle plan
+update postgres_endpoints.my_endpoint
+
+Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle plan -o json
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Updated postgres_endpoints.my_endpoint
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 2 unchanged
+
+>>> print_requests.py --del-body project_id,branch_id,endpoint_id --sort //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/
+
+=== Plan again: the change must leave no drift behind
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.postgres_branches.main
+  delete resources.postgres_endpoints.my_endpoint
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.my_project
+
+This action will result in the deletion of the following Lakebase branches.
+All data stored in them will be permanently lost:
+  delete resources.postgres_branches.main
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Destroy: 3 deleted

acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script

@@ -0,0 +1,33 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+   trace $CLI bundle destroy --auto-approve
+   rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+endpoint_name="projects/test-pg-proj-${UNIQUE_NAME}/branches/main/endpoints/my-endpoint"
+
+title "Initial deployment with pg_settings.statement_timeout: 8000"
+trace $CLI bundle deploy
+rm -f out.requests.txt
+
+title "Change the value of one pg_settings key"
+trace update_file.py databricks.yml "statement_timeout: 8000" "statement_timeout: 16000"
+
+# pg_settings is a map, and the plan diffs maps entry by entry, so the changed path
+# is settings.pg_settings['statement_timeout']. A map or repeated field is
+# addressable only as a whole, so it has to be masked as spec.settings.pg_settings.
+# Verified against a real workspace on 2026-08-31: spec.settings and
+# spec.settings.pg_settings are both accepted, while an indexed path is answered with
+# 400 "Unknown field path in update_mask".
+trace $CLI bundle plan
+trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
+rm -f out.requests.txt
+trace $CLI bundle deploy
+
+# The mask each engine sends: direct names the map, terraform replaces the whole spec.
+trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json
+
+title "Plan again: the change must leave no drift behind"
+trace $CLI bundle plan

bundle/direct/dresources/util.go

@@ -52,9 +52,22 @@ func collectUpdatePathsWithPrefix(changes Changes, prefix string) []string {
 			}
 		}
 		if !hasChild {
-			paths = append(paths, prefix+path)
+			paths = append(paths, prefix+maskPath(path))
 		}
 	}
 	slices.Sort(paths)
-	return paths
+	// Truncating subscripts can map two changed entries of the same map onto one path.
+	return slices.Compact(paths)
+}
+
+// maskPath converts a change path into the path the API accepts in update_mask. A map or
+// repeated field is addressable only as a whole, so everything from the first subscript
+// onwards is dropped: settings.pg_settings['work_mem'] is masked as settings.pg_settings.
+// Verified against a real workspace on 2026-08-31, which answers the indexed form with
+// "Unknown field path in update_mask: 'spec.settings.pg_settings['work_mem']'".
+func maskPath(path string) string {
+	if before, _, ok := strings.Cut(path, "["); ok {
+		return before
+	}
+	return path
 }