direct: Fix perpetual update on grants with ALL_PRIVILEGES

.nextchanges/bundles/grants-normalization.md

@@ -0,0 +1 @@
+Fixed spurious `update` on catalog/schema/volume grants (direct engine): a principal granted `ALL_PRIVILEGES` no longer drifts when the backend also reports the concrete privileges it implies, and privilege names are normalized to match the case and ordering Unity Catalog returns ([#6030](https://github.com/databricks/cli/issues/6030)).

acceptance/bundle/deploy/readplan/grants-remove-principal/output.txt

@@ -13,21 +13,15 @@ Deployment complete!
     "changes": [
       {
         "add": [
-          "USE_SCHEMA"
+          "CREATE_TABLE"
         ],
-        "principal": "[USERNAME]",
-        "remove": [
-          "ALL_PRIVILEGES"
-        ]
+        "principal": "extra@example.test"
       },
       {
         "add": [
-          "CREATE_TABLE"
+          "USE_SCHEMA"
         ],
-        "principal": "extra@example.test",
-        "remove": [
-          "ALL_PRIVILEGES"
-        ]
+        "principal": "[USERNAME]"
       }
     ]
   }
@@ -51,19 +45,10 @@ Deployment complete!
   "path": "/api/2.1/unity-catalog/permissions/schema/main.myschema",
   "body": {
     "changes": [
-      {
-        "add": [
-          "USE_SCHEMA"
-        ],
-        "principal": "[USERNAME]",
-        "remove": [
-          "ALL_PRIVILEGES"
-        ]
-      },
       {
         "principal": "extra@example.test",
         "remove": [
-          "ALL_PRIVILEGES"
+          "CREATE_TABLE"
         ]
       }
     ]

acceptance/bundle/invariant/configs/grants_privilege_normalization.yml.tmpl

@@ -0,0 +1,18 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  schemas:
+    foo:
+      catalog_name: main
+      name: test-schema-grantnorm-$UNIQUE_NAME
+      grants:
+        # UC normalizes privileges (uppercases, converts spaces to underscores)
+        # and does not guarantee their order on readback. The direct engine must
+        # apply the same normalization so a deploy of non-canonical privileges
+        # does not report perpetual drift.
+        - principal: account users
+          privileges:
+            - use_schema
+            - CREATE TABLE
+            - APPLY_TAG

acceptance/bundle/invariant/test.toml

@@ -35,6 +35,7 @@ EnvMatrix.INPUT_CONFIG = [
     "experiment.yml.tmpl",
     "external_location.yml.tmpl",
     "genie_space.yml.tmpl",
+    "grants_privilege_normalization.yml.tmpl",
     "instance_pool.yml.tmpl",
     "job.yml.tmpl",
     "job_pydabs_10_tasks.yml.tmpl",

acceptance/bundle/resources/grants/catalogs/script

@@ -9,7 +9,7 @@ trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_
 trace $CLI bundle plan
 
 # Since we're sending different requests between terraform and direct, assert that the end result is the same:
-trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys
+trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys '(.privilege_assignments[].privileges) |= sort'
 
 update_file.py databricks.yml CREATE_SCHEMA USE_SCHEMA
 
@@ -18,7 +18,7 @@ trace print_requests.py --get //permissions
 trace $CLI bundle deploy
 trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json
 
-trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys
+trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys '(.privilege_assignments[].privileges) |= sort'
 
 trace $CLI bundle destroy --auto-approve
 trace print_requests.py //permissions > out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json

acceptance/bundle/resources/grants/schemas/all_privileges_coexist/databricks.yml.tmpl

@@ -0,0 +1,17 @@
+bundle:
+  name: schema-grants-all-privileges-coexist-$UNIQUE_NAME
+
+resources:
+  schemas:
+    grants_schema:
+      name: schema_all_priv_coexist_$UNIQUE_NAME
+      catalog_name: main
+      grants:
+        # Config declares only ALL_PRIVILEGES for the principal. Reproduces
+        # issue #6030: the backend also reports a concrete privilege for this
+        # principal (added out of band below / materialized for owners), and
+        # buildGrantChanges skips the ALL_PRIVILEGES removal, so the concrete
+        # privilege is never reconciled and the plan reports perpetual update.
+        - principal: $CURRENT_USER_NAME
+          privileges:
+            - ALL_PRIVILEGES

acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.plan.direct.txt

@@ -0,0 +1,3 @@
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged

acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.plan.terraform.txt

@@ -0,0 +1,5 @@
+
+>>> [CLI] bundle plan
+update schemas.grants_schema.grants
+
+Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged

acceptance/bundle/resources/grants/schemas/all_privileges_coexist/output.txt

@@ -0,0 +1,20 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-all-privileges-coexist-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] grants update schema main.schema_all_priv_coexist_[UNIQUE_NAME] --json @update.resolved.json
+
+>>> errcode [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.schemas.grants_schema
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.grants_schema
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/schema-grants-all-privileges-coexist-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/grants/schemas/all_privileges_coexist/script

@@ -0,0 +1,22 @@
+SCHEMA_FULL_NAME=main.schema_all_priv_coexist_$UNIQUE_NAME
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace errcode $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+
+# Out of band, grant a concrete privilege in addition to the ALL_PRIVILEGES that
+# the config declares for the same principal. This reproduces issue #6030: the
+# backend then reports both ALL_PRIVILEGES and the concrete privilege.
+envsubst < update.json > update.resolved.json
+trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.resolved.json > /dev/null
+
+# The plan output differs between engines: the direct engine treats ALL_PRIVILEGES
+# as implying every concrete privilege, so it reports no drift (issue #6030),
+# while terraform revokes the extra privilege to match the config exactly.
+trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1

acceptance/bundle/resources/grants/schemas/all_privileges_coexist/test.toml

@@ -0,0 +1 @@
+Ignore = ['update.resolved.json']

acceptance/bundle/resources/grants/schemas/all_privileges_coexist/update.json

@@ -0,0 +1,8 @@
+{
+  "changes": [
+    {
+      "principal": "$CURRENT_USER_NAME",
+      "add": ["USE_SCHEMA"]
+    }
+  ]
+}

acceptance/bundle/resources/grants/schemas/change_privilege/script

@@ -9,7 +9,7 @@ trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_
 trace $CLI bundle plan
 
 # Since we're sending different requests between terraform and direct, assert that the end result is the same:
-trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys
+trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys '(.privilege_assignments[].privileges) |= sort'
 
 update_file.py databricks.yml USE_SCHEMA APPLY_TAG
 
@@ -18,7 +18,7 @@ trace print_requests.py --get //permissions
 trace $CLI bundle deploy
 trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json
 
-trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys
+trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys '(.privilege_assignments[].privileges) |= sort'
 
 trace $CLI bundle destroy --auto-approve
 trace print_requests.py //permissions > out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json

acceptance/bundle/resources/grants/volumes/script

@@ -5,7 +5,7 @@ trace $CLI bundle deploy
 trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json
 
 # Since we're sending different requests between terraform and direct, assert that the end result is the same:
-trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq --sort-keys
+trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq --sort-keys '(.privilege_assignments[].privileges) |= sort'
 
 update_file.py databricks.yml WRITE_VOLUME MANAGE
 
@@ -14,7 +14,7 @@ trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_op
 trace $CLI bundle deploy
 trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json
 
-trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq --sort-keys
+trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq --sort-keys '(.privilege_assignments[].privileges) |= sort'
 
 trace $CLI bundle destroy --auto-approve
 trace print_requests.py //permissions > out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json

bundle/direct/dresources/grants.go

@@ -48,10 +48,9 @@ func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVa
 		return nil, fmt.Errorf("expected *[]catalog.PrivilegeAssignment, got %T", inputConfig)
 	}
 
-	// Backend sorts privileges, so we sort here as well.
-	for i := range *grantsPtr {
-		slices.Sort((*grantsPtr)[i].Privileges)
-	}
+	// Normalize the same way the backend does (uppercase privileges, sort) so
+	// the config and the value read back by DoRead compare equal.
+	normalizeAssignments(*grantsPtr)
 
 	return &structvar.StructVar{
 		Value: &GrantsState{
@@ -222,9 +221,45 @@ func (r *ResourceGrants) listGrants(ctx context.Context, securableType, fullName
 		}
 		pageToken = resp.NextPageToken
 	}
+	// The backend does not guarantee a stable privilege order across reads, so
+	// normalize the same way as the config side to avoid false drift.
+	normalizeAssignments(assignments)
 	return assignments, nil
 }
 
+// normalizePrivilege matches the backend's normalization: privileges are
+// uppercased and spaces are converted to underscores (e.g. "use schema" ->
+// "USE_SCHEMA"). Mirrors the Terraform provider's permissions.NormalizePrivilege.
+func normalizePrivilege(p catalog.Privilege) catalog.Privilege {
+	return catalog.Privilege(strings.ToUpper(strings.ReplaceAll(string(p), " ", "_")))
+}
+
+// normalizeAssignments normalizes each assignment's privileges in place so that
+// config and remote state compare equal. It uppercases privilege names, sorts
+// them (the backend does not guarantee a stable order), and collapses a
+// principal holding ALL_PRIVILEGES down to just ALL_PRIVILEGES. The collapse is
+// applied to both sides, so config granting only ALL_PRIVILEGES matches a
+// backend that reports ALL_PRIVILEGES plus the concrete privileges it implies,
+// instead of reporting a perpetual update.
+func normalizeAssignments(assignments []catalog.PrivilegeAssignment) {
+	for i := range assignments {
+		for j := range assignments[i].Privileges {
+			assignments[i].Privileges[j] = normalizePrivilege(assignments[i].Privileges[j])
+		}
+		if slices.Contains(assignments[i].Privileges, catalog.PrivilegeAllPrivileges) {
+			assignments[i].Privileges = []catalog.Privilege{catalog.PrivilegeAllPrivileges}
+			continue
+		}
+		slices.Sort(assignments[i].Privileges)
+		// Dedupe here rather than relying on MergeGrants: it deduplicates the raw
+		// config strings before normalization, so distinct spellings of the same
+		// privilege (e.g. "use schema" and "USE_SCHEMA") only collide once
+		// normalized. Without this the config side keeps a duplicate the backend
+		// read side never returns, causing perpetual drift.
+		assignments[i].Privileges = slices.Compact(assignments[i].Privileges)
+	}
+}
+
 func extractGrantResourceType(node string) (string, error) {
 	rest, ok := strings.CutPrefix(node, "resources.")
 	if !ok {

bundle/direct/dresources/grants_test.go

@@ -74,3 +74,61 @@ func TestBuildGrantChanges(t *testing.T) {
 		})
 	}
 }
+
+func TestNormalizeAssignments(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []catalog.PrivilegeAssignment
+		expected []catalog.PrivilegeAssignment
+	}{
+		{
+			name: "uppercases and sorts privileges",
+			input: []catalog.PrivilegeAssignment{
+				{Principal: "alice", Privileges: []catalog.Privilege{"use_schema", "apply_tag"}},
+			},
+			expected: []catalog.PrivilegeAssignment{
+				{Principal: "alice", Privileges: []catalog.Privilege{"APPLY_TAG", "USE_SCHEMA"}},
+			},
+		},
+		{
+			name: "converts spaces to underscores",
+			input: []catalog.PrivilegeAssignment{
+				{Principal: "alice", Privileges: []catalog.Privilege{"create table", "USE SCHEMA"}},
+			},
+			expected: []catalog.PrivilegeAssignment{
+				{Principal: "alice", Privileges: []catalog.Privilege{"CREATE_TABLE", "USE_SCHEMA"}},
+			},
+		},
+		{
+			// Regression test for #6030: ALL_PRIVILEGES implies every concrete
+			// privilege, so a principal holding it collapses to just
+			// ALL_PRIVILEGES. Applied to both config and remote, this stops the
+			// backend's extra concrete privileges from showing as drift.
+			name: "collapses ALL_PRIVILEGES with concrete privileges",
+			input: []catalog.PrivilegeAssignment{
+				{Principal: "alice", Privileges: []catalog.Privilege{catalog.PrivilegeUseCatalog, catalog.PrivilegeAllPrivileges, catalog.PrivilegeCreateSchema}},
+			},
+			expected: []catalog.PrivilegeAssignment{
+				{Principal: "alice", Privileges: []catalog.Privilege{catalog.PrivilegeAllPrivileges}},
+			},
+		},
+		{
+			// Distinct spellings collide only after normalization, so dedupe
+			// here; MergeGrants deduplicates the raw config strings earlier.
+			name: "deduplicates privileges that collide after normalization",
+			input: []catalog.PrivilegeAssignment{
+				{Principal: "alice", Privileges: []catalog.Privilege{"USE SCHEMA", "use_schema", "USE_SCHEMA"}},
+			},
+			expected: []catalog.PrivilegeAssignment{
+				{Principal: "alice", Privileges: []catalog.Privilege{"USE_SCHEMA"}},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			normalizeAssignments(tt.input)
+			assert.Equal(t, tt.expected, tt.input)
+		})
+	}
+}