auth: decouple CLI from SDK's Experimental_IsUnifiedHost field

NEXT_CHANGELOG.md

@@ -8,6 +8,7 @@
 
 * Added `--limit` flag to all paginated list commands for client-side result capping ([#4984](https://github.com/databricks/cli/pull/4984)).
 * Accept `yes` in addition to `y` for confirmation prompts, and show `[y/N]` to indicate that no is the default.
+* Stop persisting `experimental_is_unified_host` to new profiles and ignore the `DATABRICKS_EXPERIMENTAL_IS_UNIFIED_HOST` env var. Unified hosts are now detected automatically from `/.well-known/databricks-config`. Existing profiles with the key set continue to work via a legacy fallback; `--experimental-is-unified-host` is deprecated but still honored as a routing fallback for this release.
 
 ### Bundles
 * Remove `experimental-jobs-as-code` template, superseded by `pydabs` ([#4999](https://github.com/databricks/cli/pull/4999)).

bundle/config/workspace.go

@@ -46,6 +46,11 @@ type Workspace struct {
 	AzureLoginAppID  string `json:"azure_login_app_id,omitempty"`
 
 	// Unified host specific attributes.
+	//
+	// ExperimentalIsUnifiedHost is a deprecated no-op. Unified hosts are now
+	// detected automatically from /.well-known/databricks-config. The field is
+	// retained so existing databricks.yml files using it still validate against
+	// the bundle schema.
 	ExperimentalIsUnifiedHost bool   `json:"experimental_is_unified_host,omitempty"`
 	AccountID                 string `json:"account_id,omitempty"`
 	WorkspaceID               string `json:"workspace_id,omitempty"`
@@ -135,9 +140,8 @@ func (w *Workspace) Config(ctx context.Context) *config.Config {
 		AzureLoginAppID:  w.AzureLoginAppID,
 
 		// Unified host
-		Experimental_IsUnifiedHost: w.ExperimentalIsUnifiedHost,
-		AccountID:                  w.AccountID,
-		WorkspaceID:                w.WorkspaceID,
+		AccountID:   w.AccountID,
+		WorkspaceID: w.WorkspaceID,
 	}
 
 	for k := range config.ConfigAttributes {

cmd/auth/auth.go

@@ -28,7 +28,7 @@ GCP: https://docs.gcp.databricks.com/dev-tools/auth/index.html`,
 	var authArguments auth.AuthArguments
 	cmd.PersistentFlags().StringVar(&authArguments.Host, "host", "", "Databricks Host")
 	cmd.PersistentFlags().StringVar(&authArguments.AccountID, "account-id", "", "Databricks Account ID")
-	cmd.PersistentFlags().BoolVar(&authArguments.IsUnifiedHost, "experimental-is-unified-host", false, "Flag to indicate if the host is a unified host")
+	cmd.PersistentFlags().BoolVar(&authArguments.IsUnifiedHost, "experimental-is-unified-host", false, "Deprecated: unified hosts are now detected automatically from /.well-known/databricks-config. Still honored as a routing fallback.")
 	cmd.PersistentFlags().StringVar(&authArguments.WorkspaceID, "workspace-id", "", "Databricks Workspace ID")
 
 	cmd.AddCommand(newEnvCommand())

cmd/auth/login.go

@@ -279,27 +279,22 @@ a new profile is created.
 		var clusterID, serverlessComputeID string
 
 		// Keys to explicitly remove from the profile. OAuth login always
-		// clears incompatible credential fields (PAT, basic auth, M2M).
+		// clears incompatible credential fields (PAT, basic auth, M2M) and
+		// the deprecated experimental_is_unified_host key (routing now comes
+		// from .well-known discovery, so stale values would be misleading).
 		clearKeys := oauthLoginClearKeys()
-
-		// Boolean false is zero-valued and skipped by SaveToProfile's IsZero
-		// check. Explicitly clear experimental_is_unified_host when false so
-		// it doesn't remain sticky from a previous login.
-		if !authArguments.IsUnifiedHost {
-			clearKeys = append(clearKeys, "experimental_is_unified_host")
-		}
+		clearKeys = append(clearKeys, "experimental_is_unified_host")
 
 		switch {
 		case configureCluster:
 			// Create a workspace client to list clusters for interactive selection.
 			// We use a custom CredentialsStrategy that wraps the token we just minted,
 			// avoiding the need to spawn a child CLI process (which AuthType "databricks-cli" does).
 			w, err := databricks.NewWorkspaceClient(&databricks.Config{
-				Host:                       authArguments.Host,
-				AccountID:                  authArguments.AccountID,
-				WorkspaceID:                authArguments.WorkspaceID,
-				Experimental_IsUnifiedHost: authArguments.IsUnifiedHost,
-				Credentials:                config.NewTokenSourceStrategy("login-token", authconv.AuthTokenSource(persistentAuth)),
+				Host:        authArguments.Host,
+				AccountID:   authArguments.AccountID,
+				WorkspaceID: authArguments.WorkspaceID,
+				Credentials: config.NewTokenSourceStrategy("login-token", authconv.AuthTokenSource(persistentAuth)),
 			})
 			if err != nil {
 				return err
@@ -320,17 +315,19 @@ a new profile is created.
 		}
 
 		if profileName != "" {
+			// experimental_is_unified_host is no longer written to new profiles.
+			// Routing now comes from .well-known discovery; stale keys on existing
+			// profiles are cleaned up via clearKeys above.
 			err := databrickscfg.SaveToProfile(ctx, &config.Config{
-				Profile:                    profileName,
-				Host:                       authArguments.Host,
-				AuthType:                   authTypeDatabricksCLI,
-				AccountID:                  authArguments.AccountID,
-				WorkspaceID:                authArguments.WorkspaceID,
-				Experimental_IsUnifiedHost: authArguments.IsUnifiedHost,
-				ClusterID:                  clusterID,
-				ConfigFile:                 env.Get(ctx, "DATABRICKS_CONFIG_FILE"),
-				ServerlessComputeID:        serverlessComputeID,
-				Scopes:                     scopesList,
+				Profile:             profileName,
+				Host:                authArguments.Host,
+				AuthType:            authTypeDatabricksCLI,
+				AccountID:           authArguments.AccountID,
+				WorkspaceID:         authArguments.WorkspaceID,
+				ClusterID:           clusterID,
+				ConfigFile:          env.Get(ctx, "DATABRICKS_CONFIG_FILE"),
+				ServerlessComputeID: serverlessComputeID,
+				Scopes:              scopesList,
 			}, clearKeys...)
 			if err != nil {
 				return err
@@ -407,52 +404,33 @@ func setHostAndAccountId(ctx context.Context, existingProfile *profile.Profile,
 	// are logged as warnings and never block login.
 	runHostDiscovery(ctx, authArguments)
 
-	// Determine the host type and handle account ID / workspace ID accordingly
-	cfg := &config.Config{
-		Host:                       authArguments.Host,
-		AccountID:                  authArguments.AccountID,
-		WorkspaceID:                authArguments.WorkspaceID,
-		Experimental_IsUnifiedHost: authArguments.IsUnifiedHost,
-	}
-
-	switch cfg.HostType() { //nolint:staticcheck // HostType() deprecated in SDK v0.127.0; SDK moving to host-agnostic behavior.
-	case config.AccountHost:
-		// Account host: prompt for account ID if not provided
-		if authArguments.AccountID == "" {
-			if existingProfile != nil && existingProfile.AccountID != "" {
-				authArguments.AccountID = existingProfile.AccountID
-			} else {
-				accountId, err := promptForAccountID(ctx)
-				if err != nil {
-					return err
-				}
-				authArguments.AccountID = accountId
-			}
-		}
-	case config.UnifiedHost:
-		// Unified host requires an account ID for OAuth URL construction.
-		// Workspace selection happens post-OAuth via promptForWorkspaceSelection.
-		if authArguments.AccountID == "" {
-			if existingProfile != nil && existingProfile.AccountID != "" {
-				authArguments.AccountID = existingProfile.AccountID
-			} else {
-				accountId, err := promptForAccountID(ctx)
-				if err != nil {
-					return err
-				}
-				authArguments.AccountID = accountId
+	if needsAccountIDPrompt(authArguments.Host, authArguments.IsUnifiedHost, authArguments.DiscoveryURL) && authArguments.AccountID == "" {
+		if existingProfile != nil && existingProfile.AccountID != "" {
+			authArguments.AccountID = existingProfile.AccountID
+		} else {
+			accountId, err := promptForAccountID(ctx)
+			if err != nil {
+				return err
 			}
+			authArguments.AccountID = accountId
 		}
-	case config.WorkspaceHost:
-		// Regular workspace host: no additional prompts needed.
-		// If discovery already populated account_id/workspace_id, those are kept.
-	default:
-		return fmt.Errorf("unknown host type: %v", cfg.HostType()) //nolint:staticcheck // HostType() deprecated in SDK v0.127.0; SDK moving to host-agnostic behavior.
 	}
 
 	return nil
 }
 
+// needsAccountIDPrompt reports whether the target host requires an account ID
+// for OAuth URL construction. True for classic account hosts (accounts.*) and
+// for unified hosts (either legacy flag or account-scoped DiscoveryURL).
+func needsAccountIDPrompt(host string, isUnifiedHost bool, discoveryURL string) bool {
+	canonicalHost := (&config.Config{Host: host}).CanonicalHostName()
+	if strings.HasPrefix(canonicalHost, "https://accounts.") ||
+		strings.HasPrefix(canonicalHost, "https://accounts-dod.") {
+		return true
+	}
+	return auth.HasUnifiedHostSignal(discoveryURL, isUnifiedHost)
+}
+
 // runHostDiscovery calls EnsureResolved() with a temporary config to fetch
 // .well-known/databricks-config from the host. Populates account_id and
 // workspace_id from discovery if not already set.

cmd/auth/login_test.go

@@ -392,6 +392,31 @@ func TestShouldUseDiscovery(t *testing.T) {
 	}
 }
 
+func TestNeedsAccountIDPrompt(t *testing.T) {
+	cases := []struct {
+		name          string
+		host          string
+		isUnifiedHost bool
+		discoveryURL  string
+		want          bool
+	}{
+		{name: "classic accounts host", host: "https://accounts.cloud.databricks.com", want: true},
+		{name: "accounts-dod host", host: "https://accounts-dod.databricks.com", want: true},
+		{name: "accounts host with path", host: "https://accounts.cloud.databricks.com/some/path", want: true},
+		{name: "plain workspace host", host: "https://workspace.cloud.databricks.com"},
+		{name: "unified flag set", host: "https://spog.cloud.databricks.com", isUnifiedHost: true, want: true},
+		{name: "account-scoped DiscoveryURL", host: "https://spog.cloud.databricks.com", discoveryURL: "https://spog.cloud.databricks.com/oidc/accounts/acct-123/.well-known/oauth-authorization-server", want: true},
+		{name: "workspace-scoped DiscoveryURL", host: "https://workspace.cloud.databricks.com", discoveryURL: "https://workspace.cloud.databricks.com/oidc/.well-known/oauth-authorization-server"},
+		{name: "workspace host no signals", host: "https://workspace.cloud.databricks.com"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := needsAccountIDPrompt(tc.host, tc.isUnifiedHost, tc.discoveryURL)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
 func TestSplitScopes(t *testing.T) {
 	tests := []struct {
 		name   string

cmd/auth/profiles.go

@@ -29,6 +29,11 @@ type profileMetadata struct {
 	AuthType    string `json:"auth_type"`
 	Valid       bool   `json:"valid"`
 	Default     bool   `json:"default,omitempty"`
+
+	// isUnifiedHost carries the legacy experimental_is_unified_host value so we
+	// can route unified-host profiles without the SDK field (which is being
+	// removed). Not serialized.
+	isUnifiedHost bool
 }
 
 func (c *profileMetadata) IsEmpty() bool {
@@ -57,7 +62,7 @@ func (c *profileMetadata) Load(ctx context.Context, configFilePath string, skipV
 		return
 	}
 
-	configType := auth.ResolveConfigType(cfg)
+	configType := auth.ResolveConfigType(cfg, c.isUnifiedHost)
 	if configType != cfg.ConfigType() {
 		log.Debugf(ctx, "Profile %q: overrode config type from %s to %s (SPOG host)", c.Name, cfg.ConfigType(), configType)
 	}
@@ -126,11 +131,12 @@ func newProfilesCommand() *cobra.Command {
 		for _, v := range iniFile.Sections() {
 			hash := v.KeysHash()
 			profile := &profileMetadata{
-				Name:        v.Name(),
-				Host:        hash["host"],
-				AccountID:   hash["account_id"],
-				WorkspaceID: hash["workspace_id"],
-				Default:     v.Name() == defaultProfile,
+				Name:          v.Name(),
+				Host:          hash["host"],
+				AccountID:     hash["account_id"],
+				WorkspaceID:   hash["workspace_id"],
+				Default:       v.Name() == defaultProfile,
+				isUnifiedHost: hash["experimental_is_unified_host"] == "true",
 			}
 			if profile.IsEmpty() {
 				continue

cmd/auth/profiles_test.go

@@ -201,45 +201,6 @@ func TestProfileLoadSPOGConfigType(t *testing.T) {
 	}
 }
 
-func TestProfileLoadUnifiedHostFallback(t *testing.T) {
-	// When Experimental_IsUnifiedHost is set but .well-known is unreachable,
-	// ConfigType() returns InvalidConfig. The fallback should reclassify as
-	// AccountConfig so the profile is still validated.
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/json")
-		switch r.URL.Path {
-		case "/.well-known/databricks-config":
-			w.WriteHeader(http.StatusNotFound)
-		case "/api/2.0/accounts/unified-acct/workspaces":
-			_ = json.NewEncoder(w).Encode([]map[string]any{})
-		default:
-			w.WriteHeader(http.StatusNotFound)
-		}
-	}))
-	t.Cleanup(server.Close)
-
-	dir := t.TempDir()
-	configFile := filepath.Join(dir, ".databrickscfg")
-	t.Setenv("HOME", dir)
-	if runtime.GOOS == "windows" {
-		t.Setenv("USERPROFILE", dir)
-	}
-
-	content := "[unified-profile]\nhost = " + server.URL + "\ntoken = test-token\naccount_id = unified-acct\nexperimental_is_unified_host = true\n"
-	require.NoError(t, os.WriteFile(configFile, []byte(content), 0o600))
-
-	p := &profileMetadata{
-		Name:      "unified-profile",
-		Host:      server.URL,
-		AccountID: "unified-acct",
-	}
-	p.Load(t.Context(), configFile, false)
-
-	assert.True(t, p.Valid, "unified host profile should be valid via fallback")
-	assert.NotEmpty(t, p.Host)
-	assert.NotEmpty(t, p.AuthType)
-}
-
 func TestClassicAccountsHostConfigType(t *testing.T) {
 	// Classic accounts.* hosts can't be tested through Load() because httptest
 	// generates 127.0.0.1 URLs. Verify directly that ConfigType() classifies
@@ -256,7 +217,7 @@ func TestClassicAccountsHostConfigType(t *testing.T) {
 }
 
 func TestProfileLoadNoDiscoveryStaysWorkspace(t *testing.T) {
-	// When .well-known returns 404 and Experimental_IsUnifiedHost is false,
+	// When .well-known returns 404 and the unified-host fallback is false,
 	// the SPOG override should NOT trigger even if account_id is set. The
 	// profile should stay WorkspaceConfig and validate via CurrentUser.Me.
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {