auth: wire secure-storage cache into CLI

NEXT_CHANGELOG.md

@@ -10,6 +10,8 @@
 * Accept `yes` in addition to `y` for confirmation prompts, and show `[y/N]` to indicate that no is the default.
 * Deprecated `auth env`. The command is hidden from help listings and prints a deprecation warning to stderr; it will be removed in a future release.
 * Moved file-based OAuth token cache management from the SDK to the CLI. No user-visible change; part of a three-PR sequence that makes the CLI the sole owner of its token cache.
+* Added experimental OS-native secure token storage behind the `--secure-storage` flag on `databricks auth login` and the `DATABRICKS_AUTH_STORAGE=secure` environment variable. Hidden from help during MS1. Legacy file-backed token storage remains the default.
+* Added experimental OS-native secure token storage opt-in via `DATABRICKS_AUTH_STORAGE=secure` or `[__settings__].auth_storage = secure` in `.databrickscfg`. Legacy file-backed token storage remains the default.
 
 ### Bundles
 * Remove `experimental-jobs-as-code` template, superseded by `pydabs` ([#4999](https://github.com/databricks/cli/pull/4999)).

acceptance/cmd/auth/storage-modes/invalid-env/output.txt

@@ -0,0 +1,5 @@
+
+>>> [CLI] auth token --profile nonexistent
+Error: DATABRICKS_AUTH_STORAGE: unknown storage mode "bogus" (want legacy, secure, or plaintext)
+
+Exit code: 1

acceptance/cmd/auth/storage-modes/invalid-env/script

@@ -0,0 +1,6 @@
+export DATABRICKS_AUTH_STORAGE=bogus
+
+# Any auth command that resolves the storage mode must surface the error.
+# auth token is the smallest reproducer because it doesn't perform any
+# network I/O before resolving the mode.
+trace $CLI auth token --profile nonexistent

acceptance/cmd/auth/storage-modes/legacy-env-default/output.txt

@@ -0,0 +1,11 @@
+
+=== Token cache keys before logout
+[
+  "dev"
+]
+
+>>> [CLI] auth logout --profile dev --auto-approve
+Logged out of profile "dev". Use --delete to also remove it from the config file.
+
+=== Token cache keys after logout (should be empty)
+[]

acceptance/cmd/auth/storage-modes/legacy-env-default/script

@@ -0,0 +1,29 @@
+export DATABRICKS_AUTH_STORAGE=legacy
+
+cat > "./home/.databrickscfg" <<ENDCFG
+[dev]
+host = https://accounts.cloud.databricks.com
+account_id = account-id
+auth_type  = databricks-cli
+ENDCFG
+
+mkdir -p "./home/.databricks"
+cat > "./home/.databricks/token-cache.json" <<ENDCACHE
+{
+  "version": 1,
+  "tokens": {
+    "dev": {
+      "access_token": "dev-cached-token",
+      "token_type": "Bearer"
+    }
+  }
+}
+ENDCACHE
+
+title "Token cache keys before logout\n"
+jq -S '.tokens | keys' "./home/.databricks/token-cache.json"
+
+trace $CLI auth logout --profile dev --auto-approve
+
+title "Token cache keys after logout (should be empty)\n"
+jq -S '.tokens | keys' "./home/.databricks/token-cache.json"

acceptance/cmd/auth/storage-modes/script.prepare

@@ -0,0 +1,8 @@
+# Shared setup: isolate HOME and clear inherited auth env vars so each
+# storage-mode test is deterministic regardless of the runner.
+sethome "./home"
+
+unset DATABRICKS_HOST
+unset DATABRICKS_TOKEN
+unset DATABRICKS_CONFIG_PROFILE
+unset DATABRICKS_AUTH_STORAGE

acceptance/cmd/auth/storage-modes/test.toml

@@ -0,0 +1,3 @@
+Ignore = [
+    "home"
+]

cmd/auth/login.go

@@ -60,6 +60,23 @@ func discoveryErr(msg string, err error) error {
 	return fmt.Errorf("%s%s", msg, discoveryFallbackTip)
 }
 
+// dualWriteLegacyHostKey mirrors the freshly minted token under the legacy
+// host-based cache key so users alternating between CLI and SDK find it.
+// Skipped for secure mode to avoid multiplying keyring entries.
+func dualWriteLegacyHostKey(ctx context.Context, tokenCache cache.TokenCache, arg u2m.OAuthArgument, mode storage.StorageMode) {
+	if mode != storage.StorageModeLegacy {
+		return
+	}
+	t, err := tokenCache.Lookup(arg.GetCacheKey())
+	if err != nil || t == nil {
+		return
+	}
+	dual := storage.NewDualWritingTokenCache(tokenCache, arg)
+	if err := dual.Store(arg.GetCacheKey(), t); err != nil {
+		log.Debugf(ctx, "token cache dual-write failed: %v", err)
+	}
+}
+
 type discoveryPersistentAuth interface {
 	Challenge() error
 	Token() (*oauth2.Token, error)
@@ -147,6 +164,11 @@ a new profile is created.
 		ctx := cmd.Context()
 		profileName := cmd.Flag("profile").Value.String()
 
+		tokenCache, mode, err := storage.ResolveCache(ctx, "")
+		if err != nil {
+			return err
+		}
+
 		// Cluster and Serverless are mutually exclusive.
 		if configureCluster && configureServerless {
 			return errors.New("please either configure serverless or cluster, not both")
@@ -191,18 +213,22 @@ a new profile is created.
 			return err
 		}
 
-		tokenCache, err := storage.NewFileTokenCache(ctx)
-		if err != nil {
-			return fmt.Errorf("opening token cache: %w", err)
-		}
-
 		// If no host is available from any source, use the discovery flow
 		// via login.databricks.com.
 		if shouldUseDiscovery(authArguments.Host, args, existingProfile) {
 			if err := validateDiscoveryFlagCompatibility(cmd); err != nil {
 				return err
 			}
-			return discoveryLogin(ctx, &defaultDiscoveryClient{}, tokenCache, profileName, loginTimeout, scopes, existingProfile, getBrowserFunc(cmd))
+			return discoveryLogin(ctx, discoveryLoginInputs{
+				dc:              &defaultDiscoveryClient{},
+				profileName:     profileName,
+				timeout:         loginTimeout,
+				scopes:          scopes,
+				existingProfile: existingProfile,
+				browserFunc:     getBrowserFunc(cmd),
+				tokenCache:      tokenCache,
+				mode:            mode,
+			})
 		}
 
 		// Load unified host flag from the profile if not explicitly set via CLI flag.
@@ -235,9 +261,9 @@ a new profile is created.
 			return err
 		}
 		persistentAuthOpts := []u2m.PersistentAuthOption{
-			u2m.WithTokenCache(storage.NewDualWritingTokenCache(tokenCache, oauthArgument)),
 			u2m.WithOAuthArgument(oauthArgument),
 			u2m.WithBrowser(getBrowserFunc(cmd)),
+			u2m.WithTokenCache(tokenCache),
 		}
 		if len(scopesList) > 0 {
 			persistentAuthOpts = append(persistentAuthOpts, u2m.WithScopes(scopesList))
@@ -254,6 +280,7 @@ a new profile is created.
 		if err = persistentAuth.Challenge(); err != nil {
 			return err
 		}
+		dualWriteLegacyHostKey(ctx, tokenCache, oauthArgument, mode)
 		// At this point, an OAuth token has been successfully minted and stored
 		// in the CLI cache. The rest of the command focuses on:
 		// 1. Workspace selection for SPOG hosts (best-effort);
@@ -567,35 +594,48 @@ func validateDiscoveryFlagCompatibility(cmd *cobra.Command) error {
 	return nil
 }
 
+// discoveryLoginInputs groups the dependencies of discoveryLogin.
+// See https://google.github.io/styleguide/go/best-practices#option-structure.
+type discoveryLoginInputs struct {
+	dc              discoveryClient
+	profileName     string
+	timeout         time.Duration
+	scopes          string
+	existingProfile *profile.Profile
+	browserFunc     func(string) error
+	tokenCache      cache.TokenCache
+	mode            storage.StorageMode
+}
+
 // discoveryLogin runs the login.databricks.com discovery flow. The user
 // authenticates in the browser, selects a workspace, and the CLI receives
 // the workspace host from the OAuth callback's iss parameter.
-func discoveryLogin(ctx context.Context, dc discoveryClient, tokenCache cache.TokenCache, profileName string, timeout time.Duration, scopes string, existingProfile *profile.Profile, browserFunc func(string) error) error {
-	arg, err := dc.NewOAuthArgument(profileName)
+func discoveryLogin(ctx context.Context, in discoveryLoginInputs) error {
+	arg, err := in.dc.NewOAuthArgument(in.profileName)
 	if err != nil {
 		return discoveryErr("setting up login.databricks.com", err)
 	}
 
-	scopesList := splitScopes(scopes)
-	if len(scopesList) == 0 && existingProfile != nil && existingProfile.Scopes != "" {
-		scopesList = splitScopes(existingProfile.Scopes)
+	scopesList := splitScopes(in.scopes)
+	if len(scopesList) == 0 && in.existingProfile != nil && in.existingProfile.Scopes != "" {
+		scopesList = splitScopes(in.existingProfile.Scopes)
 	}
 
 	opts := []u2m.PersistentAuthOption{
-		u2m.WithTokenCache(storage.NewDualWritingTokenCache(tokenCache, arg)),
 		u2m.WithOAuthArgument(arg),
-		u2m.WithBrowser(browserFunc),
+		u2m.WithBrowser(in.browserFunc),
 		u2m.WithDiscoveryLogin(),
+		u2m.WithTokenCache(in.tokenCache),
 	}
 	if len(scopesList) > 0 {
 		opts = append(opts, u2m.WithScopes(scopesList))
 	}
 
 	// Apply timeout before creating PersistentAuth so Challenge() respects it.
-	ctx, cancel := context.WithTimeout(ctx, timeout)
+	ctx, cancel := context.WithTimeout(ctx, in.timeout)
 	defer cancel()
 
-	persistentAuth, err := dc.NewPersistentAuth(ctx, opts...)
+	persistentAuth, err := in.dc.NewPersistentAuth(ctx, opts...)
 	if err != nil {
 		return discoveryErr("setting up login.databricks.com", err)
 	}
@@ -605,6 +645,7 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, tokenCache cache.To
 	if err := persistentAuth.Challenge(); err != nil {
 		return discoveryErr("login via login.databricks.com failed", err)
 	}
+	dualWriteLegacyHostKey(ctx, in.tokenCache, arg, in.mode)
 
 	discoveredHost := arg.GetDiscoveredHost()
 	if discoveredHost == "" {
@@ -627,7 +668,7 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, tokenCache cache.To
 		return fmt.Errorf("retrieving token after login: %w", err)
 	}
 
-	introspection, err := dc.IntrospectToken(ctx, discoveredHost, tok.AccessToken)
+	introspection, err := in.dc.IntrospectToken(ctx, discoveredHost, tok.AccessToken)
 	if err != nil {
 		log.Debugf(ctx, "token introspection failed (non-fatal): %v", err)
 	} else {
@@ -638,10 +679,10 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, tokenCache cache.To
 			accountID = introspection.AccountID
 		}
 
-		if existingProfile != nil && existingProfile.AccountID != "" && introspection.AccountID != "" &&
-			existingProfile.AccountID != introspection.AccountID {
+		if in.existingProfile != nil && in.existingProfile.AccountID != "" && introspection.AccountID != "" &&
+			in.existingProfile.AccountID != introspection.AccountID {
 			log.Warnf(ctx, "detected account ID %q differs from existing profile account ID %q",
-				introspection.AccountID, existingProfile.AccountID)
+				introspection.AccountID, in.existingProfile.AccountID)
 		}
 	}
 
@@ -660,7 +701,7 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, tokenCache cache.To
 		"serverless_compute_id",
 	)
 	err = databrickscfg.SaveToProfile(ctx, &config.Config{
-		Profile:     profileName,
+		Profile:     in.profileName,
 		Host:        discoveredHost,
 		AuthType:    authTypeDatabricksCLI,
 		AccountID:   accountID,
@@ -670,12 +711,12 @@ func discoveryLogin(ctx context.Context, dc discoveryClient, tokenCache cache.To
 	}, clearKeys...)
 	if err != nil {
 		if configFile != "" {
-			return fmt.Errorf("saving profile %q to %s: %w", profileName, configFile, err)
+			return fmt.Errorf("saving profile %q to %s: %w", in.profileName, configFile, err)
 		}
-		return fmt.Errorf("saving profile %q: %w", profileName, err)
+		return fmt.Errorf("saving profile %q: %w", in.profileName, err)
 	}
 
-	cmdio.LogString(ctx, fmt.Sprintf("Profile %s was successfully saved", profileName))
+	cmdio.LogString(ctx, fmt.Sprintf("Profile %s was successfully saved", in.profileName))
 	return nil
 }