Pass --force-refresh to Databricks CLI auth token command

The SDK manages its own token caching via `CachedTokenSource`. When
the SDK shells out to `databricks auth token`, the CLI may return a
token from *its* own cache that is about to expire (or has already
expired from the SDK's perspective), producing unnecessary refresh
failures and retry loops.

The CLI added `--force-refresh` in v0.296.0
(databricks/cli#4767) to let callers bypass
its cache. With the version-detection infrastructure from the parent
PR already in place, opting in is a one-constant, one-branch change:

* Introduce `CLI_VERSION_FOR_FORCE_REFRESH = v0.296.0`.
* Split `buildCliCommand` into the existing profile/host decision
  (now `buildCoreCliCommand`) and a thin wrapper that appends
  `--force-refresh` when supported and otherwise logs a precise
  warning.

Future capability-gated flags slot into the same wrapper.

Mirrors:
* databricks/databricks-sdk-go#1628
* databricks/databricks-sdk-py#1378

Co-authored-by: Isaac

Author: mihaimitrea-db

NEXT_CHANGELOG.md

@@ -15,5 +15,6 @@
 
 ### Internal Changes
 * Detect Databricks CLI version at init time via `databricks version --output json`, enabling version-gated flag support. Successful detections are cached per CLI path; subprocess failures fall back to the most conservative command and are retried on the next call.
+* Pass `--force-refresh` to Databricks CLI `auth token` command (when the installed CLI is >= v0.296.0) so the SDK always receives a freshly minted token instead of a potentially stale one from the CLI's internal cache.
 
 ### API Changes

databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksCliCredentialsProvider.java

@@ -35,6 +35,10 @@ public class DatabricksCliCredentialsProvider implements CredentialsProvider {
   // --profile support added in CLI v0.207.1: https://github.com/databricks/cli/pull/855
   static final DatabricksCliVersion CLI_VERSION_FOR_PROFILE = new DatabricksCliVersion(0, 207, 1);
 
+  // --force-refresh support added in CLI v0.296.0: https://github.com/databricks/cli/pull/4767
+  static final DatabricksCliVersion CLI_VERSION_FOR_FORCE_REFRESH =
+      new DatabricksCliVersion(0, 296, 0);
+
   // 5-second cap on `databricks version` so a hung CLI (slow first-run scan, antivirus, blocked
   // stdin) does not wedge SDK init indefinitely.
   private static final long VERSION_PROBE_TIMEOUT_SECONDS = 5;
@@ -161,13 +165,40 @@ List<String> resolveCliCommand(String cliPath, DatabricksConfig config) {
   }
 
   /**
-   * Builds the {@code auth token} command for the given CLI version.
+   * Builds the full {@code auth token} command, including capability-gated flags.
    *
-   * <p>Falls back to {@code --host} when {@code --profile} is either not configured or not
-   * supported by the installed CLI.
+   * <p>Delegates the profile/host decision to {@link #buildCoreCliCommand} and appends {@code
+   * --force-refresh} when the installed CLI supports it.
    */
   List<String> buildCliCommand(
       String cliPath, DatabricksConfig config, DatabricksCliVersion version) {
+    List<String> cmd = buildCoreCliCommand(cliPath, config, version);
+    if (version.atLeast(CLI_VERSION_FOR_FORCE_REFRESH)) {
+      cmd.add("--force-refresh");
+    } else if (version.equals(DatabricksCliVersion.UNKNOWN) || version.isDefaultDevBuild()) {
+      // Detection failed or no version metadata — we can't prove the CLI lacks --force-refresh,
+      // just failed to confirm it. The version probe already logged the underlying cause.
+      LOG.warn(
+          "Could not confirm --force-refresh support for Databricks CLI {} (requires >= {}). "
+              + "The CLI's token cache may provide stale tokens.",
+          version,
+          CLI_VERSION_FOR_FORCE_REFRESH);
+    } else {
+      LOG.warn(
+          "Databricks CLI {} does not support --force-refresh (requires >= {}). "
+              + "The CLI's token cache may provide stale tokens.",
+          version,
+          CLI_VERSION_FOR_FORCE_REFRESH);
+    }
+    return cmd;
+  }
+
+  /**
+   * Builds the base {@code auth token} command without capability-gated flags. Falls back to {@code
+   * --host} when {@code --profile} is either not configured or not supported by the installed CLI.
+   */
+  List<String> buildCoreCliCommand(
+      String cliPath, DatabricksConfig config, DatabricksCliVersion version) {
     if (config.getProfile() == null) {
       return buildHostArgs(cliPath, config);
     }

databricks-sdk-java/src/test/java/com/databricks/sdk/core/DatabricksCliCredentialsProviderTest.java

@@ -111,33 +111,56 @@ void testBuildCliCommand_ProfileWithNullHost_ThrowsClearError() {
   private static Stream<Arguments> buildCliCommandCases() {
     return Stream.of(
         Arguments.of(
-            "host only — old CLI",
+            "host only — old CLI, no force-refresh",
             new DatabricksConfig().setHost(HOST),
             new DatabricksCliVersion(0, 200, 0),
             Arrays.asList(CLI_PATH, "auth", "token", "--host", HOST)),
         Arguments.of(
-            "account host — old CLI",
+            "host only — new CLI, with force-refresh",
+            new DatabricksConfig().setHost(HOST),
+            DatabricksCliCredentialsProvider.CLI_VERSION_FOR_FORCE_REFRESH,
+            Arrays.asList(CLI_PATH, "auth", "token", "--host", HOST, "--force-refresh")),
+        Arguments.of(
+            "account host — old CLI, no force-refresh",
             new DatabricksConfig().setHost(ACCOUNT_HOST).setAccountId(ACCOUNT_ID),
             new DatabricksCliVersion(0, 200, 0),
             Arrays.asList(
                 CLI_PATH, "auth", "token", "--host", ACCOUNT_HOST, "--account-id", ACCOUNT_ID)),
         Arguments.of(
-            "profile with new CLI — uses --profile",
+            "account host — new CLI, with force-refresh",
+            new DatabricksConfig().setHost(ACCOUNT_HOST).setAccountId(ACCOUNT_ID),
+            DatabricksCliCredentialsProvider.CLI_VERSION_FOR_FORCE_REFRESH,
+            Arrays.asList(
+                CLI_PATH,
+                "auth",
+                "token",
+                "--host",
+                ACCOUNT_HOST,
+                "--account-id",
+                ACCOUNT_ID,
+                "--force-refresh")),
+        Arguments.of(
+            "profile with profile-supporting CLI — uses --profile, no force-refresh",
             new DatabricksConfig().setProfile(PROFILE).setHost(HOST),
             DatabricksCliCredentialsProvider.CLI_VERSION_FOR_PROFILE,
             Arrays.asList(CLI_PATH, "auth", "token", "--profile", PROFILE)),
         Arguments.of(
-            "profile with old CLI — falls back to --host",
+            "profile with newest CLI — uses --profile and --force-refresh",
+            new DatabricksConfig().setProfile(PROFILE).setHost(HOST),
+            DatabricksCliCredentialsProvider.CLI_VERSION_FOR_FORCE_REFRESH,
+            Arrays.asList(CLI_PATH, "auth", "token", "--profile", PROFILE, "--force-refresh")),
+        Arguments.of(
+            "profile with old CLI — falls back to --host, no force-refresh",
             new DatabricksConfig().setProfile(PROFILE).setHost(HOST),
             new DatabricksCliVersion(0, 207, 0),
             Arrays.asList(CLI_PATH, "auth", "token", "--host", HOST)),
         Arguments.of(
-            "unknown version — falls back to --host",
+            "unknown version — falls back to --host, no force-refresh",
             new DatabricksConfig().setProfile(PROFILE).setHost(HOST),
             DatabricksCliVersion.UNKNOWN,
             Arrays.asList(CLI_PATH, "auth", "token", "--host", HOST)),
         Arguments.of(
-            "dev build — falls back to --host",
+            "dev build — falls back to --host, no force-refresh",
             new DatabricksConfig().setProfile(PROFILE).setHost(HOST),
             new DatabricksCliVersion(0, 0, 0),
             Arrays.asList(CLI_PATH, "auth", "token", "--host", HOST)));