apache · xylaaaaa · Aug 14, 2026
@@ -210,7 +210,8 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_
         // set other conf
         for (const THdfsConf& conf : hdfsParams.hdfs_conf) {
             builder->set_hdfs_conf(conf.key, conf.value);
-            LOG(INFO) << "set hdfs config key: " << conf.key << ", value: " << conf.value;
+            // HDFS configuration may contain short-lived storage credentials such as an ADLS SAS token.
+            LOG(INFO) << "set hdfs config key: " << conf.key;
             if (strcmp(conf.key.c_str(), "hadoop.security.authentication") == 0) {
                 auth_type = conf.value;
             }

@@ -150,6 +150,18 @@ under the License.
             <version>${iceberg.version}</version>
         </dependency>
 
+        <!-- Iceberg Azure integration: ADLSFileIO plus its shaded Azure SDK dependencies. -->
+        <dependency>
+            <groupId>org.apache.iceberg</groupId>
+            <artifactId>iceberg-azure</artifactId>
+            <version>${iceberg.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.iceberg</groupId>
+            <artifactId>iceberg-azure-bundle</artifactId>
+            <version>${iceberg.version}</version>
+        </dependency>
+
         <!-- The hms-flavor org.apache.iceberg.hive.HiveCatalog is NOT a direct dependency here: it is
              bundled (with the shared relocated thrift shade.doris.hive.org.apache.thrift) inside
              fe-connector-hms-hive-shade and reaches this plugin transitively via fe-connector-hms above —

@@ -65,6 +65,7 @@
 import org.apache.iceberg.io.FileIO;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.io.ResolvingFileIO;
 import org.apache.iceberg.io.StorageCredential;
 import org.apache.iceberg.io.SupportsStorageCredentials;
 import org.apache.iceberg.types.Conversions;
@@ -2979,6 +2980,18 @@ public void extractVendedTokenReturnsIoPropsWhenFileIoHasNoStorageCredentials()
                 IcebergScanPlanProvider.extractVendedToken(table, true));
     }
 
+    @Test
+    public void adlsFileIoIsAvailableForAbfssLocations() {
+        ResolvingFileIO fileIO = new ResolvingFileIO();
+        fileIO.initialize(Collections.emptyMap());
+        try {
+            Assertions.assertEquals("org.apache.iceberg.azure.adlsv2.ADLSFileIO",
+                    fileIO.ioClass("abfss://container@account.dfs.core.windows.net/table").getName());
+        } finally {
+            fileIO.close();
+        }
+    }
+
     @Test
     public void extractVendedTokenEmptyWhenFlagDisabled() {
         FakeIcebergTable table = fakeTable("t1");

@@ -235,7 +235,7 @@ public Map<String, String> vendStorageCredentials(Map<String, String> rawVendedC
     }
 
     /**
-     * Builds the vended {@link StorageAdapter} typed map from a raw per-table token: filter to
+     * Builds the vended {@link StorageAdapter} typed map from a raw per-table token: normalize
      * cloud-storage props, run {@link StorageAdapter#ofAll} (normalizes arbitrary token key
      * shapes + derives region/endpoint), then index by {@link StorageTypeId}. Mirrors the
      * legacy vended-credentials normalization tail exactly, so the BE-credential overlay
@@ -250,11 +250,11 @@ private Map<StorageTypeId, StorageAdapter> buildVendedStorageMap(
             return null;
         }
         try {
-            Map<String, String> filtered = CredentialUtils.filterCloudStorageProperties(rawVendedCredentials);
-            if (filtered.isEmpty()) {
+            Map<String, String> normalized = CredentialUtils.normalizeCloudStorageProperties(rawVendedCredentials);
+            if (normalized.isEmpty()) {
                 return null;
             }
-            List<StorageAdapter> vended = StorageAdapter.ofAll(filtered);
+            List<StorageAdapter> vended = StorageAdapter.ofAll(normalized);
             return vended.stream()
                     .collect(Collectors.toMap(StorageAdapter::getType, Function.identity()));
         } catch (Exception e) {

@@ -31,6 +31,11 @@
  */
 public class CredentialUtils {
 
+    private static final String ADLS_SAS_TOKEN_PREFIX = "adls.sas-token.";
+    private static final String ADLS_PROPERTY_PREFIX = "adls.";
+    private static final String HADOOP_AZURE_ACCOUNT_AUTH_TYPE_PREFIX = "fs.azure.account.auth.type.";
+    private static final String HADOOP_AZURE_FIXED_SAS_TOKEN_PREFIX = "fs.azure.sas.fixed.token.";
+
     /**
      * Supported cloud storage prefixes for filtering vended credentials
      */
@@ -42,6 +47,7 @@ public class CredentialUtils {
             "obs.",          // Huawei OBS
             "gs.",           // Google Cloud Storage
             "azure.",        // Microsoft Azure
+            "adls.",         // Iceberg Azure ADLS vended credentials
             "client.",       // Iceberg client properties (e.g., client.region)
             "iceberg.rest."  // Iceberg REST catalog properties (e.g., iceberg.rest.access-key-id)
     ));
@@ -67,6 +73,35 @@ public static Map<String, String> filterCloudStorageProperties(Map<String, Strin
         return filtered;
     }
 
+    /**
+     * Convert cloud storage credentials to the properties consumed by Doris storage adapters.
+     * Databricks Unity Catalog returns Azure SAS credentials using Iceberg's
+     * {@code adls.sas-token.<account-host>} property. Hadoop ABFS instead consumes an account-scoped
+     * authentication type and fixed SAS token, so translate that representation before selecting the
+     * storage adapter. Remove the raw Iceberg {@code adls.*} property names after translating them to
+     * the equivalent backend configuration.
+     *
+     * @param rawVendedCredentials Raw vended credentials map
+     * @return Normalized cloud storage properties
+     */
+    public static Map<String, String> normalizeCloudStorageProperties(
+            Map<String, String> rawVendedCredentials) {
+        Map<String, String> normalized = filterCloudStorageProperties(rawVendedCredentials);
+        Map<String, String> adlsSasTokens = new HashMap<>();
+        normalized.forEach((key, value) -> {
+            if (key.startsWith(ADLS_SAS_TOKEN_PREFIX)) {
+                String accountHost = key.substring(ADLS_SAS_TOKEN_PREFIX.length());
+                adlsSasTokens.put(accountHost, value);
+            }
+        });
+        normalized.keySet().removeIf(key -> key.startsWith(ADLS_PROPERTY_PREFIX));
+        adlsSasTokens.forEach((accountHost, sasToken) -> {
+            normalized.put(HADOOP_AZURE_ACCOUNT_AUTH_TYPE_PREFIX + accountHost, "SAS");
+            normalized.put(HADOOP_AZURE_FIXED_SAS_TOKEN_PREFIX + accountHost, sasToken);
+        });
+        return normalized;
+    }
+
     /**
      * Extract backend properties from StorageAdapter map
      * Reference: CatalogProperty.getBackendStorageProperties()

@@ -17,6 +17,9 @@
 
 package org.apache.doris.connector;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+import org.apache.hadoop.fs.azurebfs.services.AuthType;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -60,6 +63,30 @@ public void normalizesOssTokenToBackendAwsProps() {
         Assertions.assertEquals("testSessionToken789", be.get("AWS_TOKEN"));
     }
 
+    @Test
+    public void normalizesAdlsTokenToBackendAbfsProps() throws Exception {
+        String accountHost = "account.dfs.core.windows.net";
+        String sasToken = "testSasToken";
+        Map<String, String> token = Map.of(
+                "adls.sas-token." + accountHost, sasToken,
+                "adls.sas-token-expires-at-ms." + accountHost, "4102444800000");
+
+        Map<String, String> be = context().vendStorageCredentials(token);
+
+        String authTypeKey = "fs.azure.account.auth.type." + accountHost;
+        String fixedSasTokenKey = "fs.azure.sas.fixed.token." + accountHost;
+        Assertions.assertEquals("SAS", be.get(authTypeKey));
+        Assertions.assertEquals(sasToken, be.get(fixedSasTokenKey));
+        Assertions.assertTrue(be.keySet().stream().noneMatch(key -> key.startsWith("adls.")));
+
+        Configuration configuration = new Configuration(false);
+        be.forEach(configuration::set);
+        AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, accountHost);
+        Assertions.assertEquals(AuthType.SAS, abfsConfiguration.getAuthType(accountHost));
+        Assertions.assertEquals(sasToken,
+                abfsConfiguration.getSASTokenProvider().getSASToken(accountHost, "container", "/table", "read"));
+    }
+
     @Test
     public void emptyOrNullInputYieldsEmpty() {
         // WHY: a non-REST / no-token table passes an empty map; the bridge must short-circuit to

@@ -92,6 +92,23 @@ public void testFilterCloudStoragePropertiesWithMultipleCloudTypes() {
         Assertions.assertFalse(filtered.containsKey("hdfs.namenode"));
     }
 
+    @Test
+    public void testFilterCloudStoragePropertiesWithAdlsVendedCredentials() {
+        String accountHost = "account.dfs.core.windows.net";
+        Map<String, String> rawCredentials = new HashMap<>();
+        rawCredentials.put("adls.sas-token." + accountHost, "testSasToken");
+        rawCredentials.put("adls.sas-token-expires-at-ms." + accountHost, "4102444800000");
+        rawCredentials.put("table.name", "test_table");
+
+        Map<String, String> filtered = CredentialUtils.filterCloudStorageProperties(rawCredentials);
+
+        Assertions.assertEquals(2, filtered.size());
+        Assertions.assertEquals("testSasToken", filtered.get("adls.sas-token." + accountHost));
+        Assertions.assertEquals("4102444800000",
+                filtered.get("adls.sas-token-expires-at-ms." + accountHost));
+        Assertions.assertFalse(filtered.containsKey("table.name"));
+    }
+
     @Test
     public void testFilterCloudStoragePropertiesWithEmptyInput() {
         Map<String, String> filtered = CredentialUtils.filterCloudStorageProperties(new HashMap<>());

@@ -1485,6 +1485,16 @@ under the License.
                 <artifactId>iceberg-aws</artifactId>
                 <version>${iceberg.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.iceberg</groupId>
+                <artifactId>iceberg-azure</artifactId>
+                <version>${iceberg.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.iceberg</groupId>
+                <artifactId>iceberg-azure-bundle</artifactId>
+                <version>${iceberg.version}</version>
+            </dependency>
             <!-- S3Tables catalog impl for the iceberg s3tables flavor; not part of the AWS SDK BOM. -->
             <dependency>
                 <groupId>software.amazon.s3tables</groupId>