From ef3f98f7ccd4bd02db193725e5cfb04275b7c211 Mon Sep 17 00:00:00 2001 From: rtrivedi Date: Thu, 19 Mar 2026 14:08:23 -0500 Subject: [PATCH 1/3] HIVE-29513: Increase HikariCP connection timeout default from 30s to 60s --- .../datasource/HikariCPDataSourceProvider.java | 4 ++++ .../TestDataSourceProviderFactory.java | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java index 91f1ea3d5574..9c59c3d7e639 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java @@ -40,6 +40,7 @@ public class HikariCPDataSourceProvider implements DataSourceProvider { private static final Logger LOG = LoggerFactory.getLogger(HikariCPDataSourceProvider.class); static final String HIKARI = "hikaricp"; + static final long DEFAULT_CONNECTION_TIMEOUT_MS = 60000; @Override public DataSource create(Configuration hdpConfig, int maxPoolSize) throws SQLException { @@ -66,6 +67,9 @@ public DataSource create(Configuration hdpConfig, int maxPoolSize) throws SQLExc config.setPoolName(poolName); } + long connectionTimeout = hdpConfig.getLong(HIKARI + ".connectionTimeout", DEFAULT_CONNECTION_TIMEOUT_MS); + config.setConnectionTimeout(connectionTimeout); + // It's kind of a waste to create a fixed size connection pool as same as the TxnHandler#connPool, // TxnHandler#connPoolMutex is mostly used for MutexAPI that is primarily designed to // provide coarse-grained mutex support to maintenance tasks running inside the Metastore, diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java index 9e6690315cf8..4f660903967d 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java @@ -72,7 +72,7 @@ public void testDefaultHikariCpProperties() throws SQLException { DataSource ds = dsp.create(conf); Assert.assertTrue(ds instanceof HikariDataSource); HikariDataSource hds = (HikariDataSource) ds; - Assert.assertEquals(30000L, hds.getConnectionTimeout()); + Assert.assertEquals(60000L, hds.getConnectionTimeout()); Assert.assertEquals(1800000L, hds.getMaxLifetime()); Assert.assertEquals(0L, hds.getLeakDetectionThreshold()); Assert.assertEquals(1L, hds.getInitializationFailTimeout()); @@ -99,6 +99,20 @@ public void testSetHikariCpProperties() throws SQLException { Assert.assertEquals(-1L, hds.getInitializationFailTimeout()); } + @Test + public void testHikariCpConnectionTimeout60Seconds() throws SQLException { + MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI); + conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1"); + + DataSourceProvider dsp = DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf); + Assert.assertNotNull(dsp); + + DataSource ds = dsp.create(conf); + Assert.assertTrue(ds instanceof HikariDataSource); + HikariDataSource hds = (HikariDataSource) ds; + Assert.assertEquals(60000L, hds.getConnectionTimeout()); + } + @Test public void testHikariCpMaxPoolSize() { MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI); From a83ba28d3872b6b6ef4f41b25be4fca905e1896c Mon Sep 17 00:00:00 2001 From: rtrivedi Date: Wed, 25 Mar 2026 11:31:29 -0500 Subject: [PATCH 2/3] HIVE-29513: Addressed review comments --- .../HikariCPDataSourceProvider.java | 6 +++- .../TestDataSourceProviderFactory.java | 34 +++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java index 9c59c3d7e639..8d947c94eba2 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java @@ -31,6 +31,7 @@ import java.sql.SQLException; import java.util.Map; import java.util.Properties; +import java.util.concurrent.TimeUnit; /** * DataSourceProvider for the HikariCP connection pool. @@ -52,6 +53,8 @@ public DataSource create(Configuration hdpConfig, int maxPoolSize) throws SQLExc String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig); Properties properties = replacePrefix(DataSourceProvider.getPrefixedProperties(hdpConfig, HIKARI)); + // connectionTimeout is handled separately with getTimeDuration() + properties.remove("connectionTimeout"); HikariConfig config; try { @@ -67,7 +70,8 @@ public DataSource create(Configuration hdpConfig, int maxPoolSize) throws SQLExc config.setPoolName(poolName); } - long connectionTimeout = hdpConfig.getLong(HIKARI + ".connectionTimeout", DEFAULT_CONNECTION_TIMEOUT_MS); + long connectionTimeout = hdpConfig.getTimeDuration(HIKARI + ".connectionTimeout", + DEFAULT_CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS); config.setConnectionTimeout(connectionTimeout); // It's kind of a waste to create a fixed size connection pool as same as the TxnHandler#connPool, diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java index 4f660903967d..ca2a7b6b16db 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java @@ -100,17 +100,29 @@ public void testSetHikariCpProperties() throws SQLException { } @Test - public void testHikariCpConnectionTimeout60Seconds() throws SQLException { - MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI); - conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1"); - - DataSourceProvider dsp = DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf); - Assert.assertNotNull(dsp); - - DataSource ds = dsp.create(conf); - Assert.assertTrue(ds instanceof HikariDataSource); - HikariDataSource hds = (HikariDataSource) ds; - Assert.assertEquals(60000L, hds.getConnectionTimeout()); + public void testHikariCpConnectionTimeout() throws SQLException { + Object[][] timeoutConfigs = { + {"90000", 90000L}, + {"60s", 60000L}, + {"90000ms", 90000L} + }; + + for (Object[] config : timeoutConfigs) { + String timeoutValue = (String) config[0]; + long expectedValue = (long) config[1]; + + MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI); + conf.set(HikariCPDataSourceProvider.HIKARI + ".connectionTimeout", timeoutValue); + conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1"); + + DataSourceProvider dsp = DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf); + Assert.assertNotNull(dsp); + + DataSource ds = dsp.create(conf); + Assert.assertTrue(ds instanceof HikariDataSource); + HikariDataSource hds = (HikariDataSource) ds; + Assert.assertEquals(expectedValue, hds.getConnectionTimeout()); + } } @Test From 392620698d22bd115076046cb716730abf293e1c Mon Sep 17 00:00:00 2001 From: rtrivedi Date: Wed, 8 Apr 2026 11:19:11 -0500 Subject: [PATCH 3/3] HIVE-29513: Addressed review comments --- .../datasource/DataSourceProvider.java | 32 ++++++++++++++++--- .../HikariCPDataSourceProvider.java | 16 ++++------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java index 78fa406444bc..44160f2390fd 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java @@ -20,7 +20,9 @@ import java.io.Closeable; import java.io.IOException; import java.sql.SQLException; +import java.util.Map; import java.util.Properties; +import java.util.concurrent.TimeUnit; import javax.sql.DataSource; import com.google.common.collect.Iterables; @@ -51,16 +53,36 @@ default DataSource create(Configuration hdpConfig) throws SQLException { * @return The pooling type string associated with the data source. */ String getPoolingType(); - + /** - * @param hdpConfig + * @param hdpConfig Hadoop configuration object + * @param prefix the prefix for connection pool specific properties + * @param timeDurationConfigs map of property names (without prefix) to their default values in milliseconds. * @return subset of properties prefixed by a connection pool specific substring */ - static Properties getPrefixedProperties(Configuration hdpConfig, String factoryPrefix) { + static Properties getPrefixedProperties(Configuration hdpConfig, String prefix, + Map timeDurationConfigs) { Properties dataSourceProps = new Properties(); Iterables.filter( - hdpConfig, (entry -> entry.getKey() != null && entry.getKey().startsWith(factoryPrefix))) - .forEach(entry -> dataSourceProps.put(entry.getKey(), entry.getValue())); + hdpConfig, (entry -> entry.getKey() != null && entry.getKey().startsWith(prefix))) + .forEach(entry -> { + String fullKey = entry.getKey(); + String keyName = fullKey.substring(prefix.length() + 1); + Long defaultVal = timeDurationConfigs.get(keyName); + if (defaultVal != null) { + long timeMs = hdpConfig.getTimeDuration(fullKey, defaultVal, TimeUnit.MILLISECONDS); + dataSourceProps.setProperty(fullKey, String.valueOf(timeMs)); + } else { + dataSourceProps.setProperty(fullKey, entry.getValue()); + } + }); + + // Setting defaults for time duration configs if not set already + timeDurationConfigs.forEach((keyName, defaultVal) -> { + String fullKey = prefix + "." + keyName; + dataSourceProps.putIfAbsent(fullKey, String.valueOf(defaultVal)); + }); + return dataSourceProps; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java index 8d947c94eba2..e715a7a1188b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java @@ -31,7 +31,6 @@ import java.sql.SQLException; import java.util.Map; import java.util.Properties; -import java.util.concurrent.TimeUnit; /** * DataSourceProvider for the HikariCP connection pool. @@ -43,6 +42,10 @@ public class HikariCPDataSourceProvider implements DataSourceProvider { static final String HIKARI = "hikaricp"; static final long DEFAULT_CONNECTION_TIMEOUT_MS = 60000; + private static final Map TIME_DURATION_CONFIGS = Map.of( + "connectionTimeout", DEFAULT_CONNECTION_TIMEOUT_MS + ); + @Override public DataSource create(Configuration hdpConfig, int maxPoolSize) throws SQLException { String poolName = DataSourceProvider.getDataSourceName(hdpConfig); @@ -52,10 +55,9 @@ public DataSource create(Configuration hdpConfig, int maxPoolSize) throws SQLExc String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig); String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig); - Properties properties = replacePrefix(DataSourceProvider.getPrefixedProperties(hdpConfig, HIKARI)); - // connectionTimeout is handled separately with getTimeDuration() - properties.remove("connectionTimeout"); - + Properties properties = replacePrefix( + DataSourceProvider.getPrefixedProperties(hdpConfig, HIKARI, TIME_DURATION_CONFIGS)); + HikariConfig config; try { config = new HikariConfig(properties); @@ -70,10 +72,6 @@ public DataSource create(Configuration hdpConfig, int maxPoolSize) throws SQLExc config.setPoolName(poolName); } - long connectionTimeout = hdpConfig.getTimeDuration(HIKARI + ".connectionTimeout", - DEFAULT_CONNECTION_TIMEOUT_MS, TimeUnit.MILLISECONDS); - config.setConnectionTimeout(connectionTimeout); - // It's kind of a waste to create a fixed size connection pool as same as the TxnHandler#connPool, // TxnHandler#connPoolMutex is mostly used for MutexAPI that is primarily designed to // provide coarse-grained mutex support to maintenance tasks running inside the Metastore,