diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDatabaseTableDefault.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDatabaseTableDefault.java index a64bba594f60..3c5ab869c055 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDatabaseTableDefault.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDatabaseTableDefault.java @@ -98,6 +98,7 @@ public void setUp() throws Exception { TestTxnDbUtil.prepDb(hiveConf); SessionState.start(new SessionState(hiveConf)); + MetastoreConf.setBoolVar(hiveConf, ConfVars.HIVE_IN_TEST, false); d = DriverFactory.newDriver(hiveConf); wh = new File(System.getProperty("java.io.tmpdir") + File.separator + @@ -110,7 +111,6 @@ public void setUp() throws Exception { MetastoreConf.setVar(hiveConf, ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS, "org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer"); - MetastoreConf.setBoolVar(hiveConf, ConfVars.HIVE_IN_TEST, false); MetastoreConf.setVar(hiveConf, ConfVars.WAREHOUSE, wh.getCanonicalPath()); MetastoreConf.setVar(hiveConf, ConfVars.WAREHOUSE_EXTERNAL, ext_wh.getCanonicalPath()); MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID, true); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 61dbfdd633c0..447102f91c49 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -55,6 +55,10 @@ import org.apache.hadoop.hive.common.io.QTestFetchConverter; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.DatabaseType; +import org.apache.hadoop.hive.metastore.api.GetDatabaseObjectsRequest; +import org.apache.hadoop.hive.metastore.api.GetDatabaseObjectsResponse; import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreClientWithLocalCache; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.QTestMiniClusters.FsType; @@ -355,6 +359,31 @@ public void clearUDFsCreatedDuringTests() throws Exception { } } + public void clearConnectorsCreatedInTests() throws Exception { + if (System.getenv(QTEST_LEAVE_FILES) != null) { + return; + } + conf.set("hive.metastore.filter.hook", "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl"); + db = Hive.get(conf); + + GetDatabaseObjectsRequest request = new GetDatabaseObjectsRequest(); + request.setPattern("*"); + GetDatabaseObjectsResponse response = db.getMSC().get_databases_req(request); + if (response != null && response.getDatabasesSize() > 0) { + for (Database database : response.getDatabases()) { + if (database.getType() == DatabaseType.REMOTE) { + db.dropDatabase(database.getName(), true, true); + } + } + } + List connectors = db.getAllDataConnectorNames(); + if (connectors != null) { + for (String connectorName : connectors) { + db.dropDataConnector(connectorName, true); + } + } + } + /** * Clear out any side effects of running tests */ @@ -482,6 +511,7 @@ public void clearTestSideEffects() throws Exception { Utilities.clearWorkMap(conf); NotificationEventPoll.shutdown(); QueryResultsCache.cleanupInstance(); + clearConnectorsCreatedInTests(); clearTablesCreatedDuringTests(); clearUDFsCreatedDuringTests(); clearKeysCreatedInTests(); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index 842b9390af3e..5a5d7ac95dfc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -18,13 +18,11 @@ package org.apache.hadoop.hive.metastore; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.repl.ReplConst; import org.apache.hadoop.hive.metastore.api.*; @@ -39,6 +37,7 @@ import org.apache.hadoop.hive.metastore.handler.BaseHandler; import org.apache.hadoop.hive.metastore.handler.DropPartitionsHandler; import org.apache.hadoop.hive.metastore.handler.GetPartitionsHandler; +import org.apache.hadoop.hive.metastore.handler.GetTableHandler; import org.apache.hadoop.hive.metastore.handler.PrivilegeHandler; import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType; import org.apache.hadoop.hive.metastore.metrics.PerfLogger; @@ -76,7 +75,6 @@ import static org.apache.hadoop.hive.metastore.ExceptionHandler.throwMetaException; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; -import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HIVE_IN_TEST; import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.getPartValsFromName; import static org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.isDbReplicationTarget; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME; @@ -91,7 +89,6 @@ */ public class HMSHandler extends PrivilegeHandler { public static final Logger LOG = LoggerFactory.getLogger(HMSHandler.class); - private final boolean isInTest; private StorageSchemaReader storageSchemaReader; public static final String PARTITION_NUMBER_EXCEED_LIMIT_MSG = @@ -134,7 +131,6 @@ public static String getIPAddress() { public HMSHandler(String name, Configuration conf) { super(name, conf); - isInTest = MetastoreConf.getBoolVar(this.conf, HIVE_IN_TEST); } @Override @@ -1406,7 +1402,7 @@ public TruncateTableResponse truncate_table_req(TruncateTableRequest req) Exception ex = null; boolean success = false; try { - success = AbstractRequestHandler.offer(this, req).success(); + success = AbstractRequestHandler.offer(this, req).success(); return new TruncateTableResponse(); } catch (Exception e) { ex = e; @@ -1420,149 +1416,34 @@ public TruncateTableResponse truncate_table_req(TruncateTableRequest req) } @Override - public List get_tables_ext(final GetTablesExtRequest req) throws MetaException { - List tables = new ArrayList(); - List ret = new ArrayList(); - String catalog = req.getCatalog(); - String database = req.getDatabase(); - String pattern = req.getTableNamePattern(); - List processorCapabilities = req.getProcessorCapabilities(); - int limit = req.getLimit(); - String processorId = req.getProcessorIdentifier(); - List tObjects = new ArrayList<>(); - - startTableFunction("get_tables_ext", catalog, database, pattern); - Exception ex = null; - try { - tables = getMS().getTables(catalog, database, pattern, null, limit); - LOG.debug("get_tables_ext:getTables() returned " + tables.size()); - tables = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled, filterHook, - catalog, database, tables); - - if (tables.size() > 0) { - tObjects = getMS().getTableObjectsByName(catalog, database, tables); - LOG.debug("get_tables_ext:getTableObjectsByName() returned " + tObjects.size()); - if (processorCapabilities == null || processorCapabilities.size() == 0 || - processorCapabilities.contains("MANAGERAWMETADATA")) { - LOG.info("Skipping translation for processor with " + processorId); - } else { - if (transformer != null) { - Map> retMap = transformer.transform(tObjects, processorCapabilities, processorId); - - for (Map.Entry> entry : retMap.entrySet()) { - LOG.debug("Table " + entry.getKey().getTableName() + " requires " + Arrays.toString((entry.getValue()).toArray())); - ret.add(convertTableToExtendedTable(entry.getKey(), entry.getValue(), req.getRequestedFields())); - } - } else { - for (Table table : tObjects) { - ret.add(convertTableToExtendedTable(table, processorCapabilities, req.getRequestedFields())); - } - } - } - } - } catch (Exception e) { - ex = e; - throw newMetaException(e); - } finally { - endFunction("get_tables_ext", ret != null, ex); - } - return ret; - } - - private ExtendedTableInfo convertTableToExtendedTable (Table table, - List processorCapabilities, int mask) { - ExtendedTableInfo extTable = new ExtendedTableInfo(table.getTableName()); - if ((mask & GetTablesExtRequestFields.ACCESS_TYPE.getValue()) == GetTablesExtRequestFields.ACCESS_TYPE.getValue()) { - extTable.setAccessType(table.getAccessType()); - } - - if ((mask & GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue()) - == GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue()) { - extTable.setRequiredReadCapabilities(table.getRequiredReadCapabilities()); - extTable.setRequiredWriteCapabilities(table.getRequiredWriteCapabilities()); - } - - return extTable; + public List get_tables_ext(final GetTablesExtRequest req) throws TException { + req.setCatalog(req.isSetCatalog() ? req.getCatalog() : getDefaultCatalog(conf)); + return GetTableHandler.getTables( + () -> startTableFunction("get_tables_ext", req.getCatalog(), req.getDatabase(), req.getTableNamePattern()), + this, req, + t -> endFunction("get_tables_ext", t.getLeft() != null, t.getRight())); } @Override public GetTableResult get_table_req(GetTableRequest req) throws MetaException, NoSuchObjectException { req.setCatName(req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf)); - return new GetTableResult(getTableInternal(req)); - } - - /** - * This function retrieves table from metastore. If getColumnStats flag is true, - * then engine should be specified so the table is retrieve with the column stats - * for that engine. - */ - private Table getTableInternal(GetTableRequest getTableRequest) throws MetaException, NoSuchObjectException { - - Preconditions.checkArgument(!getTableRequest.isGetColumnStats() || getTableRequest.getEngine() != null, - "To retrieve column statistics with a table, engine parameter cannot be null"); - - if (isInTest) { - assertClientHasCapability(getTableRequest.getCapabilities(), ClientCapability.TEST_CAPABILITY, "Hive tests", - "get_table_req"); - } - - Table t = null; - startTableFunction("get_table", getTableRequest.getCatName(), getTableRequest.getDbName(), - getTableRequest.getTblName()); - Exception ex = null; - try { - t = get_table_core(getTableRequest); - if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) { - assertClientHasCapability(getTableRequest.getCapabilities(), ClientCapability.INSERT_ONLY_TABLES, - "insert-only tables", "get_table_req"); - } - - if (CollectionUtils.isEmpty(getTableRequest.getProcessorCapabilities()) || getTableRequest - .getProcessorCapabilities().contains("MANAGERAWMETADATA")) { - LOG.info("Skipping translation for processor with " + getTableRequest.getProcessorIdentifier()); - } else { - if (transformer != null) { - List
tList = new ArrayList<>(); - tList.add(t); - Map> ret = transformer - .transform(tList, getTableRequest.getProcessorCapabilities(), getTableRequest.getProcessorIdentifier()); - if (ret.size() > 1) { - LOG.warn("Unexpected resultset size:" + ret.size()); - throw new MetaException("Unexpected result from metadata transformer:return list size is " + ret.size()); - } - t = ret.keySet().iterator().next(); - } - } - - firePreEvent(new PreReadTableEvent(t, this)); - } catch (MetaException | NoSuchObjectException e) { - ex = e; - throw e; - } finally { - endFunction("get_table", t != null, ex, getTableRequest.getTblName()); - } - return t; + return new GetTableResult(GetTableHandler.getTable( + () -> startTableFunction("get_table", req.getCatName(), req.getDbName(), req.getTblName()), + this, req, false, + t -> endFunction("get_table", t.getLeft() != null, t.getRight(), req.getTblName()))); } @Override public List get_table_meta(String dbnames, String tblNames, List tblTypes) - throws MetaException, NoSuchObjectException { - List t = null; + throws TException { String[] parsedDbName = parseDbName(dbnames, conf); - startTableFunction("get_table_metas", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblNames); - Exception ex = null; - try { - t = getMS().getTableMeta(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblNames, tblTypes); - t = FilterUtils.filterTableMetasIfEnabled(isServerFilterEnabled, filterHook, t); - t = filterReadableTables(parsedDbName[CAT_NAME], t); - } catch (Exception e) { - ex = e; - throw newMetaException(e); - } finally { - endFunction("get_table_metas", t != null, ex); - } - return t; + return GetTableHandler.getTables( + () -> startTableFunction("get_table_metas", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblNames), + this, + GetTableHandler.GetTableNamesRequest.forTableMeta(dbnames, conf) + .byType(tblTypes != null && !tblTypes.isEmpty()? String.join(",", tblTypes) : null, tblNames), + t -> endFunction("get_table_metas", t.getLeft() != null, t.getRight(), join(t.getLeft(), ","))); } /** @@ -1572,45 +1453,7 @@ public List get_table_meta(String dbnames, String tblNames, List filterTablesByName(List
tables, List tableNames) { - List
filteredTables = new ArrayList<>(); - for (Table table : tables) { - if (tableNames.contains(table.getTableName())) { - filteredTables.add(table); - } - } - return filteredTables; - } - - private List
getRemoteTableObjectsInternal(String dbname, List tableNames, String pattern) throws MetaException { - String[] parsedDbName = parseDbName(dbname, conf); - try { - // retrieve tables from remote database - Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); - List
tables = DataConnectorProviderFactory.getDataConnectorProvider(db).getTables(null); - - // filtered out undesired tables - if (tableNames != null) { - tables = filterTablesByName(tables, tableNames); - } - - // set remote tables' local hive database reference - for (Table table : tables) { - table.setDbName(dbname); - } - - return FilterUtils.filterTablesIfEnabled(isServerFilterEnabled, filterHook, tables); - } catch (Exception e) { - LOG.warn("Unexpected exception while getting table(s) in remote database " + dbname , e); - if (isInTest) { - // ignore the exception - return new ArrayList
(); - } else { - throw newMetaException(e); - } - } - } - - private List
getTableObjectsInternal(String catName, String dbName, - List tableNames, - ClientCapabilities capabilities, - GetProjectionsSpec projectionsSpec, String tablePattern) - throws MetaException, InvalidOperationException, UnknownDBException { - if (isInTest) { - assertClientHasCapability(capabilities, ClientCapability.TEST_CAPABILITY, - "Hive tests", "get_table_objects_by_name_req"); - } - - if (projectionsSpec != null) { - if (!projectionsSpec.isSetFieldList() && (projectionsSpec.isSetIncludeParamKeyPattern() || - projectionsSpec.isSetExcludeParamKeyPattern())) { - throw new InvalidOperationException("Include and Exclude Param key are not supported."); - } - } - - List
tables = new ArrayList<>(); - startMultiTableFunction("get_multi_table", dbName, tableNames); - Exception ex = null; - int tableBatchSize = MetastoreConf.getIntVar(conf, - ConfVars.BATCH_RETRIEVE_MAX); - - try { - if (dbName == null || dbName.isEmpty()) { - throw new UnknownDBException("DB name is null or empty"); - } - RawStore ms = getMS(); - if(tablePattern != null){ - tables = ms.getTableObjectsByName(catName, dbName, tableNames, projectionsSpec, tablePattern); - }else { - if (tableNames == null) { - throw new InvalidOperationException(dbName + " cannot find null tables"); - } - - // The list of table names could contain duplicates. RawStore.getTableObjectsByName() - // only guarantees returning no duplicate table objects in one batch. If we need - // to break into multiple batches, remove duplicates first. - List distinctTableNames = tableNames; - if (distinctTableNames.size() > tableBatchSize) { - List lowercaseTableNames = new ArrayList<>(); - for (String tableName : tableNames) { - lowercaseTableNames.add(normalizeIdentifier(tableName)); - } - distinctTableNames = new ArrayList<>(new HashSet<>(lowercaseTableNames)); - } - - int startIndex = 0; - // Retrieve the tables from the metastore in batches. Some databases like - // Oracle cannot have over 1000 expressions in a in-list - while (startIndex < distinctTableNames.size()) { - int endIndex = Math.min(startIndex + tableBatchSize, distinctTableNames.size()); - tables.addAll(ms.getTableObjectsByName(catName, dbName, distinctTableNames.subList( - startIndex, endIndex), projectionsSpec, tablePattern)); - startIndex = endIndex; - } - } - for (Table t : tables) { - if (t.getParameters() != null && MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) { - assertClientHasCapability(capabilities, ClientCapability.INSERT_ONLY_TABLES, - "insert-only tables", "get_table_req"); - } - } - - tables = FilterUtils.filterTablesIfEnabled(isServerFilterEnabled, filterHook, tables); - } catch (Exception e) { - ex = e; - throw handleException(e) - .throwIfInstance(MetaException.class, InvalidOperationException.class, UnknownDBException.class) - .defaultMetaException(); - } finally { - endFunction("get_multi_table", tables != null, ex, join(tableNames, ",")); - } - return tables; + List
tables = GetTableHandler.getTables( + () -> startMultiTableFunction("get_multi_table", req.getDbName(), req.getTblNames()), + this, req, + t -> endFunction("get_multi_table", t.getLeft() != null, t.getRight(), join(req.getTblNames(), ","))); + return new GetTablesResult(tables); } @Override @@ -1757,54 +1481,14 @@ public void update_creation_metadata(String catName, final String dbName, final getMS().updateCreationMetadata(catName, dbName, tableName, cm); } - private void assertClientHasCapability(ClientCapabilities client, - ClientCapability value, String what, String call) throws MetaException { - if (!doesClientHaveCapability(client, value)) { - throw new MetaException("Your client does not appear to support " + what + ". To skip" - + " capability checks, please set " + ConfVars.CAPABILITY_CHECK.toString() - + " to false. This setting can be set globally, or on the client for the current" - + " metastore session. Note that this may lead to incorrect results, data loss," - + " undefined behavior, etc. if your client is actually incompatible. You can also" - + " specify custom client capabilities via " + call + " API."); - } - } - - private boolean doesClientHaveCapability(ClientCapabilities client, ClientCapability value) { - if (!MetastoreConf.getBoolVar(getConf(), ConfVars.CAPABILITY_CHECK)) { - return true; - } - return (client != null && client.isSetValues() && client.getValues().contains(value)); - } - @Override public List get_table_names_by_filter( final String dbName, final String filter, final short maxTables) - throws MetaException, InvalidOperationException, UnknownDBException { - List tables = null; - startFunction("get_table_names_by_filter", ": db = " + dbName + ", filter = " + filter); - Exception ex = null; - String[] parsedDbName = parseDbName(dbName, conf); - try { - if (parsedDbName[CAT_NAME] == null || parsedDbName[CAT_NAME].isEmpty() || - parsedDbName[DB_NAME] == null || parsedDbName[DB_NAME].isEmpty()) { - throw new UnknownDBException("DB name is null or empty"); - } - if (filter == null) { - throw new InvalidOperationException(filter + " cannot apply null filter"); - } - tables = getMS().listTableNamesByFilter(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], filter, maxTables); - tables = FilterUtils.filterTableNamesIfEnabled( - isServerFilterEnabled, filterHook, parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tables); - - } catch (Exception e) { - ex = e; - throw handleException(e) - .throwIfInstance(MetaException.class, InvalidOperationException.class, UnknownDBException.class) - .defaultMetaException(); - } finally { - endFunction("get_table_names_by_filter", tables != null, ex, join(tables, ",")); - } - return tables; + throws TException { + return GetTableHandler.getTables( + () -> startFunction("get_table_names_by_filter", ": db = " + dbName + ", filter = " + filter), + this, GetTableHandler.GetTableNamesRequest.fromDatabase(dbName, conf).byFilter(filter, maxTables), + t -> endFunction("get_table_names_by_filter", t.getLeft() != null, t.getRight(), join(t.getLeft(), ","))); } @Override @@ -2136,20 +1820,21 @@ public DropPartitionsResult drop_partitions_req( @Override public boolean drop_partition_req(final DropPartitionRequest dropPartitionReq) throws TException { String dbName = dropPartitionReq.getDbName(); - String catName = dropPartitionReq.getCatName(); + String catName = dropPartitionReq.isSetCatName() ? dropPartitionReq.getCatName() : getDefaultCatalog(conf); String tbl_name = dropPartitionReq.getTblName(); List part_vals = dropPartitionReq.getPartVals(); try { Table t = getMS().getTable(catName, dbName, tbl_name, null); if (t == null) { - throw new InvalidObjectException(dbName + "." + tbl_name + " table not found"); + throw new NoSuchObjectException(dbName + "." + tbl_name + " table not found"); } List partNames = new ArrayList<>(); if (part_vals == null || part_vals.isEmpty()) { part_vals = getPartValsFromName(t, dropPartitionReq.getPartName()); } partNames.add(Warehouse.makePartName(t.getPartitionKeys(), part_vals)); - LOG.info("drop_partition_req partition values: {}", part_vals); + LOG.info("drop_partition_req partition values: {}, table: {}", part_vals, + new TableName(catName, dbName, tbl_name)); RequestPartsSpec requestPartsSpec = RequestPartsSpec.names(partNames); DropPartitionsRequest request = new DropPartitionsRequest(dbName, tbl_name, requestPartsSpec); request.setCatName(catName); @@ -2289,7 +1974,7 @@ public PartitionValuesResponse get_partition_values(PartitionValuesRequest reque return resps.getFirst(); } catch (Exception e) { ex = e; - throw handleException(e).throwIfInstance(MetaException.class).defaultMetaException(); + throw handleException(e).defaultMetaException(); } finally { endFunction("get_partition_values", ex == null, ex, tableName.toString()); } @@ -2474,86 +2159,19 @@ private void alter_table_core(String catName, String dbname, String name, Table @Override public List get_tables(final String dbname, final String pattern) - throws MetaException { - startFunction("get_tables", ": db=" + dbname + " pat=" + pattern); - - List ret = null; - Exception ex = null; - String[] parsedDbName = parseDbName(dbname, conf); - try { - if (isDatabaseRemote(dbname)) { - Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); - return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames(); - } - } catch (Exception e) { - throw newMetaException(e); - } - - try { - ret = getMS().getTables(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], pattern); - if(ret != null && !ret.isEmpty()) { - List
tableInfo = new ArrayList<>(); - tableInfo = getMS().getTableObjectsByName(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], ret); - tableInfo = FilterUtils.filterTablesIfEnabled(isServerFilterEnabled, filterHook, tableInfo);// tableInfo object has the owner information of the table which is being passed to FilterUtils. - ret = new ArrayList<>(); - for (Table tbl : tableInfo) { - ret.add(tbl.getTableName()); - } - } - } catch (Exception e) { - ex = e; - throw newMetaException(e); - } finally { - endFunction("get_tables", ret != null, ex); - } - return ret; + throws TException { + return GetTableHandler.getTables(() -> startFunction("get_tables", ": db=" + dbname + " pat=" + pattern), + this, GetTableHandler.GetTableNamesRequest.fromDatabase(dbname, conf).byPattern(pattern), + t -> endFunction("get_tables", t.getLeft() != null, t.getRight())); } @Override public List get_tables_by_type(final String dbname, final String pattern, final String tableType) - throws MetaException { - startFunction("get_tables_by_type", ": db=" + dbname + " pat=" + pattern + ",type=" + tableType); - - List ret = null; - Exception ex = null; - String[] parsedDbName = parseDbName(dbname, conf); - try { - ret = getTablesByTypeCore(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], pattern, tableType); - ret = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled, filterHook, - parsedDbName[CAT_NAME], parsedDbName[DB_NAME], ret); - } catch (Exception e) { - ex = e; - throw newMetaException(e); - } finally { - endFunction("get_tables_by_type", ret != null, ex); - } - return ret; - } - - private List getTablesByTypeCore(final String catName, final String dbname, - final String pattern, final String tableType) throws MetaException { - startFunction("getTablesByTypeCore", ": catName=" + catName + - ": db=" + dbname + " pat=" + pattern + ",type=" + tableType); - - List ret = null; - Exception ex = null; - Database db = null; - try { - db = get_database_core(catName, dbname); - if (MetaStoreUtils.isDatabaseRemote(db)) { - return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames(); - } - } catch (Exception e) { /* ignore */ } - - try { - ret = getMS().getTables(catName, dbname, pattern, TableType.valueOf(tableType), -1); - } catch (Exception e) { - ex = e; - throw newMetaException(e); - } finally { - endFunction("getTablesByTypeCore", ret != null, ex); - } - return ret; + throws TException { + return GetTableHandler.getTables( + () -> startFunction("get_tables_by_type", ": db=" + dbname + " pat=" + pattern + ",type=" + tableType), + this, GetTableHandler.GetTableNamesRequest.fromDatabase(dbname, conf).byType(tableType, pattern), + t -> endFunction("get_tables_by_type", t.getLeft() != null, t.getRight())); } @Override @@ -2594,35 +2212,19 @@ public List get_materialized_views_for_rewriting(final String dbname) } @Override - public List get_all_tables(final String dbname) throws MetaException { - startFunction("get_all_tables", ": db=" + dbname); - - List ret = null; - Exception ex = null; - String[] parsedDbName = parseDbName(dbname, conf); + public List get_all_tables(final String dbname) throws TException { try { - if (isDatabaseRemote(dbname)) { - Database db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); - return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames(); + return GetTableHandler.getTables(() -> startFunction("get_all_tables", ": db=" + dbname), this, + GetTableHandler.GetTableNamesRequest.fromDatabase(dbname, conf), + t -> endFunction("get_all_tables", t.getLeft() != null, t.getRight())); + } catch (UnknownDBException ude) { + String[] parsedDbName = parseDbName(dbname, conf); + if (StringUtils.isEmpty(parsedDbName[DB_NAME])) { + throw new MetaException(ude.getMessage()); } - } catch (Exception e) { /* ignore */ } - - try { - if (isServerFilterEnabled) { - List filteredTableMetas = get_table_meta(dbname, "*", null); - ret = filteredTableMetas.stream() - .map(TableMeta::getTableName) - .collect(Collectors.toList()); - } else { - ret = getMS().getAllTables(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); - } - } catch (Exception e) { - ex = e; - throw newMetaException(e); - } finally { - endFunction("get_all_tables", ret != null, ex); + // should throw the exception instead? in our tests we return an empty list if dbName is valid + return Collections.emptyList(); } - return ret; } private List get_fields_with_environment_context_core(String db, String tableName, final EnvironmentContext envContext) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/BaseHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/BaseHandler.java index b91af16a35cf..5c4f5d868ba4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/BaseHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/BaseHandler.java @@ -24,6 +24,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; +import com.google.common.base.Supplier; import com.google.common.util.concurrent.Striped; import javax.jdo.JDOException; @@ -89,12 +90,10 @@ import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo; import org.apache.hadoop.hive.metastore.api.Role; -import org.apache.hadoop.hive.metastore.api.TableMeta; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; import org.apache.hadoop.hive.metastore.events.PreEventContext; -import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent; import org.apache.hadoop.hive.metastore.messaging.EventMessage; import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; @@ -123,7 +122,6 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName; -import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; /** * This class serves as the super class for all handlers that implement the IHMSHandler @@ -878,36 +876,6 @@ public CmRecycleResponse cm_recycle(final CmRecycleRequest request) throws MetaE return new CmRecycleResponse(); } - /** - * filters out the table meta for which read database access is not granted - * @param catName catalog name - * @param tableMetas list of table metas - * @return filtered list of table metas - * @throws RuntimeException - * @throws NoSuchObjectException - */ - protected List filterReadableTables(String catName, List tableMetas) - throws RuntimeException, NoSuchObjectException { - List finalT = new ArrayList<>(); - Map databaseNames = new HashMap(); - for (TableMeta tableMeta : tableMetas) { - String fullDbName = prependCatalogToDbName(catName, tableMeta.getDbName(), conf); - if (databaseNames.get(fullDbName) == null) { - boolean isExecptionThrown = false; - try { - fireReadDatabasePreEvent(fullDbName); - } catch (MetaException e) { - isExecptionThrown = true; - } - databaseNames.put(fullDbName, isExecptionThrown); - } - if (!databaseNames.get(fullDbName)) { - finalT.add(tableMeta); - } - } - return finalT; - } - @Override public void setMetaConf(String key, String value) throws MetaException { MetastoreConf.ConfVars confVar = MetastoreConf.getMetaConf(key); @@ -981,27 +949,6 @@ public String getMetaConf(String key) throws MetaException { return getConf().get(key, confVar.getDefaultVal().toString()); } - /** - * Fire a pre-event for read database operation, if there are any - * pre-event listeners registered - */ - protected void fireReadDatabasePreEvent(final String name) - throws MetaException, RuntimeException, NoSuchObjectException { - if(preListeners.size() > 0) { - String[] parsedDbName = parseDbName(name, conf); - Database db = null; - try { - db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); - if (db == null) { - throw new NoSuchObjectException("Database: " + name + " not found"); - } - } catch(MetaException | NoSuchObjectException e) { - throw new RuntimeException(e); - } - firePreEvent(new PreReadDatabaseEvent(db, this)); - } - } - public void firePreEvent(PreEventContext event) throws MetaException { for (MetaStorePreEventListener listener : preListeners) { try { @@ -1013,4 +960,11 @@ public void firePreEvent(PreEventContext event) throws MetaException { } } } + + public void firePreEvent(Supplier supplier) throws MetaException { + if (preListeners.isEmpty()) { + return; + } + firePreEvent(supplier.get()); + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetPartitionsHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetPartitionsHandler.java index 2dcb768e83ee..4902ec3d8af5 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetPartitionsHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetPartitionsHandler.java @@ -61,7 +61,7 @@ // Collect get partitions APIs together @SuppressWarnings({"unchecked", "rawtypes"}) @RequestHandler(requestBody = GetPartitionsHandler.GetPartitionsRequest.class) -public class GetPartitionsHandler extends AbstractRequestHandler, +public class GetPartitionsHandler extends AbstractRequestHandler, GetPartitionsHandler.GetPartitionsResult> { private static final Logger LOG = LoggerFactory.getLogger(GetPartitionsHandler.class); private static final String NO_FILTER_STRING = ""; @@ -99,7 +99,7 @@ protected void beforeExecute() throws TException, IOException { @Override protected GetPartitionsResult execute() throws TException, IOException { - Req req = request.getReq(); + R req = request.getReq(); if (req instanceof PartitionValuesRequest pvq) { return getPartitionValues(pvq); } else if (req instanceof GetPartitionsByNamesRequest gpbr) { @@ -113,7 +113,7 @@ protected GetPartitionsResult execute() throws TException, IOException { } else if (req instanceof GetPartitionsPsWithAuthRequest gpar) { return getPartitionsByVals(gpar); } - throw new UnsupportedOperationException("Not yet implemented"); + throw new UnsupportedOperationException(req + " not yet implemented"); } private GetPartitionsResult getPartitionsByVals(GetPartitionsPsWithAuthRequest gpar) throws TException { @@ -341,23 +341,23 @@ public List result() { } } - public static class GetPartitionsRequest extends TAbstractBase { + public static class GetPartitionsRequest extends TAbstractBase { private final TableName tableName; private final boolean fetchPartNames; - private final Req req; + private final R req; - public GetPartitionsRequest(Req req, TableName tableName, + public GetPartitionsRequest(R req, TableName tableName, boolean fetchPartNames) { this.tableName = tableName; this.fetchPartNames = fetchPartNames; this.req = req; } - public GetPartitionsRequest(Req req, TableName tableName) { + public GetPartitionsRequest(R req, TableName tableName) { this(req, tableName, false); } - public Req getReq() { + public R getReq() { return req; } @@ -424,16 +424,16 @@ public static List getPartitions(Consumer preHook, } } - public static GetPartitionsResult getPartitionsResult( + public static GetPartitionsResult getPartitionsResult( Consumer preHook, Consumer> postHook, - IHMSHandler handler, TableName tableName, Req req) throws TException { + IHMSHandler handler, TableName tableName, R req) throws TException { GetPartitionsResult result = null; Exception ex = null; try { GetPartitionsRequest getPartitionsRequest = new GetPartitionsRequest(req, tableName); preHook.accept(tableName); - GetPartitionsHandler getPartsHandler = + GetPartitionsHandler getPartsHandler = AbstractRequestHandler.offer(handler, getPartitionsRequest); result = getPartsHandler.getResult(); return result; @@ -445,15 +445,15 @@ public static GetPartitionsResult getPartitionsResult( } } - public static GetPartitionsResult getPartitionNames(Consumer preExecutor, + public static GetPartitionsResult getPartitionNames(Consumer preExecutor, Consumer> postConsumer, IHMSHandler handler, TableName tableName, - Req req) throws TException { + R req) throws TException { Exception ex = null; GetPartitionsResult result = null; try { preExecutor.accept(tableName); GetPartitionsRequest getPartitionsRequest = new GetPartitionsRequest(req, tableName, true); - GetPartitionsHandler getPartNamesHandler = + GetPartitionsHandler getPartNamesHandler = AbstractRequestHandler.offer(handler, getPartitionsRequest); result = getPartNamesHandler.getResult(); return result; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetTableHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetTableHandler.java new file mode 100644 index 000000000000..6c69c693fd40 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetTableHandler.java @@ -0,0 +1,623 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.handler; + +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.DatabaseName; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.HMSHandler; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.IMetaStoreMetadataTransformer; +import org.apache.hadoop.hive.metastore.MetaStoreFilterHook; +import org.apache.hadoop.hive.metastore.RawStore; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.ClientCapabilities; +import org.apache.hadoop.hive.metastore.api.ClientCapability; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.ExtendedTableInfo; +import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec; +import org.apache.hadoop.hive.metastore.api.GetTableRequest; +import org.apache.hadoop.hive.metastore.api.GetTablesExtRequest; +import org.apache.hadoop.hive.metastore.api.GetTablesExtRequestFields; +import org.apache.hadoop.hive.metastore.api.GetTablesRequest; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TableMeta; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.dataconnector.DataConnectorProviderFactory; +import org.apache.hadoop.hive.metastore.events.PreEventContext; +import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.PreReadTableEvent; +import org.apache.hadoop.hive.metastore.utils.FilterUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.hive.metastore.ExceptionHandler.handleException; +import static org.apache.hadoop.hive.metastore.ExceptionHandler.newMetaException; +import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HIVE_IN_TEST; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.isDatabaseRemote; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; + +@SuppressWarnings({"unchecked", "rawtypes"}) +@RequestHandler(requestBody = GetTableHandler.GetTableReq.class) +public class GetTableHandler extends + AbstractRequestHandler, GetTableHandler.GetTableResult> { + private static final Logger LOG = LoggerFactory.getLogger(GetTableHandler.class); + private RawStore ms; + private IMetaStoreMetadataTransformer transformer; + private MetaStoreFilterHook filterHook; + private Configuration conf; + private boolean isInTest; + GetTableHandler(IHMSHandler handler, GetTableReq request) { + super(handler, false, request); + } + + @Override + protected void beforeExecute() throws TException, IOException { + this.ms = handler.getMS(); + this.transformer = handler.getMetadataTransformer(); + this.filterHook = handler.getMetaFilterHook(); + this.conf = handler.getConf(); + + this.isInTest = MetastoreConf.getBoolVar(handler.getConf(), HIVE_IN_TEST); + } + + @Override + protected GetTableResult execute() throws TException, IOException { + R req = request.getRequest(); + if (req instanceof GetTablesExtRequest getTablesExtRequest) { + List extendedTableInfos = getTablesExt(getTablesExtRequest); + return new GetTableResult(extendedTableInfos, true); + } else if (req instanceof GetTableRequest getTableRequest) { + Table table = request.rawTable ? getTableCore(getTableRequest) : getTable(getTableRequest); + return new GetTableResult(List.of(table), true); + } else if (req instanceof GetTablesRequest getTablesRequest) { + List
tables = getTableObjects(getTablesRequest); + return new GetTableResult(tables, true); + } else if (req instanceof GetTableNamesRequest getTableNames) { + return getTableNames.forTableMeta ? new GetTableResult(getTableMeta(getTableNames), true) : + new GetTableResult(getTableNames(getTableNames), true); + } + throw new UnsupportedOperationException(req + " not yet implemented"); + } + + private List getTablesExt(GetTablesExtRequest req) throws MetaException, UnknownDBException { + List ret = new ArrayList(); + String pattern = req.getTableNamePattern(); + List processorCapabilities = req.getProcessorCapabilities(); + int limit = req.getLimit(); + String catalog = req.isSetCatalog() ? req.getCatalog() : getDefaultCatalog(conf); + String database = req.getDatabase(); + String processorId = req.getProcessorIdentifier(); + List tables = ms.getTables(catalog, database, pattern, null, limit); + LOG.debug("get_tables_ext:getTables() returned {}", tables.size()); + tables = FilterUtils.filterTableNamesIfEnabled(filterHook != null, filterHook, + catalog, database, tables); + if (tables.isEmpty()) { + return ret; + } + List
tObjects = ms.getTableObjectsByName(catalog, database, tables); + LOG.debug("get_tables_ext:getTableObjectsByName() returned {}", tObjects.size()); + if (processorCapabilities == null || processorCapabilities.isEmpty() || + processorCapabilities.contains("MANAGERAWMETADATA")) { + LOG.info("Skipping translation for processor with {}", processorId); + } else { + if (transformer != null) { + Map> retMap = transformer.transform(tObjects, processorCapabilities, processorId); + for (Map.Entry> entry : retMap.entrySet()) { + LOG.debug("Table " + entry.getKey().getTableName() + " requires " + Arrays.toString((entry.getValue()).toArray())); + ret.add(convertTableToExtendedTable(entry.getKey(), entry.getValue(), req.getRequestedFields())); + } + } else { + for (Table table : tObjects) { + ret.add(convertTableToExtendedTable(table, processorCapabilities, req.getRequestedFields())); + } + } + } + return ret; + } + + private ExtendedTableInfo convertTableToExtendedTable(Table table, + List processorCapabilities, int mask) { + ExtendedTableInfo extTable = new ExtendedTableInfo(table.getTableName()); + if ((mask & GetTablesExtRequestFields.ACCESS_TYPE.getValue()) == GetTablesExtRequestFields.ACCESS_TYPE.getValue()) { + extTable.setAccessType(table.getAccessType()); + } + + if ((mask & GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue()) + == GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue()) { + extTable.setRequiredReadCapabilities(table.getRequiredReadCapabilities()); + extTable.setRequiredWriteCapabilities(table.getRequiredWriteCapabilities()); + } + + return extTable; + } + + /** + * This function retrieves table from metastore. If getColumnStats flag is true, + * then engine should be specified so the table is retrieve with the column stats + * for that engine. + */ + private Table getTable(GetTableRequest getTableRequest) throws MetaException, NoSuchObjectException { + + Preconditions.checkArgument(!getTableRequest.isGetColumnStats() || getTableRequest.getEngine() != null, + "To retrieve column statistics with a table, engine parameter cannot be null"); + + if (isInTest) { + assertClientHasCapability(getTableRequest.getCapabilities(), ClientCapability.TEST_CAPABILITY, "Hive tests", + "get_table_req"); + } + + Table t = getTableCore(getTableRequest); + if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) { + assertClientHasCapability(getTableRequest.getCapabilities(), ClientCapability.INSERT_ONLY_TABLES, + "insert-only tables", "get_table_req"); + } + + if (CollectionUtils.isEmpty(getTableRequest.getProcessorCapabilities()) || getTableRequest + .getProcessorCapabilities().contains("MANAGERAWMETADATA")) { + LOG.info("Skipping translation for processor with " + getTableRequest.getProcessorIdentifier()); + } else { + if (transformer != null) { + List
tList = new ArrayList<>(); + tList.add(t); + Map> ret = transformer + .transform(tList, getTableRequest.getProcessorCapabilities(), getTableRequest.getProcessorIdentifier()); + if (ret.size() > 1) { + LOG.warn("Unexpected resultset size:{}", ret.size()); + throw new MetaException("Unexpected result from metadata transformer:return list size is " + ret.size()); + } + t = ret.keySet().iterator().next(); + } + } + + ((HMSHandler) handler).firePreEvent(new PreReadTableEvent(t, handler)); + return t; + } + + /** + * This function retrieves table from metastore. If getColumnStats flag is true, + * then engine should be specified so the table is retrieve with the column stats + * for that engine. + * This method is slightly different from {@link #getTable(GetTableRequest)}, it retrieves the table stored as it is + * without transformation and access check, and is called inside the Metastore service. + */ + private Table getTableCore(GetTableRequest getTableRequest) throws MetaException, NoSuchObjectException { + Preconditions.checkArgument(!getTableRequest.isGetColumnStats() || getTableRequest.getEngine() != null, + "To retrieve column statistics with a table, engine parameter cannot be null"); + String catName = getTableRequest.getCatName(); + String dbName = getTableRequest.getDbName(); + String tblName = getTableRequest.getTblName(); + Database db = null; + Table t; + try { + db = handler.get_database_core(catName, dbName); + } catch (Exception e) { /* appears exception is not thrown currently if db doesnt exist */ } + + if (MetaStoreUtils.isDatabaseRemote(db)) { + t = DataConnectorProviderFactory.getDataConnectorProvider(db).getTable(tblName); + if (t == null) { + throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName) + " table not found"); + } + t.setDbName(dbName); + return t; + } + + t = ms.getTable(catName, dbName, tblName, getTableRequest.getValidWriteIdList(), getTableRequest.getId()); + if (t == null) { + throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName) + " table not found"); + } + + // If column statistics was requested and is valid fetch it. + if (getTableRequest.isGetColumnStats()) { + ColumnStatistics colStats = ms.getTableColumnStatistics(catName, dbName, tblName, + StatsSetupConst.getColumnsHavingStats(t.getParameters()), getTableRequest.getEngine(), + getTableRequest.getValidWriteIdList()); + if (colStats != null) { + t.setColStats(colStats); + } + } + return t; + } + + private List
getTableObjects(GetTablesRequest req) throws TException { + String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf); + String dbName = req.getDbName(); + if (dbName == null || dbName.isEmpty()) { + throw new UnknownDBException("DB name is null or empty"); + } + try { + Database database = handler.get_database_core(catName, dbName); + if (isDatabaseRemote(database)) { + return getRemoteTableObjectsInternal(database, req.getTblNames(), req.getTablesPattern()); + } + } catch (NoSuchObjectException nse) { + // The caller consumes UnknownDBException other than NoSuchObjectException + // in case database doesn't exist + throw new UnknownDBException("Could not find database " + DatabaseName.getQualified(catName, dbName)); + } + return getTableObjectsInternal(req); + } + + private List
filterTablesByName(List
tables, List tableNames) { + List
filteredTables = new ArrayList<>(); + for (Table table : tables) { + if (tableNames.contains(table.getTableName())) { + filteredTables.add(table); + } + } + return filteredTables; + } + + private List
getRemoteTableObjectsInternal(Database db, List tableNames, String pattern) throws MetaException { + try { + List
tables = DataConnectorProviderFactory.getDataConnectorProvider(db).getTables(null); + // filtered out undesired tables + if (tableNames != null) { + tables = filterTablesByName(tables, tableNames); + } + // set remote tables' local hive database reference + for (Table table : tables) { + table.setDbName(db.getName()); + } + return FilterUtils.filterTablesIfEnabled(filterHook != null, filterHook, tables); + } catch (Exception e) { + LOG.warn("Unexpected exception while getting table(s) in remote database " + db.getName() , e); + if (isInTest) { + // ignore the exception + return new ArrayList
(); + } else { + throw newMetaException(e); + } + } + } + + private List
getTableObjectsInternal(GetTablesRequest req) + throws MetaException, InvalidOperationException, UnknownDBException { + if (isInTest) { + assertClientHasCapability(req.getCapabilities(), ClientCapability.TEST_CAPABILITY, + "Hive tests", "get_table_objects_by_name_req"); + } + + GetProjectionsSpec projectionsSpec = req.getProjectionSpec(); + if (projectionsSpec != null) { + if (!projectionsSpec.isSetFieldList() && (projectionsSpec.isSetIncludeParamKeyPattern() || + projectionsSpec.isSetExcludeParamKeyPattern())) { + throw new InvalidOperationException("Include and Exclude Param key are not supported."); + } + } + + String dbName = req.getDbName(); + String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf); + List
tables = new ArrayList<>(); + int tableBatchSize = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX); + List tableNames = req.getTblNames(); + if(req.getTablesPattern() != null) { + tables = ms.getTableObjectsByName(catName, dbName, tableNames, projectionsSpec, req.getTablesPattern()); + } else { + if (tableNames == null) { + throw new InvalidOperationException(dbName + " cannot find null tables"); + } + + // The list of table names could contain duplicates. RawStore.getTableObjectsByName() + // only guarantees returning no duplicate table objects in one batch. If we need + // to break into multiple batches, remove duplicates first. + List distinctTableNames = tableNames; + if (distinctTableNames.size() > tableBatchSize) { + List lowercaseTableNames = new ArrayList<>(); + for (String tableName : tableNames) { + lowercaseTableNames.add(normalizeIdentifier(tableName)); + } + distinctTableNames = new ArrayList<>(new HashSet<>(lowercaseTableNames)); + } + + int startIndex = 0; + // Retrieve the tables from the metastore in batches. Some databases like + // Oracle cannot have over 1000 expressions in a in-list + while (startIndex < distinctTableNames.size()) { + int endIndex = Math.min(startIndex + tableBatchSize, distinctTableNames.size()); + tables.addAll(ms.getTableObjectsByName(catName, dbName, distinctTableNames.subList( + startIndex, endIndex), projectionsSpec, null)); + startIndex = endIndex; + } + } + for (Table t : tables) { + if (t.getParameters() != null && MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) { + assertClientHasCapability(req.getCapabilities(), ClientCapability.INSERT_ONLY_TABLES, + "insert-only tables", "get_table_req"); + } + } + + tables = FilterUtils.filterTablesIfEnabled(filterHook != null, filterHook, tables); + return tables; + } + + private void assertClientHasCapability(ClientCapabilities client, + ClientCapability value, String what, String call) throws MetaException { + if (!doesClientHaveCapability(client, value)) { + throw new MetaException("Your client does not appear to support " + what + ". To skip" + + " capability checks, please set " + MetastoreConf.ConfVars.CAPABILITY_CHECK.toString() + + " to false. This setting can be set globally, or on the client for the current" + + " metastore session. Note that this may lead to incorrect results, data loss," + + " undefined behavior, etc. if your client is actually incompatible. You can also" + + " specify custom client capabilities via " + call + " API."); + } + } + + private boolean doesClientHaveCapability(ClientCapabilities client, ClientCapability value) { + if (!MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.CAPABILITY_CHECK)) { + return true; + } + return (client != null && client.isSetValues() && client.getValues().contains(value)); + } + + private List getTableMeta(GetTableNamesRequest getNamesReq) throws TException { + String catName = getNamesReq.catName; + String dbname = getNamesReq.dbName; + List tblTypes = null; + if (getNamesReq.tableType != null) { + tblTypes = Arrays.asList(getNamesReq.tableType.split(",")); + } + List t = ms.getTableMeta(catName, dbname, getNamesReq.pattern, tblTypes); + t = FilterUtils.filterTableMetasIfEnabled(filterHook != null, filterHook, t); + return filterReadableTables(catName, t); + } + + /** + * filters out the table meta for which read database access is not granted + * @param catName catalog name + * @param tableMetas list of table metas + * @return filtered list of table metas + * @throws RuntimeException + * @throws NoSuchObjectException + */ + private List filterReadableTables(String catName, List tableMetas) + throws RuntimeException, NoSuchObjectException { + List finalT = new ArrayList<>(); + Map databaseNames = new HashMap(); + for (TableMeta tableMeta : tableMetas) { + String fullDbName = prependCatalogToDbName(catName, tableMeta.getDbName(), conf); + if (databaseNames.get(fullDbName) == null) { + boolean isExecptionThrown = false; + try { + fireReadDatabasePreEvent(fullDbName); + } catch (MetaException e) { + isExecptionThrown = true; + } + databaseNames.put(fullDbName, isExecptionThrown); + } + if (!databaseNames.get(fullDbName)) { + finalT.add(tableMeta); + } + } + return finalT; + } + + /** + * Fire a pre-event for read database operation, if there are any + * pre-event listeners registered + */ + private void fireReadDatabasePreEvent(final String name) + throws MetaException, RuntimeException, NoSuchObjectException { + Supplier supplier = () -> { + String[] parsedDbName = parseDbName(name, conf); + Database db = null; + try { + db = handler.get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); + if (db == null) { + throw new NoSuchObjectException("Database: " + name + " not found"); + } + } catch(MetaException | NoSuchObjectException e) { + throw new RuntimeException(e); + } + return new PreReadDatabaseEvent(db, handler); + }; + ((HMSHandler) handler).firePreEvent(supplier); + } + + + private List getTableNames(GetTableNamesRequest getNamesReq) throws TException { + String catName = getNamesReq.catName; + String dbname = getNamesReq.dbName; + try { + Database database = handler.get_database_core(catName, dbname); + if (isDatabaseRemote(database)) { + List ret = DataConnectorProviderFactory.getDataConnectorProvider(database).getTableNames(); + return FilterUtils.filterTableNamesIfEnabled(filterHook != null, filterHook, catName, dbname, ret); + } + } catch (NoSuchObjectException nse) { + throw new UnknownDBException("Could not find database " + DatabaseName.getQualified(catName, dbname)); + } + + List names; + if (getNamesReq.filter != null) { + names = ms.listTableNamesByFilter(catName, dbname, getNamesReq.filter, getNamesReq.limit); + } else if (getNamesReq.tableType != null) { + names = ms.getTables(catName, dbname, getNamesReq.pattern, + TableType.valueOf(getNamesReq.tableType), -1); + } else if (getNamesReq.pattern != null) { + names = ms.getTables(catName, dbname, getNamesReq.pattern); + } else { + names = ms.getAllTables(catName, dbname); + } + if (filterHook != null && !names.isEmpty()) { + String tables = String.join("|", names); + List tableMetas = ms.getTableMeta(catName, dbname, tables, null); + if (tableMetas == null || tableMetas.isEmpty()) { + return new ArrayList<>(); + } + List filteredTableMetas = FilterUtils.filterTableMetasIfEnabled(filterHook != null, filterHook, tableMetas); + return filteredTableMetas.stream().map(TableMeta::getTableName).collect(Collectors.toList()); + } + return names; + } + + public static class GetTableReq extends TAbstractBase { + private final Req request; + private boolean rawTable; + public GetTableReq(Req req) { + this.request = req; + } + public Req getRequest() { + return request; + } + } + + public static class GetTableNamesRequest extends TAbstractBase { + private final String catName; + private final String dbName; + private String filter; + private String pattern; + private String tableType; + private short limit; + private boolean forTableMeta; + private GetTableNamesRequest(String catalog, String database) { + this.catName = catalog; + this.dbName = database; + } + public static GetTableNamesRequest fromDatabase(String database, Configuration configuration) + throws MetaException, UnknownDBException { + String[] parsedDbName = parseDbName(database, configuration); + if (parsedDbName[CAT_NAME] == null || parsedDbName[CAT_NAME].isEmpty() || + parsedDbName[DB_NAME] == null || parsedDbName[DB_NAME].isEmpty()) { + throw new UnknownDBException("DB name is null or empty"); + } + return new GetTableNamesRequest(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); + } + public GetTableNamesRequest byFilter(String filter, short limit) throws InvalidOperationException{ + if (filter == null) { + throw new InvalidOperationException(filter + " cannot apply null filter"); + } + this.filter = filter; + this.limit = limit; + return this; + } + + public GetTableNamesRequest byType(String tableType, String pattern) { + this.tableType = tableType; + this.pattern = pattern; + return this; + } + + public GetTableNamesRequest byPattern(String pattern) { + return byType(null, pattern); + } + + public static GetTableNamesRequest forTableMeta(String dbName, Configuration configuration) { + // get_table_meta accepts null database name + String catName; + String database = null; + if (dbName == null) { + catName = getDefaultCatalog(configuration); + } else { + String[] parsedDbName = parseDbName(dbName, configuration); + catName = parsedDbName[CAT_NAME]; + database = parsedDbName[DB_NAME]; + } + GetTableNamesRequest req = new GetTableNamesRequest(catName, database); + req.forTableMeta = true; + return req; + } + } + + public record GetTableResult(List result, boolean success) implements Result { + + } + + public static List getTables(Runnable preHook, + IHMSHandler handler, R req, + Consumer, Exception>> postHook) throws TException { + if (preHook != null) { + preHook.run(); + } + List tables = null; + Exception ex = null; + try { + GetTableReq internalRequest = new GetTableReq<>(req); + GetTableHandler getTablesHandler = + AbstractRequestHandler.offer(handler, internalRequest); + tables = getTablesHandler.getResult().result(); + return tables; + } catch (Exception e) { + ex = e; + throw handleException(e).defaultTException(); + } finally { + if (postHook != null) { + Pair, Exception> result = Pair.of(tables, ex); + postHook.accept(result); + } + } + } + + public static Table getTable(Runnable preHook, + IHMSHandler handler, GetTableRequest request, boolean rawTable, + Consumer> postHook) + throws NoSuchObjectException, MetaException { + if (preHook != null) { + preHook.run(); + } + Table t = null; + Exception ex = null; + try { + GetTableReq internalRequest = new GetTableReq<>(request); + internalRequest.rawTable = rawTable; + GetTableHandler getTableHandler = + AbstractRequestHandler.offer(handler, internalRequest); + List
tables = getTableHandler.getResult().result(); + t = tables.getFirst(); + return t; + } catch (Exception e) { + ex = e; + throw handleException(e).throwIfInstance(NoSuchObjectException.class, MetaException.class) + .defaultMetaException(); + } finally { + if (postHook != null) { + Pair result = Pair.of(t, ex); + postHook.accept(result); + } + } + } +}